summaryrefslogtreecommitdiffstats
path: root/searchcore
diff options
context:
space:
mode:
authorHenning Baldersheim <balder@yahoo-inc.com>2019-09-26 12:16:12 +0000
committerHenning Baldersheim <balder@yahoo-inc.com>2019-09-26 12:16:12 +0000
commit1c3aca165311d9a2d0cb4328578b2b08f281f32d (patch)
treeb8a77bf9767ef142008a5cdcdbccf1e75e20caf1 /searchcore
parent4fadd99eca043ceb8fbc0d796bc6542a69de6ddd (diff)
Remove fdispatch
Diffstat (limited to 'searchcore')
-rw-r--r--searchcore/CMakeLists.txt6
-rw-r--r--searchcore/src/apps/fdispatch/.gitignore1
-rw-r--r--searchcore/src/apps/fdispatch/CMakeLists.txt15
-rw-r--r--searchcore/src/apps/fdispatch/fdispatch.cpp208
-rw-r--r--searchcore/src/tests/fdispatch/fnet_search/.gitignore2
-rw-r--r--searchcore/src/tests/fdispatch/fnet_search/CMakeLists.txt18
-rw-r--r--searchcore/src/tests/fdispatch/fnet_search/search_coverage_test.cpp141
-rw-r--r--searchcore/src/tests/fdispatch/fnet_search/search_path_test.cpp123
-rw-r--r--searchcore/src/tests/fdispatch/randomrow/.gitignore1
-rw-r--r--searchcore/src/tests/fdispatch/randomrow/CMakeLists.txt10
-rw-r--r--searchcore/src/tests/fdispatch/randomrow/randomrow_test.cpp99
-rw-r--r--searchcore/src/vespa/searchcore/fdispatch/.gitignore1
-rw-r--r--searchcore/src/vespa/searchcore/fdispatch/OWNERS2
-rw-r--r--searchcore/src/vespa/searchcore/fdispatch/common/CMakeLists.txt9
-rw-r--r--searchcore/src/vespa/searchcore/fdispatch/common/appcontext.cpp56
-rw-r--r--searchcore/src/vespa/searchcore/fdispatch/common/appcontext.h39
-rw-r--r--searchcore/src/vespa/searchcore/fdispatch/common/properties.h19
-rw-r--r--searchcore/src/vespa/searchcore/fdispatch/common/rpc.cpp92
-rw-r--r--searchcore/src/vespa/searchcore/fdispatch/common/rpc.h51
-rw-r--r--searchcore/src/vespa/searchcore/fdispatch/common/search.cpp221
-rw-r--r--searchcore/src/vespa/searchcore/fdispatch/common/search.h393
-rw-r--r--searchcore/src/vespa/searchcore/fdispatch/common/stdincl.h12
-rw-r--r--searchcore/src/vespa/searchcore/fdispatch/common/timestat.cpp116
-rw-r--r--searchcore/src/vespa/searchcore/fdispatch/common/timestat.h215
-rw-r--r--searchcore/src/vespa/searchcore/fdispatch/program/.gitignore13
-rw-r--r--searchcore/src/vespa/searchcore/fdispatch/program/CMakeLists.txt11
-rw-r--r--searchcore/src/vespa/searchcore/fdispatch/program/description.html3
-rw-r--r--searchcore/src/vespa/searchcore/fdispatch/program/docsumadapter.cpp114
-rw-r--r--searchcore/src/vespa/searchcore/fdispatch/program/docsumadapter.h52
-rw-r--r--searchcore/src/vespa/searchcore/fdispatch/program/engineadapter.cpp77
-rw-r--r--searchcore/src/vespa/searchcore/fdispatch/program/engineadapter.h46
-rw-r--r--searchcore/src/vespa/searchcore/fdispatch/program/fdispatch.cpp397
-rw-r--r--searchcore/src/vespa/searchcore/fdispatch/program/fdispatch.h111
-rw-r--r--searchcore/src/vespa/searchcore/fdispatch/program/rpc.cpp113
-rw-r--r--searchcore/src/vespa/searchcore/fdispatch/program/rpc.h28
-rw-r--r--searchcore/src/vespa/searchcore/fdispatch/program/searchadapter.cpp109
-rw-r--r--searchcore/src/vespa/searchcore/fdispatch/program/searchadapter.h48
-rw-r--r--searchcore/src/vespa/searchcore/fdispatch/search/.gitignore3
-rw-r--r--searchcore/src/vespa/searchcore/fdispatch/search/CMakeLists.txt21
-rw-r--r--searchcore/src/vespa/searchcore/fdispatch/search/child_info.h22
-rw-r--r--searchcore/src/vespa/searchcore/fdispatch/search/configdesc.cpp344
-rw-r--r--searchcore/src/vespa/searchcore/fdispatch/search/configdesc.h374
-rw-r--r--searchcore/src/vespa/searchcore/fdispatch/search/dataset_base.cpp294
-rw-r--r--searchcore/src/vespa/searchcore/fdispatch/search/dataset_base.h227
-rw-r--r--searchcore/src/vespa/searchcore/fdispatch/search/datasetcollection.cpp266
-rw-r--r--searchcore/src/vespa/searchcore/fdispatch/search/datasetcollection.h87
-rw-r--r--searchcore/src/vespa/searchcore/fdispatch/search/engine_base.cpp417
-rw-r--r--searchcore/src/vespa/searchcore/fdispatch/search/engine_base.h195
-rw-r--r--searchcore/src/vespa/searchcore/fdispatch/search/fnet_dataset.cpp144
-rw-r--r--searchcore/src/vespa/searchcore/fdispatch/search/fnet_dataset.h66
-rw-r--r--searchcore/src/vespa/searchcore/fdispatch/search/fnet_engine.cpp231
-rw-r--r--searchcore/src/vespa/searchcore/fdispatch/search/fnet_engine.h112
-rw-r--r--searchcore/src/vespa/searchcore/fdispatch/search/fnet_search.cpp1572
-rw-r--r--searchcore/src/vespa/searchcore/fdispatch/search/fnet_search.h375
-rw-r--r--searchcore/src/vespa/searchcore/fdispatch/search/mergehits.cpp283
-rw-r--r--searchcore/src/vespa/searchcore/fdispatch/search/mergehits.h158
-rw-r--r--searchcore/src/vespa/searchcore/fdispatch/search/nodemanager.cpp438
-rw-r--r--searchcore/src/vespa/searchcore/fdispatch/search/nodemanager.h89
-rw-r--r--searchcore/src/vespa/searchcore/fdispatch/search/plain_dataset.cpp560
-rw-r--r--searchcore/src/vespa/searchcore/fdispatch/search/plain_dataset.h216
-rw-r--r--searchcore/src/vespa/searchcore/fdispatch/search/poss_count.h16
-rw-r--r--searchcore/src/vespa/searchcore/fdispatch/search/query.cpp122
-rw-r--r--searchcore/src/vespa/searchcore/fdispatch/search/query.h75
-rw-r--r--searchcore/src/vespa/searchcore/fdispatch/search/querycacheutil.cpp156
-rw-r--r--searchcore/src/vespa/searchcore/fdispatch/search/querycacheutil.h151
-rw-r--r--searchcore/src/vespa/searchcore/fdispatch/search/rowstate.cpp90
-rw-r--r--searchcore/src/vespa/searchcore/fdispatch/search/rowstate.h60
-rw-r--r--searchcore/src/vespa/searchcore/fdispatch/search/search_path.cpp114
-rw-r--r--searchcore/src/vespa/searchcore/fdispatch/search/search_path.h58
69 files changed, 0 insertions, 10308 deletions
diff --git a/searchcore/CMakeLists.txt b/searchcore/CMakeLists.txt
index 45113e95df4..ee262a5fa5c 100644
--- a/searchcore/CMakeLists.txt
+++ b/searchcore/CMakeLists.txt
@@ -21,9 +21,6 @@ vespa_define_module(
LIBS
src/vespa/searchcore/config
- src/vespa/searchcore/fdispatch/common
- src/vespa/searchcore/fdispatch/program
- src/vespa/searchcore/fdispatch/search
src/vespa/searchcore/grouping
src/vespa/searchcore/proton/attribute
src/vespa/searchcore/proton/bucketdb
@@ -46,7 +43,6 @@ vespa_define_module(
src/vespa/searchcore/util
APPS
- src/apps/fdispatch
src/apps/proton
src/apps/tests
src/apps/verify_ranksetup
@@ -56,8 +52,6 @@ vespa_define_module(
src/apps/vespa-transactionlog-inspect
TESTS
- src/tests/fdispatch/randomrow
- src/tests/fdispatch/fnet_search
src/tests/grouping
src/tests/proton/attribute
src/tests/proton/attribute/attribute_aspect_delayer
diff --git a/searchcore/src/apps/fdispatch/.gitignore b/searchcore/src/apps/fdispatch/.gitignore
deleted file mode 100644
index 36a9a584d01..00000000000
--- a/searchcore/src/apps/fdispatch/.gitignore
+++ /dev/null
@@ -1 +0,0 @@
-vespa-dispatch-bin
diff --git a/searchcore/src/apps/fdispatch/CMakeLists.txt b/searchcore/src/apps/fdispatch/CMakeLists.txt
deleted file mode 100644
index 4399968a761..00000000000
--- a/searchcore/src/apps/fdispatch/CMakeLists.txt
+++ /dev/null
@@ -1,15 +0,0 @@
-# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-vespa_add_executable(searchcore_fdispatch_app
- SOURCES
- fdispatch.cpp
- OUTPUT_NAME vespa-dispatch-bin
- INSTALL sbin
- DEPENDS
- searchcore_fdispatch_program
- searchcore_fdispatch_search
- searchcore_grouping
- searchcore_fdcommon
- searchcore_util
- searchcore_fconfig
- searchlib_searchlib_uca
-)
diff --git a/searchcore/src/apps/fdispatch/fdispatch.cpp b/searchcore/src/apps/fdispatch/fdispatch.cpp
deleted file mode 100644
index 0aa16260737..00000000000
--- a/searchcore/src/apps/fdispatch/fdispatch.cpp
+++ /dev/null
@@ -1,208 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-
-#include <vespa/searchcore/fdispatch/program/fdispatch.h>
-#include <vespa/vespalib/net/state_server.h>
-#include <vespa/vespalib/net/simple_health_producer.h>
-#include <vespa/vespalib/net/simple_metrics_producer.h>
-#include <vespa/searchlib/expression/forcelink.hpp>
-#include <vespa/searchlib/aggregation/forcelink.hpp>
-#include <vespa/vespalib/util/signalhandler.h>
-#include <vespa/fastos/app.h>
-#include <thread>
-#include <getopt.h>
-
-#include <vespa/log/log.h>
-LOG_SETUP("fdispatch");
-
-
-using fdispatch::Fdispatch;
-using vespa::config::search::core::FdispatchrcConfig;
-using namespace std::literals;
-
-extern char FastS_VersionTag[];
-
-class FastS_FDispatchApp : public FastOS_Application
-{
-private:
- FastS_FDispatchApp(const FastS_FDispatchApp &);
- FastS_FDispatchApp& operator=(const FastS_FDispatchApp &);
-
-protected:
- vespalib::string _configId;
-
- bool CheckShutdownFlags () const {
- return (vespalib::SignalHandler::INT.check() || vespalib::SignalHandler::TERM.check());
- }
-
- void Usage();
- bool GetOptions(int *exitCode);
-
-public:
- int Main() override;
- FastS_FDispatchApp();
- ~FastS_FDispatchApp();
-};
-
-
-FastS_FDispatchApp::FastS_FDispatchApp()
-{
-}
-
-
-/**
- * Main program for a dispatcher node.
- */
-int
-FastS_FDispatchApp::Main()
-{
- int exitCode;
-
- forcelink_searchlib_expression();
- forcelink_searchlib_aggregation();
-
- if (!GetOptions(&exitCode)) {
- EV_STOPPING("fdispatch", (exitCode == 0) ? "clean shutdown" : "error");
- return exitCode;
- }
-
- exitCode = 0;
-
- EV_STARTED("fdispatch");
-
- vespalib::SignalHandler::INT.hook();
- vespalib::SignalHandler::TERM.hook();
- vespalib::SignalHandler::PIPE.ignore();
-
- std::unique_ptr<Fdispatch> myfdispatch;
- try {
- myfdispatch.reset(new Fdispatch(_configId));
- } catch (std::exception &ex) {
- LOG(error, "getting config: %s", (const char *) ex.what());
- exitCode = 1;
- EV_STOPPING("fdispatch", "error getting config");
- return exitCode;
- }
-
- try {
-
- if (!myfdispatch->Init()) {
- throw std::runtime_error("myfdispatch->Init(" + _configId + ") failed");
- }
- if (myfdispatch->Failed()) {
- throw std::runtime_error("myfdispatch->Failed()");
- }
- { // main loop scope
- vespalib::SimpleHealthProducer health;
- vespalib::SimpleMetricsProducer metrics;
- vespalib::StateServer stateServer(myfdispatch->getHealthPort(), health, metrics, myfdispatch->getComponentConfig());
- while (!CheckShutdownFlags()) {
- if (myfdispatch->Failed()) {
- throw std::runtime_error("myfdispatch->Failed()");
- }
- std::this_thread::sleep_for(100ms);
- if (!myfdispatch->CheckTempFail())
- break;
- }
- } // main loop scope
- if (myfdispatch->Failed()) {
- throw std::runtime_error("myfdispatch->Failed()");
- }
- } catch (std::runtime_error &e) {
- LOG(warning, "got std::runtime_error during init: %s", e.what());
- exitCode = 1;
- } catch (std::exception &e) {
- LOG(error, "got exception during init: %s", e.what());
- exitCode = 1;
- }
-
- LOG(debug, "Deleting fdispatch");
- myfdispatch.reset();
- LOG(debug, "COMPLETION: Exiting");
- EV_STOPPING("fdispatch", (exitCode == 0) ? "clean shutdown" : "error");
- return exitCode;
-}
-
-
-bool
-FastS_FDispatchApp::GetOptions(int *exitCode)
-{
- int errflg = 0;
- int c;
- const char *optArgument;
- int longopt_index; /* Shows which long option was used */
- static struct option longopts[] = {
- { "config-id", 1, NULL, 0 },
- { NULL, 0, NULL, 0 }
- };
- enum longopts_enum {
- LONGOPT_CONFIGID
- };
- int optIndex = 1; // Start with argument 1
- while ((c = GetOptLong("c:", optArgument, optIndex, longopts, &longopt_index)) != -1) {
- switch (c) {
- case 0:
- switch (longopt_index) {
- case LONGOPT_CONFIGID:
- break;
- default:
- if (optArgument != NULL) {
- LOG(info, "longopt %s with arg %s", longopts[longopt_index].name, optArgument);
- } else {
- LOG(info, "longopt %s", longopts[longopt_index].name);
- }
- break;
- }
- break;
- case 'c':
- _configId = optArgument;
- break;
- case '?':
- default:
- errflg++;
- }
- }
- if (errflg) {
- Usage();
- *exitCode = 1;
- return false;
- }
- return true;
-}
-
-void
-FastS_FDispatchApp::Usage()
-{
- printf("FAST Search - fdispatch %s\n", FastS_VersionTag);
- printf("\n"
- "USAGE:\n"
- "\n"
- "fdispatch [-C fsroot] [-c rcFile] [-P preHttPort] [-V] [-w FFF]\n"
- "\n"
- " -C fsroot Fast Search's root directory\n"
- " (default /usr/fastsearch/fastserver4)\n"
- " -c rcFile fdispatchrc file (default FASTSEARCHROOT/etc/fdispatchrc)\n"
- " -P preHttPort pre-allocated socket number for http service\n"
- " -V show version and exit\n"
- " -w FFF hex value (max 32 bit) for the Verbose mask\n"
- "\n");
-}
-
-
-FastS_FDispatchApp::~FastS_FDispatchApp()
-{
-}
-
-
-int
-main(int argc, char **argv)
-{
- FastS_FDispatchApp app;
- int retval;
-
- // Maybe this should be handeled by FastOS
- setlocale(LC_ALL, "C");
-
- retval = app.Entry(argc, argv);
-
- return retval;
-}
diff --git a/searchcore/src/tests/fdispatch/fnet_search/.gitignore b/searchcore/src/tests/fdispatch/fnet_search/.gitignore
deleted file mode 100644
index b525d6fcd38..00000000000
--- a/searchcore/src/tests/fdispatch/fnet_search/.gitignore
+++ /dev/null
@@ -1,2 +0,0 @@
-searchcore_search_path_test_app
-searchcore_search_coverage_test_app
diff --git a/searchcore/src/tests/fdispatch/fnet_search/CMakeLists.txt b/searchcore/src/tests/fdispatch/fnet_search/CMakeLists.txt
deleted file mode 100644
index c4e1608d6da..00000000000
--- a/searchcore/src/tests/fdispatch/fnet_search/CMakeLists.txt
+++ /dev/null
@@ -1,18 +0,0 @@
-# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-vespa_add_executable(searchcore_search_path_test_app TEST
- SOURCES
- search_path_test.cpp
- DEPENDS
- searchcore_fdispatch_search
-)
-vespa_add_test(NAME searchcore_search_path_test_app COMMAND searchcore_search_path_test_app)
-
-vespa_add_executable(searchcore_search_coverage_test_app TEST
- SOURCES
- search_coverage_test.cpp
- DEPENDS
- searchcore_fdispatch_search
- searchcore_fdcommon
- searchcore_grouping
-)
-vespa_add_test(NAME searchcore_search_coverage_test_app COMMAND searchcore_search_coverage_test_app)
diff --git a/searchcore/src/tests/fdispatch/fnet_search/search_coverage_test.cpp b/searchcore/src/tests/fdispatch/fnet_search/search_coverage_test.cpp
deleted file mode 100644
index d598583437d..00000000000
--- a/searchcore/src/tests/fdispatch/fnet_search/search_coverage_test.cpp
+++ /dev/null
@@ -1,141 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-#include <vespa/vespalib/testkit/testapp.h>
-
-#include <vespa/searchcore/fdispatch/search/fnet_search.h>
-#include <vespa/searchlib/engine/searchreply.h>
-
-#include <vespa/log/log.h>
-LOG_SETUP("search_coverage_test");
-
-using namespace fdispatch;
-using search::engine::SearchReply;
-
-std::vector<FastS_FNET_SearchNode>
-createNodes(uint32_t count) {
- std::vector<FastS_FNET_SearchNode> nodes;
- nodes.reserve(count);
- for (uint32_t partid(0); partid < count; partid++) {
- nodes.emplace_back(nullptr, partid);
- }
- return nodes;
-}
-
-void
-query(FastS_FNET_SearchNode & node) {
- node.DirtySetChannelOnlyForTesting((FNET_Channel *) 1);
-}
-
-void
-respond(FastS_FNET_SearchNode & node, size_t covered, size_t active, size_t soonActive, uint32_t degradeReason) {
- node._qresult = new FS4Packet_QUERYRESULTX();
- node._qresult->_coverageDocs = covered;
- node._qresult->_activeDocs = active;
- node._qresult->_soonActiveDocs = soonActive;
- node._qresult->_coverageDegradeReason = degradeReason;
-}
-
-void
-respond(FastS_FNET_SearchNode & node, size_t covered, size_t active, size_t soonActive) {
- respond(node, covered, active, soonActive, 0);
-}
-
-void disconnectNodes(std::vector<FastS_FNET_SearchNode> & nodes) {
- for (auto & node : nodes) {
- node.DirtySetChannelOnlyForTesting(nullptr);
- }
-}
-TEST("testCoverageWhenAllNodesAreUp") {
- std::vector<FastS_FNET_SearchNode> nodes = createNodes(4);
- for (auto & node : nodes) {
- query(node);
- respond(node, 25, 30, 50);
- }
- FastS_SearchInfo si = FastS_FNET_Search::computeCoverage(nodes, 1, false);
- EXPECT_EQUAL(4u, si._nodesQueried);
- EXPECT_EQUAL(4u, si._nodesReplied);
- EXPECT_EQUAL(100u, si._coverageDocs);
- EXPECT_EQUAL(120u, si._activeDocs);
- EXPECT_EQUAL(200u, si._soonActiveDocs);
- EXPECT_EQUAL(0u, si._degradeReason);
- disconnectNodes(nodes);
-}
-
-TEST("testCoverageWhenNoNodesAreUp") {
- std::vector<FastS_FNET_SearchNode> nodes = createNodes(4);
- for (auto & node : nodes) {
- query(node);
- }
- FastS_SearchInfo si = FastS_FNET_Search::computeCoverage(nodes, 1, false);
- EXPECT_EQUAL(4u, si._nodesQueried);
- EXPECT_EQUAL(0u, si._nodesReplied);
- EXPECT_EQUAL(0u, si._coverageDocs);
- EXPECT_EQUAL(0u, si._activeDocs);
- EXPECT_EQUAL(0u, si._soonActiveDocs);
- EXPECT_EQUAL(SearchReply::Coverage::TIMEOUT, si._degradeReason);
- disconnectNodes(nodes);
-}
-
-TEST("testCoverageWhenNoNodesAreUpWithAdaptiveTimeout") {
- std::vector<FastS_FNET_SearchNode> nodes = createNodes(4);
- for (auto & node : nodes) {
- query(node);
- }
- FastS_SearchInfo si = FastS_FNET_Search::computeCoverage(nodes, 1, true);
- EXPECT_EQUAL(4u, si._nodesQueried);
- EXPECT_EQUAL(0u, si._nodesReplied);
- EXPECT_EQUAL(0u, si._coverageDocs);
- EXPECT_EQUAL(0u, si._activeDocs);
- EXPECT_EQUAL(0u, si._soonActiveDocs);
- EXPECT_EQUAL(SearchReply::Coverage::ADAPTIVE_TIMEOUT, si._degradeReason);
- disconnectNodes(nodes);
-}
-
-TEST("testCoverageWhen1NodesIsDown") {
- std::vector<FastS_FNET_SearchNode> nodes = createNodes(4);
- for (auto & node : nodes) {
- query(node);
- }
- respond(nodes[0], 25, 30, 50);
- respond(nodes[2], 25, 30, 50);
- respond(nodes[3], 25, 30, 50);
-
- FastS_SearchInfo si = FastS_FNET_Search::computeCoverage(nodes, 1, false);
- EXPECT_EQUAL(4u, si._nodesQueried);
- EXPECT_EQUAL(3u, si._nodesReplied);
- EXPECT_EQUAL(75u, si._coverageDocs);
- EXPECT_EQUAL(120u, si._activeDocs);
- EXPECT_EQUAL(200u, si._soonActiveDocs);
- EXPECT_EQUAL(SearchReply::Coverage::TIMEOUT, si._degradeReason);
-
- // Do not trigger dirty magic when you still have enough coverage in theory
- si = FastS_FNET_Search::computeCoverage(nodes, 2, false);
- EXPECT_EQUAL(4u, si._nodesQueried);
- EXPECT_EQUAL(3u, si._nodesReplied);
- EXPECT_EQUAL(75u, si._coverageDocs);
- EXPECT_EQUAL(90u, si._activeDocs);
- EXPECT_EQUAL(150u, si._soonActiveDocs);
- EXPECT_EQUAL(0u, si._degradeReason);
- disconnectNodes(nodes);
-}
-
-TEST("testCoverageWhen1NodeDoesnotReplyWithAdaptiveTimeout") {
- std::vector<FastS_FNET_SearchNode> nodes = createNodes(4);
- for (auto & node : nodes) {
- query(node);
- }
- respond(nodes[0], 25, 30, 50);
- respond(nodes[2], 25, 30, 50);
- respond(nodes[3], 25, 30, 50);
-
- FastS_SearchInfo si = FastS_FNET_Search::computeCoverage(nodes, 1, true);
- EXPECT_EQUAL(4u, si._nodesQueried);
- EXPECT_EQUAL(3u, si._nodesReplied);
- EXPECT_EQUAL(75u, si._coverageDocs);
- EXPECT_EQUAL(120u, si._activeDocs);
- EXPECT_EQUAL(200u, si._soonActiveDocs);
- EXPECT_EQUAL(SearchReply::Coverage::ADAPTIVE_TIMEOUT, si._degradeReason);
- disconnectNodes(nodes);
-}
-
-
-TEST_MAIN() { TEST_RUN_ALL(); }
diff --git a/searchcore/src/tests/fdispatch/fnet_search/search_path_test.cpp b/searchcore/src/tests/fdispatch/fnet_search/search_path_test.cpp
deleted file mode 100644
index b62fb8d14f2..00000000000
--- a/searchcore/src/tests/fdispatch/fnet_search/search_path_test.cpp
+++ /dev/null
@@ -1,123 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-#include <vespa/log/log.h>
-LOG_SETUP("search_path_test");
-#include <vespa/vespalib/testkit/testapp.h>
-
-#include <vespa/searchcore/fdispatch/search/search_path.h>
-#include <vespa/searchcore/fdispatch/search/fnet_search.h>
-#include <iostream>
-
-using namespace fdispatch;
-
-template <typename T>
-vespalib::string
-toString(const T &val)
-{
- std::ostringstream oss;
- oss << "[";
- bool first = true;
- for (auto v : val) {
- if (!first) oss << ",";
- oss << v;
- first = false;
- }
- oss << "]";
- return oss.str();
-}
-
-void
-assertParts(const std::vector<size_t> &exp, const SearchPath::NodeList &act)
-{
- std::string expStr = toString(exp);
- std::string actStr = toString(act);
- std::cout << "assertParts(" << expStr << "," << actStr << ")" << std::endl;
- EXPECT_EQUAL(expStr, actStr);
-}
-
-void
-assertElement(const std::vector<size_t> &parts, size_t row, const SearchPath::Element &elem)
-{
- assertParts(parts, elem.nodes());
- EXPECT_TRUE(elem.hasRow());
- EXPECT_EQUAL(row, elem.row());
-}
-
-void
-assertElement(const std::vector<size_t> &parts, const SearchPath::Element &elem)
-{
- assertParts(parts, elem.nodes());
- EXPECT_FALSE(elem.hasRow());
-}
-
-void
-assertSinglePath(const std::vector<size_t> &parts, const vespalib::string &spec, size_t numNodes=0)
-{
- SearchPath p(spec, numNodes);
- EXPECT_EQUAL(1u, p.elements().size());
- assertElement(parts, p.elements().front());
-}
-
-void
-assertSinglePath(const std::vector<size_t> &parts, size_t row, const vespalib::string &spec, size_t numNodes=0)
-{
- SearchPath p(spec, numNodes);
- EXPECT_EQUAL(1u, p.elements().size());
- assertElement(parts, row, p.elements().front());
-}
-
-TEST("requireThatSinglePartCanBeSpecified")
-{
- assertSinglePath({0}, "0/");
-}
-
-TEST("requireThatMultiplePartsCanBeSpecified")
-{
- assertSinglePath({1,3,5}, "1,3,5/");
-}
-
-TEST("requireThatRangePartsCanBeSpecified")
-{
- assertSinglePath({1,2,3}, "[1,4>/", 6);
-}
-
-TEST("requireThatAllPartsCanBeSpecified")
-{
- assertSinglePath({0,1,2,3}, "*/", 4);
-}
-
-TEST("requireThatRowCanBeSpecified")
-{
- assertSinglePath({1}, 2, "1/2");
-}
-
-TEST("requireThatMultipleSimpleElementsCanBeSpecified")
-{
- SearchPath p("0/1;2/3", 3);
- EXPECT_EQUAL(2u, p.elements().size());
- assertElement({0}, 1, p.elements()[0]);
- assertElement({2}, 3, p.elements()[1]);
-}
-
-TEST("requireThatMultipleComplexElementsCanBeSpecified")
-{
- SearchPath p("0,2,4/1;1,3,5/3", 6);
- EXPECT_EQUAL(2u, p.elements().size());
- assertElement({0,2,4}, 1, p.elements()[0]);
- assertElement({1,3,5}, 3, p.elements()[1]);
-}
-
-TEST("requireThatMultipleElementsWithoutRowsCanBeSpecified")
-{
- SearchPath p("0/;1/", 2);
- EXPECT_EQUAL(2u, p.elements().size());
- assertElement({0}, p.elements()[0]);
- assertElement({1}, p.elements()[1]);
-}
-
-TEST("require that sizeof FastS_FNET_SearchNode is reasonable")
-{
- EXPECT_EQUAL(232u, sizeof(FastS_FNET_SearchNode));
- EXPECT_EQUAL(40u, sizeof(search::common::SortDataIterator));
-}
-
-TEST_MAIN() { TEST_RUN_ALL(); }
diff --git a/searchcore/src/tests/fdispatch/randomrow/.gitignore b/searchcore/src/tests/fdispatch/randomrow/.gitignore
deleted file mode 100644
index bfe075b287a..00000000000
--- a/searchcore/src/tests/fdispatch/randomrow/.gitignore
+++ /dev/null
@@ -1 +0,0 @@
-searchcore_randomrow_test_app
diff --git a/searchcore/src/tests/fdispatch/randomrow/CMakeLists.txt b/searchcore/src/tests/fdispatch/randomrow/CMakeLists.txt
deleted file mode 100644
index 2b05af6a2f6..00000000000
--- a/searchcore/src/tests/fdispatch/randomrow/CMakeLists.txt
+++ /dev/null
@@ -1,10 +0,0 @@
-# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-vespa_add_executable(searchcore_randomrow_test_app TEST
- SOURCES
- randomrow_test.cpp
- DEPENDS
- searchcore_fdispatch_search
- searchcore_util
- searchcore_fdcommon
-)
-vespa_add_test(NAME searchcore_randomrow_test_app COMMAND searchcore_randomrow_test_app)
diff --git a/searchcore/src/tests/fdispatch/randomrow/randomrow_test.cpp b/searchcore/src/tests/fdispatch/randomrow/randomrow_test.cpp
deleted file mode 100644
index c6791436bff..00000000000
--- a/searchcore/src/tests/fdispatch/randomrow/randomrow_test.cpp
+++ /dev/null
@@ -1,99 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-#include <vespa/log/log.h>
-LOG_SETUP("randomrow_test");
-#include <vespa/vespalib/testkit/testapp.h>
-
-#include <vespa/searchcore/fdispatch/search/plain_dataset.h>
-
-using fdispatch::StateOfRows;
-
-TEST("requireThatEmpyStateReturnsRowZero")
-{
- StateOfRows s(1, 1.0, 1000);
- EXPECT_EQUAL(0u, s.getRandomWeightedRow());
- EXPECT_EQUAL(1.0, s.getRowState(0).getAverageSearchTime());
-}
-
-TEST("requireThatDecayWorks")
-{
- constexpr double SMALL = 0.00001;
- StateOfRows s(1, 1.0, 1000);
- s.updateSearchTime(1.0, 0);
- EXPECT_EQUAL(1.0, s.getRowState(0).getAverageSearchTime());
- s.updateSearchTime(2.0, 0);
- EXPECT_APPROX(1.02326, s.getRowState(0).getAverageSearchTime(), SMALL);
- s.updateSearchTime(2.0, 0);
- EXPECT_APPROX(1.04545, s.getRowState(0).getAverageSearchTime(), SMALL);
- s.updateSearchTime(0.1, 0);
- s.updateSearchTime(0.1, 0);
- s.updateSearchTime(0.1, 0);
- s.updateSearchTime(0.1, 0);
- EXPECT_APPROX(0.966667, s.getRowState(0).getAverageSearchTime(), SMALL);
- for (size_t i(0); i < 10000; i++) {
- s.updateSearchTime(1.0, 0);
- }
- EXPECT_APPROX(1.0, s.getRowState(0).getAverageSearchTime(), SMALL);
- s.updateSearchTime(0.1, 0);
- EXPECT_APPROX(0.9991, s.getRowState(0).getAverageSearchTime(), SMALL);
- for (size_t i(0); i < 10000; i++) {
- s.updateSearchTime(0.0, 0);
- }
- EXPECT_APPROX(0.001045, s.getRowState(0).getAverageSearchTime(), SMALL);
-}
-
-TEST("requireWeightedSelectionWorks")
-{
- StateOfRows s(5, 1.0, 1000);
- EXPECT_EQUAL(0u, s.getWeightedNode(-0.1));
- EXPECT_EQUAL(0u, s.getWeightedNode(0.0));
- EXPECT_EQUAL(0u, s.getWeightedNode(0.1));
- EXPECT_EQUAL(1u, s.getWeightedNode(0.2));
- EXPECT_EQUAL(1u, s.getWeightedNode(0.39));
- EXPECT_EQUAL(2u, s.getWeightedNode(0.4));
- EXPECT_EQUAL(3u, s.getWeightedNode(0.6));
- EXPECT_EQUAL(4u, s.getWeightedNode(0.8));
- EXPECT_EQUAL(4u, s.getWeightedNode(2.0));
-}
-
-TEST("requireWeightedSelectionWorksFineWithDifferentWeights")
-{
- StateOfRows s(5, 1.0, 1000);
- s.getRowState(0).setAverageSearchTime(0.1);
- s.getRowState(1).setAverageSearchTime(0.2);
- s.getRowState(2).setAverageSearchTime(0.3);
- s.getRowState(3).setAverageSearchTime(0.4);
- s.getRowState(4).setAverageSearchTime(0.5);
- EXPECT_EQUAL(0.1, s.getRowState(0).getAverageSearchTime());
- EXPECT_EQUAL(0.2, s.getRowState(1).getAverageSearchTime());
- EXPECT_EQUAL(0.3, s.getRowState(2).getAverageSearchTime());
- EXPECT_EQUAL(0.4, s.getRowState(3).getAverageSearchTime());
- EXPECT_EQUAL(0.5, s.getRowState(4).getAverageSearchTime());
- EXPECT_EQUAL(0u, s.getWeightedNode(-0.1));
- EXPECT_EQUAL(0u, s.getWeightedNode(0.0));
- EXPECT_EQUAL(0u, s.getWeightedNode(0.4379));
- EXPECT_EQUAL(1u, s.getWeightedNode(0.4380));
- EXPECT_EQUAL(1u, s.getWeightedNode(0.6569));
- EXPECT_EQUAL(2u, s.getWeightedNode(0.6570));
- EXPECT_EQUAL(2u, s.getWeightedNode(0.8029));
- EXPECT_EQUAL(3u, s.getWeightedNode(0.8030));
- EXPECT_EQUAL(3u, s.getWeightedNode(0.9124));
- EXPECT_EQUAL(4u, s.getWeightedNode(0.9125));
- EXPECT_EQUAL(4u, s.getWeightedNode(2.0));
-}
-
-TEST("require randomness")
-{
- StateOfRows s(3, 1.0, 1000);
- s.getRowState(0).setAverageSearchTime(1.0);
- s.getRowState(1).setAverageSearchTime(1.0);
- s.getRowState(2).setAverageSearchTime(1.0);
- size_t counts[3] = {0,0,0};
- for (size_t i(0); i < 1000; i++) {
- counts[s.getRandomWeightedRow()]++;
- }
- EXPECT_EQUAL(322ul, counts[0]);
- EXPECT_EQUAL(345ul, counts[1]);
- EXPECT_EQUAL(333ul, counts[2]);
-}
-
-TEST_MAIN() { TEST_RUN_ALL(); }
diff --git a/searchcore/src/vespa/searchcore/fdispatch/.gitignore b/searchcore/src/vespa/searchcore/fdispatch/.gitignore
deleted file mode 100644
index fe8e24952a5..00000000000
--- a/searchcore/src/vespa/searchcore/fdispatch/.gitignore
+++ /dev/null
@@ -1 +0,0 @@
-ID
diff --git a/searchcore/src/vespa/searchcore/fdispatch/OWNERS b/searchcore/src/vespa/searchcore/fdispatch/OWNERS
deleted file mode 100644
index f4d47806ed9..00000000000
--- a/searchcore/src/vespa/searchcore/fdispatch/OWNERS
+++ /dev/null
@@ -1,2 +0,0 @@
-baldersheim
-toregge
diff --git a/searchcore/src/vespa/searchcore/fdispatch/common/CMakeLists.txt b/searchcore/src/vespa/searchcore/fdispatch/common/CMakeLists.txt
deleted file mode 100644
index 45261162e93..00000000000
--- a/searchcore/src/vespa/searchcore/fdispatch/common/CMakeLists.txt
+++ /dev/null
@@ -1,9 +0,0 @@
-# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-vespa_add_library(searchcore_fdcommon STATIC
- SOURCES
- appcontext.cpp
- rpc.cpp
- search.cpp
- timestat.cpp
- DEPENDS
-)
diff --git a/searchcore/src/vespa/searchcore/fdispatch/common/appcontext.cpp b/searchcore/src/vespa/searchcore/fdispatch/common/appcontext.cpp
deleted file mode 100644
index a859c53b4e3..00000000000
--- a/searchcore/src/vespa/searchcore/fdispatch/common/appcontext.cpp
+++ /dev/null
@@ -1,56 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-
-#include "appcontext.h"
-#include <chrono>
-
-double FastS_TimeKeeper::GetTime() const {
- using clock = std::chrono::steady_clock;
- using seconds = std::chrono::duration<double, std::ratio<1,1>>;
- return std::chrono::duration_cast<seconds>(clock::now().time_since_epoch()).count();
-}
-
-//---------------------------------------------------------------------
-
-FastS_AppContext::FastS_AppContext()
- : _timeKeeper(),
- _createTime(_timeKeeper.GetTime())
-{
-}
-
-FastS_AppContext::~FastS_AppContext() = default;
-
-FNET_Transport *
-FastS_AppContext::GetFNETTransport()
-{
- return nullptr;
-}
-
-FNET_Scheduler *
-FastS_AppContext::GetFNETScheduler()
-{
- return nullptr;
-}
-
-FastS_NodeManager *
-FastS_AppContext::GetNodeManager()
-{
- return nullptr;
-}
-
-FastS_DataSetCollection *
-FastS_AppContext::GetDataSetCollection()
-{
- return nullptr;
-}
-
-FastOS_ThreadPool *
-FastS_AppContext::GetThreadPool()
-{
- return nullptr;
-}
-
-uint32_t
-FastS_AppContext::getDispatchLevel()
-{
- return 0u;
-}
diff --git a/searchcore/src/vespa/searchcore/fdispatch/common/appcontext.h b/searchcore/src/vespa/searchcore/fdispatch/common/appcontext.h
deleted file mode 100644
index 5235722d3b4..00000000000
--- a/searchcore/src/vespa/searchcore/fdispatch/common/appcontext.h
+++ /dev/null
@@ -1,39 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-
-#pragma once
-
-#include <cstdint>
-
-class FastS_NodeManager;
-class FNET_Transport;
-class FNET_Scheduler;
-class FastS_DataSetCollection;
-class FastOS_ThreadPool;
-
-class FastS_TimeKeeper
-{
-public:
- double GetTime() const;
-};
-
-
-class FastS_AppContext
-{
-private:
- FastS_TimeKeeper _timeKeeper;
- double _createTime;
-
-public:
- FastS_AppContext();
- virtual ~FastS_AppContext();
-
- FastS_TimeKeeper *GetTimeKeeper() { return &_timeKeeper; }
-
- virtual FastS_NodeManager *GetNodeManager();
- virtual FNET_Transport *GetFNETTransport();
- virtual FNET_Scheduler *GetFNETScheduler();
- virtual FastS_DataSetCollection *GetDataSetCollection();
- virtual FastOS_ThreadPool *GetThreadPool();
- virtual uint32_t getDispatchLevel();
-private:
-};
diff --git a/searchcore/src/vespa/searchcore/fdispatch/common/properties.h b/searchcore/src/vespa/searchcore/fdispatch/common/properties.h
deleted file mode 100644
index 783022ac96e..00000000000
--- a/searchcore/src/vespa/searchcore/fdispatch/common/properties.h
+++ /dev/null
@@ -1,19 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-
-#pragma once
-
-class FastS_IProperties
-{
-public:
- /**
- * Destructor. No cleanup needed for base class.
- */
- virtual ~FastS_IProperties() { }
-
- virtual bool IsSet (const char *key) = 0;
- virtual bool BoolVal (const char *key) = 0;
- virtual const char *StrVal (const char *key, const char *def = NULL) = 0;
- virtual int IntVal (const char *key, int def = -1) = 0;
- virtual double DoubleVal(const char *key, double def = 0.0) = 0;
-};
-
diff --git a/searchcore/src/vespa/searchcore/fdispatch/common/rpc.cpp b/searchcore/src/vespa/searchcore/fdispatch/common/rpc.cpp
deleted file mode 100644
index 437482dddd2..00000000000
--- a/searchcore/src/vespa/searchcore/fdispatch/common/rpc.cpp
+++ /dev/null
@@ -1,92 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-
-
-#include "rpc.h"
-#include "appcontext.h"
-#include <vespa/fnet/info.h>
-
-extern char FastS_VersionTag[];
-
-FastS_RPC::FastS_RPC(FastS_AppContext *appCtx) :
- _appCtx(appCtx),
- _transport(),
- _supervisor(&_transport),
- _sbregister(_supervisor, slobrok::ConfiguratorFactory("admin/slobrok.0"))
-{
-}
-
-bool FastS_RPC::Start() {
- return _transport.Start(_appCtx->GetThreadPool());
-}
-
-void FastS_RPC::ShutDown() {
- _transport.ShutDown(true);
-}
-
-bool
-FastS_RPC::Init(int port, const vespalib::string &myHeartbeatId)
-{
- bool rc = true;
-
- char spec[4096];
- snprintf(spec, 4096, "tcp/%d", port);
- rc = rc && _supervisor.Listen(spec);
- if (rc) {
- FRT_ReflectionBuilder rb(&_supervisor);
- RegisterMethods(&rb);
- _sbregister.registerName(myHeartbeatId);
- }
- return rc;
-}
-
-
-void
-FastS_RPC::RegisterMethods(FRT_ReflectionBuilder *rb)
-{
- rb->DefineMethod("fs.admin.getNodeType", "", "s",
- FRT_METHOD(FastS_RPC::RPC_GetNodeType), this);
- rb->MethodDesc("Get string indicating the node type");
- rb->ReturnDesc("type", "node type");
- //---------------------------------------------------------------//
- rb->DefineMethod("fs.admin.getCompileInfo", "", "*",
- FRT_METHOD(FastS_RPC::RPC_GetCompileInfo), this);
- rb->MethodDesc("Obtain compile info for this node");
- rb->ReturnDesc("info", "any number of descriptive strings");
- //---------------------------------------------------------------//
-}
-
-
-void
-FastS_RPC::RPC_GetCompileInfo(FRT_RPCRequest *req)
-{
- FRT_Values &ret = *req->GetReturn();
- ret.AddString("using juniper (api version 2)");
-
-#ifdef NO_MONITOR_LATENCY_CHECK
- ret.AddString("monitor latency check disabled");
-#endif
-#ifdef CUSTOM_TEST_SHUTDOWN
- ret.AddString("Win32: debug shutdown for memory leak detection enabled");
-#endif
- ret.AddString("default transport is 'fnet'");
-
- const char *prefix = "version tag: ";
- uint32_t prefix_len = strlen(prefix);
- uint32_t len = prefix_len + strlen(FastS_VersionTag);
- if (len == prefix_len) {
- ret.AddString("version tag not available");
- } else {
- char *str = ret.AddString(len + 1);
- sprintf(str, "%s%s", prefix, FastS_VersionTag);
- }
-
- ret.AddString("fastos X current");
- ret.AddString(FNET_Info::GetFNETVersion());
-}
-
-
-void
-FastS_RPC::RPC_GetNodeType_Proxy(FRT_RPCRequest *req)
-{
- RPC_GetNodeType(req);
-}
diff --git a/searchcore/src/vespa/searchcore/fdispatch/common/rpc.h b/searchcore/src/vespa/searchcore/fdispatch/common/rpc.h
deleted file mode 100644
index b5519aab22d..00000000000
--- a/searchcore/src/vespa/searchcore/fdispatch/common/rpc.h
+++ /dev/null
@@ -1,51 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-
-#pragma once
-
-#include <vespa/fnet/frt/invokable.h>
-#include <vespa/fnet/frt/supervisor.h>
-#include <vespa/fnet/transport.h>
-#include <vespa/vespalib/stllike/string.h>
-#include <vespa/slobrok/sbregister.h>
-
-class FastS_AppContext;
-
-class FastS_RPC : public FRT_Invokable
-{
-private:
- FastS_RPC(const FastS_RPC &);
- FastS_RPC& operator=(const FastS_RPC &);
-
- FastS_AppContext *_appCtx;
- FNET_Transport _transport;
- FRT_Supervisor _supervisor;
- slobrok::api::RegisterAPI _sbregister;
-
-public:
- FastS_RPC(FastS_AppContext *appCtx);
- ~FastS_RPC() {}
-
- FastS_AppContext *GetAppCtx() { return _appCtx; }
- FRT_Supervisor *GetSupervisor() { return &_supervisor; }
- bool Init(int port, const vespalib::string& myHeartbeatId);
- bool Start();
- void ShutDown();
-
- // Register RPC Methods
-
- virtual void RegisterMethods(FRT_ReflectionBuilder *rb);
-
- // RPC methods implemented here
-
- void RPC_GetCompileInfo(FRT_RPCRequest *req);
- void RPC_GetResultConfig(FRT_RPCRequest *req);
-
- // RPC Proxy Methods
-
- void RPC_GetNodeType_Proxy(FRT_RPCRequest *req);
-
- // RPC methods to be implemented by subclasses
-
- virtual void RPC_GetNodeType(FRT_RPCRequest *req) = 0;
-};
-
diff --git a/searchcore/src/vespa/searchcore/fdispatch/common/search.cpp b/searchcore/src/vespa/searchcore/fdispatch/common/search.cpp
deleted file mode 100644
index 3694fad7882..00000000000
--- a/searchcore/src/vespa/searchcore/fdispatch/common/search.cpp
+++ /dev/null
@@ -1,221 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-
-#include "search.h"
-
-//---------------------------------------------------------------------
-
-FastS_SearchAdapter::FastS_SearchAdapter(FastS_ISearch *search)
- : _search(search)
-{
-}
-
-
-FastS_SearchAdapter::~FastS_SearchAdapter()
-{
-}
-
-
-bool
-FastS_SearchAdapter::IsAsync()
-{
- return _search->IsAsync();
-}
-
-
-uint32_t
-FastS_SearchAdapter::GetDataSetID()
-{
- return _search->GetDataSetID();
-}
-
-
-FastS_SearchInfo *
-FastS_SearchAdapter::GetSearchInfo()
-{
- return _search->GetSearchInfo();
-}
-
-
-FastS_ISearch::RetCode
-FastS_SearchAdapter::SetAsyncArgs(FastS_ISearchOwner *owner)
-{
- return _search->SetAsyncArgs(owner);
-}
-
-
-FastS_ISearch::RetCode
-FastS_SearchAdapter::setSearchRequest(const search::engine::SearchRequest * request)
-{
- return _search->setSearchRequest(request);
-}
-
-
-FastS_ISearch::RetCode
-FastS_SearchAdapter::SetGetDocsumArgs(search::docsummary::GetDocsumArgs *docsumArgs)
-{
- return _search->SetGetDocsumArgs(docsumArgs);
-}
-
-
-FastS_ISearch::RetCode
-FastS_SearchAdapter::Search(uint32_t searchOffset,
- uint32_t maxhits, uint32_t minhits)
-{
- return _search->Search(searchOffset, maxhits, minhits);
-}
-
-
-FastS_ISearch::RetCode
-FastS_SearchAdapter::ProcessQueryDone()
-{
- return _search->ProcessQueryDone();
-}
-
-
-FastS_QueryResult *
-FastS_SearchAdapter::GetQueryResult()
-{
- return _search->GetQueryResult();
-}
-
-
-FastS_ISearch::RetCode
-FastS_SearchAdapter::GetDocsums(const FastS_hitresult *hits, uint32_t hitcnt)
-{
- return _search->GetDocsums(hits, hitcnt);
-}
-
-
-FastS_ISearch::RetCode
-FastS_SearchAdapter::ProcessDocsumsDone()
-{
- return _search->ProcessDocsumsDone();
-}
-
-
-FastS_DocsumsResult *
-FastS_SearchAdapter::GetDocsumsResult()
-{
- return _search->GetDocsumsResult();
-}
-
-
-search::engine::ErrorCode
-FastS_SearchAdapter::GetErrorCode()
-{
- return _search->GetErrorCode();
-}
-
-
-const char *
-FastS_SearchAdapter::GetErrorMessage()
-{
- return _search->GetErrorMessage();
-}
-
-
-void
-FastS_SearchAdapter::Interrupt()
-{
- _search->Interrupt();
-}
-
-
-void
-FastS_SearchAdapter::Free()
-{
- _search->Free();
- delete this;
-}
-
-//---------------------------------------------------------------------
-
-FastS_SyncSearchAdapter::FastS_SyncSearchAdapter(FastS_ISearch *search)
- : FastS_SearchAdapter(search),
- _cond(),
- _waitQuery(false),
- _queryDone(false),
- _waitDocsums(false),
- _docsumsDone(false)
-{}
-
-FastS_SyncSearchAdapter::~FastS_SyncSearchAdapter() = default;
-
-void
-FastS_SyncSearchAdapter::DoneQuery(FastS_ISearch *)
-{
- std::lock_guard<std::mutex> guard(_lock);
- _queryDone = true;
- if (_waitQuery) {
- _cond.notify_one();
- }
-}
-
-
-void
-FastS_SyncSearchAdapter::DoneDocsums(FastS_ISearch *)
-{
- std::lock_guard<std::mutex> guard(_lock);
- _docsumsDone = true;
- if (_waitDocsums) {
- _cond.notify_one();
- }
-}
-
-
-void
-FastS_SyncSearchAdapter::WaitQueryDone()
-{
- std::unique_lock<std::mutex> guard(_lock);
- _waitQuery = true;
- while (!_queryDone) {
- _cond.wait(guard);
- }
-}
-
-
-void
-FastS_SyncSearchAdapter::WaitDocsumsDone()
-{
- std::unique_lock<std::mutex> guard(_lock);
- _waitDocsums = true;
- while (!_docsumsDone) {
- _cond.wait(guard);
- }
-}
-
-
-bool
-FastS_SyncSearchAdapter::IsAsync()
-{
- return false;
-}
-
-
-FastS_ISearch::RetCode
-FastS_SyncSearchAdapter::SetAsyncArgs(FastS_ISearchOwner *)
-{
- return RET_ERROR;
-}
-
-
-FastS_ISearch::RetCode
-FastS_SyncSearchAdapter::Search(uint32_t searchOffset, uint32_t maxhits, uint32_t minhits)
-{
- RetCode res = _search->Search(searchOffset, maxhits, minhits);
- if (res == RET_INPROGRESS) {
- WaitQueryDone();
- }
- return (res == RET_ERROR) ? RET_ERROR : RET_OK;
-}
-
-
-FastS_ISearch::RetCode
-FastS_SyncSearchAdapter::GetDocsums(const FastS_hitresult *hits, uint32_t hitcnt)
-{
- RetCode res = _search->GetDocsums(hits, hitcnt);
- if (res == RET_INPROGRESS) {
- WaitDocsumsDone();
- }
- return (res == RET_ERROR) ? RET_ERROR : RET_OK;
-}
diff --git a/searchcore/src/vespa/searchcore/fdispatch/common/search.h b/searchcore/src/vespa/searchcore/fdispatch/common/search.h
deleted file mode 100644
index d3383abdee7..00000000000
--- a/searchcore/src/vespa/searchcore/fdispatch/common/search.h
+++ /dev/null
@@ -1,393 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-
-#pragma once
-
-#include <vespa/searchsummary/docsummary/getdocsumargs.h>
-#include <vespa/searchlib/common/fslimits.h>
-#include <vespa/searchlib/engine/errorcodes.h>
-#include <vespa/searchlib/engine/searchrequest.h>
-#include <vespa/searchlib/common/packets.h>
-#include <vespa/document/base/globalid.h>
-#include <limits>
-#include <mutex>
-#include <condition_variable>
-
-class FastS_ISearch;
-
-
-//----------------------------------------------------------------
-
-class FastS_ISearchOwner
-{
-public:
- /**
- * Destructor. No cleanup needed for base class.
- */
- virtual ~FastS_ISearchOwner() { }
-
- virtual void DoneQuery(FastS_ISearch *search) = 0;
- virtual void DoneDocsums(FastS_ISearch *search) = 0;
-};
-
-//----------------------------------------------------------------
-
-class FastS_hitresult
-{
-public:
- const document::GlobalId & HT_GetGlobalID() const { return _gid; }
- search::HitRank HT_GetMetric() const { return _metric; }
- uint32_t HT_GetPartID() const { return _partition; }
- uint32_t getDistributionKey() const { return _distributionKey; }
-
- void HT_SetGlobalID(const document::GlobalId & val) { _gid = val; }
- void HT_SetMetric(search::HitRank val) { _metric = val; }
- void HT_SetPartID(uint32_t val) { _partition = val; }
- void setDistributionKey(uint32_t val) { _distributionKey = val; }
- document::GlobalId _gid;
- search::HitRank _metric;
- uint32_t _partition;
-private:
- uint32_t _distributionKey;
-};
-
-//----------------------------------------------------------------
-
-struct FastS_fullresult {
- uint32_t _partition;
- uint32_t _docid;
- document::GlobalId _gid;
- search::HitRank _metric;
- search::fs4transport::FS4Packet_DOCSUM::Buf _buf;
-};
-
-//----------------------------------------------------------------
-
-class FastS_SearchInfo
-{
-public:
- uint32_t _searchOffset;
- uint32_t _maxHits;
- uint64_t _coverageDocs;
- uint64_t _activeDocs;
- uint64_t _soonActiveDocs;
- uint32_t _degradeReason;
- uint16_t _nodesQueried;
- uint16_t _nodesReplied;
-
- FastS_SearchInfo()
- : _searchOffset(0),
- _maxHits(0),
- _coverageDocs(0),
- _activeDocs(0),
- _soonActiveDocs(0),
- _degradeReason(0),
- _nodesQueried(0),
- _nodesReplied(0)
- { }
-};
-
-//----------------------------------------------------------------
-
-class FastS_QueryResult
-{
-public:
- FastS_hitresult *_hitbuf;
- uint32_t _hitCount;
- uint64_t _totalHitCount;
- search::HitRank _maxRank;
- double _queryResultTime;
-
- uint32_t _groupResultLen;
- const char *_groupResult;
-
- uint32_t *_sortIndex;
- const char *_sortData;
-
- FastS_QueryResult()
- : _hitbuf(NULL),
- _hitCount(0),
- _totalHitCount(0),
- _maxRank(std::numeric_limits<search::HitRank>::is_integer ?
- std::numeric_limits<search::HitRank>::min() :
- - std::numeric_limits<search::HitRank>::max()),
- _queryResultTime(0.0),
- _groupResultLen(0),
- _groupResult(NULL),
- _sortIndex(NULL),
- _sortData(NULL)
- {}
-};
-
-//----------------------------------------------------------------
-
-class FastS_DocsumsResult
-{
-private:
- FastS_DocsumsResult(const FastS_DocsumsResult &);
- FastS_DocsumsResult& operator=(const FastS_DocsumsResult &);
-
-public:
- FastS_fullresult *_fullresult;
- uint32_t _fullResultCount;
- double _queryDocSumTime;
-
- FastS_DocsumsResult()
- : _fullresult(NULL),
- _fullResultCount(0),
- _queryDocSumTime(0.0)
- {}
-};
-
-//----------------------------------------------------------------
-
-class FastS_ISearch
-{
-public:
- /**
- * Destructor. No cleanup needed for base class.
- */
- virtual ~FastS_ISearch() { }
-
-
- enum RetCode {
- RET_OK = 0, // sync operation performed
- RET_INPROGRESS = 1, // async operation started
- RET_ERROR = 2 // illegal method invocation
- };
-
- // OBTAIN META-DATA
-
- virtual bool IsAsync() = 0;
- virtual uint32_t GetDataSetID() = 0;
- virtual FastS_SearchInfo *GetSearchInfo() = 0;
-
- // SET PARAMETERS
-
- virtual RetCode SetAsyncArgs(FastS_ISearchOwner *owner) = 0;
- virtual RetCode setSearchRequest(const search::engine::SearchRequest * request) = 0;
- virtual RetCode SetGetDocsumArgs(search::docsummary::GetDocsumArgs *docsumArgs) = 0;
-
- // SEARCH API
-
- virtual RetCode Search(uint32_t searchOffset,
- uint32_t maxhits, uint32_t minhits = 0) = 0;
- virtual RetCode ProcessQueryDone() = 0;
- virtual FastS_QueryResult *GetQueryResult() = 0;
-
- // DOCSUM API
-
- virtual RetCode GetDocsums(const FastS_hitresult *hits, uint32_t hitcnt) = 0;
- virtual RetCode ProcessDocsumsDone() = 0;
- virtual FastS_DocsumsResult *GetDocsumsResult() = 0;
-
- // ERROR HANDLING
-
- virtual search::engine::ErrorCode GetErrorCode() = 0;
- virtual const char *GetErrorMessage() = 0;
-
- // INTERRUPT OPERATION
-
- virtual void Interrupt() = 0;
-
- // GET RID OF OBJECT
-
- virtual void Free() = 0;
-};
-
-//----------------------------------------------------------------
-
-class FastS_SearchBase : public FastS_ISearch
-{
-protected:
- uint32_t _dataSetID;
- search::engine::ErrorCode _errorCode;
- char *_errorMessage;
- const search::engine::SearchRequest *_queryArgs;
- search::docsummary::GetDocsumArgs *_docsumArgs;
- FastS_SearchInfo _searchInfo;
- FastS_QueryResult _queryResult;
- FastS_DocsumsResult _docsumsResult;
-
-public:
- FastS_SearchBase(const FastS_SearchBase &) = delete;
- FastS_SearchBase& operator=(const FastS_SearchBase &) = delete;
- FastS_SearchBase(uint32_t dataSetID)
- : _dataSetID(dataSetID),
- _errorCode(search::engine::ECODE_NO_ERROR),
- _errorMessage(NULL),
- _queryArgs(NULL),
- _docsumArgs(NULL),
- _searchInfo(),
- _queryResult(),
- _docsumsResult()
- {
- }
-
- ~FastS_SearchBase() override {
- free(_errorMessage);
- }
-
- void SetError(search::engine::ErrorCode errorCode, const char *errorMessage)
- {
- _errorCode = errorCode;
- if (errorMessage != NULL)
- _errorMessage = strdup(errorMessage);
- else
- _errorMessage = NULL;
- }
-
-
- uint32_t GetDataSetID() override { return _dataSetID; }
- FastS_SearchInfo *GetSearchInfo() override { return &_searchInfo; }
-
- RetCode setSearchRequest(const search::engine::SearchRequest * request) override {
- _queryArgs = request;
- return RET_OK;
- }
-
- RetCode SetGetDocsumArgs(search::docsummary::GetDocsumArgs *docsumArgs) override {
- _docsumArgs = docsumArgs;
- return RET_OK;
- }
-
- RetCode Search(uint32_t searchOffset, uint32_t maxhits, uint32_t minhits = 0) override {
- (void) minhits;
- _searchInfo._searchOffset = searchOffset;
- _searchInfo._maxHits = maxhits;
- return RET_OK;
- }
-
- RetCode ProcessQueryDone() override { return RET_OK; }
- FastS_QueryResult *GetQueryResult() override { return &_queryResult; }
-
- RetCode GetDocsums(const FastS_hitresult *hits, uint32_t hitcnt) override {
- (void) hits;
- (void) hitcnt;
- return RET_OK;
- }
-
- RetCode ProcessDocsumsDone() override { return RET_OK; }
- FastS_DocsumsResult *GetDocsumsResult() override { return &_docsumsResult; }
- search::engine::ErrorCode GetErrorCode() override { return _errorCode; }
-
- const char *GetErrorMessage() override {
- if (_errorMessage != NULL)
- return _errorMessage;
- return search::engine::getStringFromErrorCode(_errorCode);
- }
-
- void Interrupt() override {}
- void Free() override { delete this; }
-};
-
-//----------------------------------------------------------------
-
-class FastS_FailedSearch : public FastS_SearchBase
-{
-private:
- bool _async;
-
-public:
- FastS_FailedSearch(uint32_t dataSetID, bool async, search::engine::ErrorCode errorCode, const char *errorMessage)
- : FastS_SearchBase(dataSetID),
- _async(async)
- {
- SetError(errorCode, errorMessage);
- }
-
- bool IsAsync() override { return _async; }
-
- RetCode SetAsyncArgs(FastS_ISearchOwner *owner) override {
- (void) owner;
- return (_async) ? RET_OK : RET_ERROR;
- }
-};
-
-//----------------------------------------------------------------
-
-class FastS_AsyncSearch : public FastS_SearchBase
-{
-protected:
- FastS_ISearchOwner *_searchOwner;
-
-public:
- FastS_AsyncSearch(const FastS_AsyncSearch &) = delete;
- FastS_AsyncSearch& operator=(const FastS_AsyncSearch &) = delete;
- FastS_AsyncSearch(uint32_t dataSetID)
- : FastS_SearchBase(dataSetID),
- _searchOwner(NULL)
- {}
-
- bool IsAsync() override { return true; }
-
- RetCode SetAsyncArgs(FastS_ISearchOwner *owner) override {
- _searchOwner = owner;
- return RET_OK;
- }
-};
-
-//----------------------------------------------------------------
-
-class FastS_SearchAdapter : public FastS_ISearch
-{
-protected:
- FastS_ISearch *_search;
-
-public:
- explicit FastS_SearchAdapter(FastS_ISearch *search);
- FastS_SearchAdapter(const FastS_SearchAdapter &) = delete;
- FastS_SearchAdapter& operator=(const FastS_SearchAdapter &) = delete;
- ~FastS_SearchAdapter() override;
-
- bool IsAsync() override;
- uint32_t GetDataSetID() override;
- FastS_SearchInfo *GetSearchInfo() override;
- RetCode SetAsyncArgs(FastS_ISearchOwner *owner) override;
- RetCode setSearchRequest(const search::engine::SearchRequest * request) override;
- RetCode SetGetDocsumArgs(search::docsummary::GetDocsumArgs *docsumArgs) override;
- RetCode Search(uint32_t searchOffset, uint32_t maxhits, uint32_t minhits = 0) override;
- RetCode ProcessQueryDone() override;
- FastS_QueryResult *GetQueryResult() override;
- RetCode GetDocsums(const FastS_hitresult *hits, uint32_t hitcnt) override;
- RetCode ProcessDocsumsDone() override;
- FastS_DocsumsResult *GetDocsumsResult() override;
- search::engine::ErrorCode GetErrorCode() override;
- const char *GetErrorMessage() override;
- void Interrupt() override;
- void Free() override;
-};
-
-//----------------------------------------------------------------
-
-class FastS_SyncSearchAdapter : public FastS_SearchAdapter,
- public FastS_ISearchOwner
-{
-private:
- std::mutex _lock;
- std::condition_variable _cond;
- bool _waitQuery;
- bool _queryDone;
- bool _waitDocsums;
- bool _docsumsDone;
-
-protected:
- explicit FastS_SyncSearchAdapter(FastS_ISearch *search);
-
-public:
- ~FastS_SyncSearchAdapter() override;
-
-
- void DoneQuery(FastS_ISearch *) override;
- void DoneDocsums(FastS_ISearch *) override;
-
- void WaitQueryDone();
- void WaitDocsumsDone();
-
- bool IsAsync() override;
- RetCode SetAsyncArgs(FastS_ISearchOwner *owner) override;
- RetCode Search(uint32_t searchOffset, uint32_t maxhits, uint32_t minhits = 0) override;
- RetCode GetDocsums(const FastS_hitresult *hits, uint32_t hitcnt) override;
-};
-
-//----------------------------------------------------------------
-
diff --git a/searchcore/src/vespa/searchcore/fdispatch/common/stdincl.h b/searchcore/src/vespa/searchcore/fdispatch/common/stdincl.h
deleted file mode 100644
index b268d2831e2..00000000000
--- a/searchcore/src/vespa/searchcore/fdispatch/common/stdincl.h
+++ /dev/null
@@ -1,12 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-
-#pragma once
-
-#include <cstdint>
-
-/**
- * This method defines the illegal/undefined value for unsigned 32-bit
- * integer ids.
- **/
-inline uint32_t FastS_NoID32() { return static_cast<uint32_t>(-1); }
-
diff --git a/searchcore/src/vespa/searchcore/fdispatch/common/timestat.cpp b/searchcore/src/vespa/searchcore/fdispatch/common/timestat.cpp
deleted file mode 100644
index 5dec5ba4bc3..00000000000
--- a/searchcore/src/vespa/searchcore/fdispatch/common/timestat.cpp
+++ /dev/null
@@ -1,116 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-
-#include "timestat.h"
-
-void
-FastS_TimeStatHistory::Reset()
-{
- _sampleAccTime = 0.0;
- _totalAccTime = 0.0;
- _sampleIdx = 0;
- _sampleCount = 0;
- _totalCount = 0;
- for (uint32_t i = 0; i < _timestatssize; i++)
- _sampleTimes[i] = Sample();
-}
-
-
-double
-FastS_TimeStatHistory::GetMaxTime() const
-{
- double max = 0.0;
- uint32_t idx = _sampleIdx;
- for (uint32_t residue = _sampleCount;
- residue > 0; residue--)
- {
- if (idx > 0)
- idx--;
- else
- idx = _timestatssize - 1;
- if (_sampleTimes[idx]._time > max)
- max = _sampleTimes[idx]._time;
- }
- return max;
-}
-
-
-void
-FastS_TimeStatHistory::Update(double tnow, double t, bool timedout)
-{
- uint32_t timeIdx = getTimeIdx(tnow);
- if (_slotCount == 0u) {
- _timeSlots[_slotIdx].init(timeIdx);
- ++_slotCount;
- } else {
- TimeSlot &ts = _timeSlots[_slotIdx];
- if (ts._timeIdx > timeIdx)
- timeIdx = ts._timeIdx;
- if (ts._timeIdx < timeIdx) {
- if (_slotCount < NUM_TIMESLOTS)
- ++_slotCount;
- _slotIdx = nextTimeSlot(_slotIdx);
- _timeSlots[_slotIdx].init(timeIdx);
- }
- }
- _timeSlots[_slotIdx].update(t, timedout);
-
- _totalAccTime += t;
- ++_totalCount;
- if (timedout)
- ++_totalTimeouts;
- if (_sampleCount >= _timestatssize) {
- const Sample &s = _sampleTimes[_sampleIdx];
- _sampleAccTime -= s._time;
- if (s._timedout)
- --_sampleTimeouts;
- --_sampleCount;
- }
- _sampleTimes[_sampleIdx] = Sample(t, timedout);
- _sampleAccTime += t;
- if (timedout)
- ++_sampleTimeouts;
- _sampleIdx++;
- if (_sampleIdx >= _timestatssize)
- _sampleIdx = 0;
- ++_sampleCount;
-}
-
-
-void
-FastS_TimeStatHistory::getRecentStats(double tsince,
- FastS_TimeStatTotals &totals)
-{
- uint32_t timeIdx = getTimeIdx(tsince);
- uint32_t slotCount = _slotCount;
- uint32_t slotIdx = _slotIdx;
- for (; slotCount > 0u && _timeSlots[slotIdx]._timeIdx >= timeIdx;
- --slotCount, slotIdx = prevTimeSlot(slotIdx)) {
- TimeSlot &ts = _timeSlots[slotIdx];
- totals._totalCount += ts._count;
- totals._totalTimeouts += ts._timeouts;
- totals._totalAccTime += ts._accTime;
- }
-}
-
-
-double
-FastS_TimeStatHistory::getLoadTime(double tsince, double tnow)
-{
- const uint32_t holeSize = 2; // 2 missing slots => hole
- const uint32_t minSlotLoad = 4; // Mininum load for not being "missing"
- uint32_t sinceTimeIdx = getTimeIdx(tsince);
- uint32_t timeIdx = getTimeIdx(tnow);
- uint32_t slotCount = _slotCount;
- uint32_t slotIdx = _slotIdx;
- uint32_t doneTimeIdx = timeIdx;
- for (; slotCount > 0u; --slotCount, slotIdx = prevTimeSlot(slotIdx)) {
- TimeSlot &ts = _timeSlots[slotIdx];
- if (ts._timeIdx + holeSize < doneTimeIdx)
- break; // Found hole, i.e. holeSize missing slots
- if (ts._timeIdx + holeSize < sinceTimeIdx)
- break; // No point in looking further back
- if (ts._count >= minSlotLoad)
- doneTimeIdx = ts._timeIdx;
- }
- return tnow - getTimeFromTimeIdx(doneTimeIdx);
-}
diff --git a/searchcore/src/vespa/searchcore/fdispatch/common/timestat.h b/searchcore/src/vespa/searchcore/fdispatch/common/timestat.h
deleted file mode 100644
index e05bfe5ef29..00000000000
--- a/searchcore/src/vespa/searchcore/fdispatch/common/timestat.h
+++ /dev/null
@@ -1,215 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-
-
-#pragma once
-
-#include <cstdint>
-
-class FastS_TimeStatTotals
-{
-public:
- int _totalCount;
- uint32_t _totalTimeouts;
- double _totalAccTime;
-
- FastS_TimeStatTotals()
- : _totalCount(0),
- _totalTimeouts(0u),
- _totalAccTime(0)
- {
- }
-
- FastS_TimeStatTotals &
- operator+=(const FastS_TimeStatTotals &rhs)
- {
- _totalCount += rhs._totalCount;
- _totalTimeouts += rhs._totalTimeouts;
- _totalAccTime += rhs._totalAccTime;
- return *this;
- }
-
- FastS_TimeStatTotals &
- operator-=(const FastS_TimeStatTotals &rhs)
- {
- _totalCount -= rhs._totalCount;
- _totalTimeouts -= rhs._totalTimeouts;
- _totalAccTime -= rhs._totalAccTime;
- return *this;
- }
-
- double
- getAvgTime() const
- {
- if (_totalCount == 0)
- return 0.0;
- return (_totalAccTime / _totalCount);
- }
-
- double
- getTimeoutRate() const
- {
- if (_totalCount == 0)
- return 0.0;
- return (static_cast<double>(_totalTimeouts) / _totalCount);
- }
-};
-
-
-class FastS_TimeStatHistory
-{
- enum {
- _timestatssize = 100
- };
- enum {
- SLOT_SIZE = 5,
- NUM_TIMESLOTS = 128
- };
- double _sampleAccTime;
- double _totalAccTime;
- uint32_t _sampleIdx;
- uint32_t _sampleCount;
- uint32_t _sampleTimeouts;
- uint32_t _totalCount;
- uint32_t _totalTimeouts;
- struct Sample {
- double _time;
- bool _timedout;
-
- Sample()
- : _time(0.0),
- _timedout(false)
- {
- }
-
- Sample(double time, bool timedout)
- : _time(time),
- _timedout(timedout)
- {
- }
- };
- Sample _sampleTimes[_timestatssize];
-
- struct TimeSlot
- {
- double _accTime;
- uint32_t _count;
- uint32_t _timeouts;
- uint32_t _timeIdx;
-
- TimeSlot()
- : _accTime(0.0),
- _count(0u),
- _timeouts(0u),
- _timeIdx(0u)
- {
- }
-
- void
- init(uint32_t timeIdx)
- {
- _accTime = 0.0;
- _count = 0u;
- _timeouts = 0u;
- _timeIdx = timeIdx;
- }
-
- void
- update(double t, bool timedout)
- {
- _accTime += t;
- ++_count;
- if (timedout)
- ++_timeouts;
- }
- };
-
- static uint32_t
- nextTimeSlot(uint32_t timeSlot)
- {
- return (timeSlot >= (NUM_TIMESLOTS - 1)) ? 0u : timeSlot + 1;
- }
-
- static uint32_t
- prevTimeSlot(uint32_t timeSlot)
- {
- return (timeSlot == 0u) ? (NUM_TIMESLOTS - 1) : timeSlot - 1;
- }
-
-
- TimeSlot _timeSlots[NUM_TIMESLOTS];
- uint32_t _slotCount;
- uint32_t _slotIdx;
-
- static uint32_t
- getTimeIdx(double t)
- {
- // Each SLOT_SIZE second period has it's own slot of statistics
- return static_cast<uint32_t>(t / SLOT_SIZE);
- }
-
- static double
- getTimeFromTimeIdx(uint32_t timeIdx)
- {
- return static_cast<double>(timeIdx) * SLOT_SIZE;
- }
-public:
- FastS_TimeStatHistory()
- : _sampleAccTime(0.0),
- _totalAccTime(0.0),
- _sampleIdx(0),
- _sampleCount(0),
- _sampleTimeouts(0),
- _totalCount(0),
- _totalTimeouts(0u),
- _sampleTimes(),
- _timeSlots(),
- _slotCount(0u),
- _slotIdx(0u)
- {
- Reset();
- }
-
- void Reset();
-
- double GetSampleAccTime() const { return _sampleAccTime; }
- uint32_t GetSampleCount() const { return _sampleCount; }
-
- uint32_t
- getSampleTimeouts() const
- {
- return _sampleTimeouts;
- }
-
- double GetAvgTime() const
- {
- if (_sampleCount == 0)
- return 0.0;
- return (_sampleAccTime / _sampleCount);
- }
- double GetMaxTime() const;
-
- void Update(double tnow, double t, bool timedout);
-
- uint32_t GetTotalCount() const { return _totalCount; }
- double GetTotalAccTime() const { return _totalAccTime; }
-
- uint32_t
- getTotalTimeouts() const
- {
- return _totalTimeouts;
- }
-
- void AddTotal(FastS_TimeStatTotals *totals) {
- totals->_totalCount += GetTotalCount();
- totals->_totalTimeouts += getTotalTimeouts();
- totals->_totalAccTime += GetTotalAccTime();
- }
-
- void
- getRecentStats(double tsince, FastS_TimeStatTotals &totals);
-
- double
- getLoadTime(double tsince, double tnow);
-};
-
-
diff --git a/searchcore/src/vespa/searchcore/fdispatch/program/.gitignore b/searchcore/src/vespa/searchcore/fdispatch/program/.gitignore
deleted file mode 100644
index 27eb860d05b..00000000000
--- a/searchcore/src/vespa/searchcore/fdispatch/program/.gitignore
+++ /dev/null
@@ -1,13 +0,0 @@
-*.exp
-*.ilk
-*.lib
-*.pdb
-.depend
-ID
-Makefile
-access_log
-debugmalloc
-fdispatch
-fdispatch.exe
-sfdispatch
-sfdispatch.exe
diff --git a/searchcore/src/vespa/searchcore/fdispatch/program/CMakeLists.txt b/searchcore/src/vespa/searchcore/fdispatch/program/CMakeLists.txt
deleted file mode 100644
index c2c00320d82..00000000000
--- a/searchcore/src/vespa/searchcore/fdispatch/program/CMakeLists.txt
+++ /dev/null
@@ -1,11 +0,0 @@
-# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-vespa_add_library(searchcore_fdispatch_program STATIC
- SOURCES
- fdispatch.cpp
- rpc.cpp
- engineadapter.cpp
- searchadapter.cpp
- docsumadapter.cpp
- DEPENDS
- searchcore_fconfig
-)
diff --git a/searchcore/src/vespa/searchcore/fdispatch/program/description.html b/searchcore/src/vespa/searchcore/fdispatch/program/description.html
deleted file mode 100644
index a73a6c4b4d1..00000000000
--- a/searchcore/src/vespa/searchcore/fdispatch/program/description.html
+++ /dev/null
@@ -1,3 +0,0 @@
-<!-- Short description for make kdoc. -->
-<!-- Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -->
-FDispatch program.
diff --git a/searchcore/src/vespa/searchcore/fdispatch/program/docsumadapter.cpp b/searchcore/src/vespa/searchcore/fdispatch/program/docsumadapter.cpp
deleted file mode 100644
index d104aecfacc..00000000000
--- a/searchcore/src/vespa/searchcore/fdispatch/program/docsumadapter.cpp
+++ /dev/null
@@ -1,114 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-
-#include "docsumadapter.h"
-#include <vespa/searchcore/fdispatch/search/datasetcollection.h>
-
-#include <vespa/log/log.h>
-LOG_SETUP(".fdispatch.docsumadapter");
-
-namespace fdispatch {
-
-void
-DocsumAdapter::setupRequest()
-{
- const DocsumRequest &req = *_request.get();
- _args.initFromDocsumRequest(req);
- _hitcnt = req.hits.size();
- LOG(debug, "DocsumAdapter::setupRequest : hitcnt=%d", _hitcnt);
- if (_hitcnt > 0) {
- _hitbuf = (FastS_hitresult *) malloc(req.hits.size() * sizeof(FastS_hitresult));
- }
- for (uint32_t i = 0; i < _hitcnt; i++) {
- _hitbuf[i]._gid = req.hits[i].gid;
- _hitbuf[i]._partition = req.hits[i].path;
- LOG(debug, "DocsumAdapter::setupRequest : hit[%d] (gid=%s,part=%d)",
- i, _hitbuf[i]._gid.toString().c_str(), _hitbuf[i]._partition);
- }
-}
-
-void
-DocsumAdapter::handleRequest()
-{
- _dsc = _appCtx->GetDataSetCollection();
- assert(_dsc != NULL);
- _search = _dsc->CreateSearch(FastS_NoID32(), _appCtx->GetTimeKeeper());
- assert(_search != NULL);
- _docsumsResult = _search->GetDocsumsResult();
- _search->SetGetDocsumArgs(&_args);
- _search->GetDocsums(_hitbuf, _hitcnt);
- _search->ProcessDocsumsDone();
-}
-
-void
-DocsumAdapter::createReply()
-{
- DocsumReply::UP reply(new DocsumReply());
- DocsumReply &r = *reply;
-
- FastS_fullresult *hitbuf = _docsumsResult->_fullresult;
- uint32_t hitcnt = _docsumsResult->_fullResultCount;
-
- LOG(debug, "DocsumAdapter::createReply : hitcnt=%d", hitcnt);
- r.docsums.reserve(hitcnt);
- for (uint32_t i = 0; i < hitcnt; i++) {
- if ( ! hitbuf[i]._buf.empty() ) {
- r.docsums.push_back(DocsumReply::Docsum());
- DocsumReply::Docsum & d = r.docsums.back();
- d.docid = hitbuf[i]._docid;
- d.gid = hitbuf[i]._gid;
- d.data.swap(hitbuf[i]._buf);
- } else {
- LOG(debug, "DocsumAdapter::createReply : No buf for hit=%d", i);
- }
- }
- r.request = _request.release();
- _client.getDocsumsDone(std::move(reply));
-}
-
-void
-DocsumAdapter::writeLog()
-{
- // no access log for docsums
-}
-
-void
-DocsumAdapter::cleanup()
-{
- if (_search != NULL) {
- _search->Free();
- }
- if (_dsc != NULL) {
- _dsc->subRef();
- }
- free(_hitbuf);
- _hitcnt = 0;
- _hitbuf = 0;
-}
-
-void
-DocsumAdapter::Run(FastOS_ThreadInterface *, void *)
-{
- setupRequest();
- handleRequest();
- createReply();
- writeLog();
- cleanup();
- delete this;
-}
-
-DocsumAdapter::DocsumAdapter(FastS_AppContext *appCtx,
- DocsumRequest::Source request,
- DocsumClient &client)
- : _appCtx(appCtx),
- _request(std::move(request)),
- _client(client),
- _args(),
- _hitcnt(0),
- _hitbuf(0),
- _dsc(0),
- _search(0),
- _docsumsResult(0)
-{
-}
-
-} // namespace fdispatch
diff --git a/searchcore/src/vespa/searchcore/fdispatch/program/docsumadapter.h b/searchcore/src/vespa/searchcore/fdispatch/program/docsumadapter.h
deleted file mode 100644
index 18ade945102..00000000000
--- a/searchcore/src/vespa/searchcore/fdispatch/program/docsumadapter.h
+++ /dev/null
@@ -1,52 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-
-#pragma once
-
-#include <vespa/searchlib/engine/docsumapi.h>
-#include <vespa/searchcore/fdispatch/common/appcontext.h>
-#include <vespa/searchcore/fdispatch/common/search.h>
-#include <vespa/searchsummary/docsummary/getdocsumargs.h>
-#include <vespa/fastos/thread.h>
-
-namespace fdispatch {
-
-/**
- * Implementation of the common search api for the fdispatch server
- * application.
- **/
-class DocsumAdapter : public FastOS_Runnable
-{
-public:
- typedef search::engine::DocsumRequest DocsumRequest;
- typedef search::engine::DocsumReply DocsumReply;
- typedef search::engine::DocsumClient DocsumClient;
-
-private:
- FastS_AppContext *_appCtx;
- DocsumRequest::Source _request;
- DocsumClient &_client;
-
- // internal docsum related state
- search::docsummary::GetDocsumArgs _args;
- uint32_t _hitcnt;
- FastS_hitresult *_hitbuf;
- FastS_DataSetCollection *_dsc;
- FastS_ISearch *_search;
- FastS_DocsumsResult *_docsumsResult;
-
- void setupRequest();
- void handleRequest();
- void createReply();
- void writeLog();
- void cleanup();
-
- virtual void Run(FastOS_ThreadInterface *, void *) override;
-
-public:
- DocsumAdapter(FastS_AppContext *appCtx,
- DocsumRequest::Source request,
- DocsumClient &client);
-};
-
-} // namespace fdispatch
-
diff --git a/searchcore/src/vespa/searchcore/fdispatch/program/engineadapter.cpp b/searchcore/src/vespa/searchcore/fdispatch/program/engineadapter.cpp
deleted file mode 100644
index 6519b55851e..00000000000
--- a/searchcore/src/vespa/searchcore/fdispatch/program/engineadapter.cpp
+++ /dev/null
@@ -1,77 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-
-#include "engineadapter.h"
-#include "searchadapter.h"
-#include "docsumadapter.h"
-#include <vespa/searchcore/fdispatch/search/child_info.h>
-#include <vespa/searchcore/fdispatch/search/nodemanager.h>
-#include <vespa/searchcore/fdispatch/search/dataset_base.h>
-
-#include <vespa/log/log.h>
-LOG_SETUP(".fdispatch.engineadapter");
-
-namespace fdispatch {
-
-EngineAdapter::
-EngineAdapter(FastS_AppContext *appCtx, FastOS_ThreadPool *threadPool)
- : _appCtx(appCtx),
- _mypool(threadPool)
-{
-}
-
-EngineAdapter::SearchReply::UP
-EngineAdapter::search(SearchRequest::Source request, SearchClient &client)
-{
- SearchAdapter *sa = new SearchAdapter(_appCtx, std::move(request), client);
- if ((_mypool == 0) || (_mypool->NewThread(sa) == 0)) {
- delete sa;
- LOG(error, "could not allocate thread for incoming search request");
- SearchReply::UP reply(new SearchReply());
- reply->useWideHits = true; // mld
- reply->errorCode = search::engine::ECODE_OVERLOADED;
- reply->errorMessage = "could not allocate thread for query";
- return reply;
- }
- return SearchReply::UP();
-}
-
-EngineAdapter::DocsumReply::UP
-EngineAdapter::getDocsums(DocsumRequest::Source request, DocsumClient &client)
-{
- DocsumAdapter *da = new DocsumAdapter(_appCtx, std::move(request), client);
- if ((_mypool == 0) || (_mypool->NewThread(da) == 0)) {
- delete da;
- LOG(error, "could not allocate thread for incoming docsum request");
- return DocsumReply::UP(new DocsumReply());
- }
- return DocsumReply::UP();
-}
-
-EngineAdapter::MonitorReply::UP
-EngineAdapter::ping(MonitorRequest::UP request, MonitorClient &)
-{
- MonitorReply::UP reply(new MonitorReply());
- MonitorReply &mr = *reply;
-
- uint32_t timeStamp = 0;
- FastS_NodeManager *nm = _appCtx->GetNodeManager();
-
- ChildInfo ci = nm->getChildInfo();
- timeStamp = nm->GetMldDocstamp();
- // TODO: Report softoffline upwards when fdispatch has been requested
- // to go down in a controlled manner (along with zero docstamp).
- mr.partid = nm->GetMldPartition();
- mr.timestamp = timeStamp;
- mr.mld = true;
- mr.totalNodes = ci.maxNodes;
- mr.activeNodes = ci.activeNodes;
- mr.totalParts = ci.maxParts;
- mr.activeParts = ci.activeParts;
- if (ci.activeDocs.valid) {
- mr.activeDocs = ci.activeDocs.count;
- mr.activeDocsRequested = request->reportActiveDocs;
- }
- return reply;
-}
-
-} // namespace fdispatch
diff --git a/searchcore/src/vespa/searchcore/fdispatch/program/engineadapter.h b/searchcore/src/vespa/searchcore/fdispatch/program/engineadapter.h
deleted file mode 100644
index add5f045d51..00000000000
--- a/searchcore/src/vespa/searchcore/fdispatch/program/engineadapter.h
+++ /dev/null
@@ -1,46 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-
-#pragma once
-
-#include <vespa/searchlib/engine/searchapi.h>
-#include <vespa/searchlib/engine/docsumapi.h>
-#include <vespa/searchlib/engine/monitorapi.h>
-
-#include <vespa/searchcore/fdispatch/common/appcontext.h>
-
-namespace fdispatch {
-
-/**
- * Implementation of the common search api for the fdispatch server
- * application.
- **/
-class EngineAdapter : public search::engine::SearchServer,
- public search::engine::DocsumServer,
- public search::engine::MonitorServer
-{
-private:
- FastS_AppContext *_appCtx;
- FastOS_ThreadPool *_mypool;
-
-public:
- typedef search::engine::SearchRequest SearchRequest;
- typedef search::engine::DocsumRequest DocsumRequest;
- typedef search::engine::MonitorRequest MonitorRequest;
-
- typedef search::engine::SearchReply SearchReply;
- typedef search::engine::DocsumReply DocsumReply;
- typedef search::engine::MonitorReply MonitorReply;
-
- typedef search::engine::SearchClient SearchClient;
- typedef search::engine::DocsumClient DocsumClient;
- typedef search::engine::MonitorClient MonitorClient;
-
- EngineAdapter(FastS_AppContext *appCtx, FastOS_ThreadPool *threadPool);
-
- SearchReply::UP search(SearchRequest::Source request, SearchClient &client) override;
- DocsumReply::UP getDocsums(DocsumRequest::Source request, DocsumClient &client) override;
- MonitorReply::UP ping(MonitorRequest::UP request, MonitorClient &client) override;
-};
-
-} // namespace fdispatch
-
diff --git a/searchcore/src/vespa/searchcore/fdispatch/program/fdispatch.cpp b/searchcore/src/vespa/searchcore/fdispatch/program/fdispatch.cpp
deleted file mode 100644
index 3047834be85..00000000000
--- a/searchcore/src/vespa/searchcore/fdispatch/program/fdispatch.cpp
+++ /dev/null
@@ -1,397 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-
-#include "fdispatch.h"
-#include "engineadapter.h"
-#include "rpc.h"
-#include <vespa/searchcore/fdispatch/search/querycacheutil.h>
-#include <vespa/searchcore/fdispatch/search/nodemanager.h>
-#include <vespa/searchcore/util/eventloop.h>
-#include <vespa/vespalib/util/exceptions.h>
-#include <vespa/config/helper/configgetter.hpp>
-#include <vespa/vespalib/net/crypto_engine.h>
-
-#include <vespa/log/log.h>
-LOG_SETUP(".fdispatch");
-
-#ifndef V_TAG
-#define V_TAG "NOTAG"
-#endif
-
-using search::fs4transport::FS4PersistentPacketStreamer;
-using vespa::config::search::core::FdispatchrcConfig;
-using vespa::config::search::core::internal::InternalFdispatchrcType;
-using vespalib::compression::CompressionConfig;
-
-char FastS_VersionTag[] = V_TAG;
-
-namespace fdispatch
-{
-
-FastS_FNETAdapter::FastS_FNETAdapter(FastS_AppContext *appCtx)
- : _appCtx(appCtx),
- _nodeManager(),
- _timeKeeper(NULL),
- _transport(NULL),
- _last_now(0.0),
- _live_counter(0),
- _task()
-{ }
-
-FastS_FNETAdapter::~FastS_FNETAdapter()
-{
- fini();
-}
-
-void
-FastS_FNETAdapter::init()
-{
- _nodeManager = _appCtx->GetNodeManager();
- _timeKeeper = _appCtx->GetTimeKeeper();
- _transport = _appCtx->GetFNETTransport();
- _last_now = _timeKeeper->GetTime();
- _task.reset(new MyTask(_transport->GetScheduler(), *this));
- _task->ScheduleNow();
-}
-
-void
-FastS_FNETAdapter::perform()
-{
- double now = _timeKeeper->GetTime();
- double delta = now - _last_now;
- if (delta >= 3.0) {
- LOG(warning, "FNET loop high latency: %.3f", delta);
- }
- _last_now = now;
- ++_live_counter;
- _nodeManager->CheckEvents(_timeKeeper);
-}
-
-void
-FastS_FNETAdapter::fini()
-{
- if (_task) {
- _task->Kill();
- _task.reset();
- }
-}
-
-Fdispatch::~Fdispatch()
-{
- if (_transportServer) {
- _transportServer->shutDown(); // sync shutdown
- }
- _FNET_adapter.fini();
- if (_nodeManager) {
- _nodeManager->ShutdownConfig();
- }
- if (_transport && _transportStarted) {
- _transport->ShutDown(true); // sync shutdown
- }
- if (_rpc) {
- _rpc->ShutDown(); // sync shutdown
- }
-
- LOG(debug, "Will close threadpool");
- _mypool->Close();
- _executor.shutdown().sync();
- LOG(debug, "Has closed threadpool");
- _transportServer.reset();
- _engineAdapter.reset();
- _nodeManager.reset();
- _transport.reset();
- _rpc.reset();
- _mypool.reset();
-}
-
-FNET_Transport *
-Fdispatch::GetFNETTransport()
-{
- return _transport.get();
-}
-
-FNET_Scheduler *
-Fdispatch::GetFNETScheduler()
-{
- return (_transport) ? _transport->GetScheduler() : nullptr;
-}
-
-FastS_NodeManager *
-Fdispatch::GetNodeManager()
-{
- return _nodeManager.get();
-}
-
-FastS_DataSetCollection *
-Fdispatch::GetDataSetCollection()
-{
- return ( _nodeManager) ? _nodeManager->GetDataSetCollection() : nullptr;
-}
-
-FastOS_ThreadPool *
-Fdispatch::GetThreadPool()
-{
- return _mypool.get();
-}
-
-bool
-Fdispatch::Failed()
-{
- return ( (_transportServer && _transportServer->isFailed())) || _needRestart;
-}
-
-bool
-Fdispatch::CheckTempFail()
-{
- bool ret;
- bool failflag = _nodeManager->GetTempFail();
- unsigned int FNETLiveCounter;
-
- ret = true;
-
- FNETLiveCounter = _FNET_adapter.GetLiveCounter();
- if (FNETLiveCounter == _lastFNETLiveCounter) {
- if (_FNETLiveCounterFailed) {
- failflag = true; // Still failure
- } else if (!_FNETLiveCounterDanger) {
- _FNETLiveCounterDanger = true;
- _FNETLiveCounterDangerStart.SetNow();
- } else if (_FNETLiveCounterDangerStart.MilliSecsToNow() >= 6000) {
- LOG(error, "fdispatch::Fdispatch::CheckTempFail: FNET inactive for 6 seconds, deadlock ?");
- _FNETLiveCounterFailed = true; // Note that we failed
- failflag = true; // Force temporary failure
- } else if (_FNETLiveCounterDangerStart.MilliSecsToNow() >= 3000 &&
- !_FNETLiveCounterWarned) {
- _FNETLiveCounterWarned = true;
- LOG(warning, "fdispatch::Fdispatch::CheckTempFail: FNET inactive for 3 seconds");
- }
- } else {
- if (_FNETLiveCounterFailed || _FNETLiveCounterWarned) {
- LOG(warning, "fdispatch::Fdispatch::CheckTempFail: FNET active again");
- }
- _FNETLiveCounterFailed = false;
- _FNETLiveCounterWarned = false;
- _FNETLiveCounterDanger = false;
- _lastFNETLiveCounter = FNETLiveCounter;
- }
-
- if (failflag == _tempFail)
- return ret;
-
- if (_transportServer) {
- if (failflag) {
- _transportServer->setListen(false);
- LOG(error, "Disabling fnet server interface");
- } else {
- _transportServer->setListen(true);
- LOG(info, "Reenabling fnet server interface");
- }
- }
- _tempFail = failflag;
- return ret;
-}
-
-
-/**
- * Make the httpd and Monitor, and let a Thread execute each.
- * Set up stuff as specified in the fdispatch-rc-file.
- */
-Fdispatch::Fdispatch(const config::ConfigUri &configUri)
- : _executor(1, 128 * 1024),
- _mypool(),
- _engineAdapter(),
- _transportServer(),
- _componentConfig(),
- _nodeManager(),
- _transport(),
- _FNET_adapter(this),
- _rpc(),
- _config(),
- _configUri(configUri),
- _fdispatchrcFetcher(configUri.getContext()),
- _rndGen(),
- _partition(0),
- _tempFail(false),
- _FNETLiveCounterDanger(false),
- _FNETLiveCounterWarned(false),
- _FNETLiveCounterFailed(false),
- _transportStarted(false),
- _lastFNETLiveCounter(false),
- _FNETLiveCounterDangerStart(),
- _timeouts(0u),
- _checkLimit(0u),
- _healthPort(0),
- _needRestart(false)
-{
- int64_t cfgGen = -1;
- _config = config::ConfigGetter<FdispatchrcConfig>::
- getConfig(cfgGen, _configUri.getConfigId(), _configUri.getContext());
- LOG(config, "fdispatch version %s (RPC-port: %d, transport at %d)",
- FastS_VersionTag, _config->frtport, _config->ptport);
-
- _componentConfig.addConfig(vespalib::ComponentConfigProducer::Config("fdispatch", cfgGen,
- "config only obtained at startup"));
- _fdispatchrcFetcher.subscribe<FdispatchrcConfig>(configUri.getConfigId(), this);
- _fdispatchrcFetcher.start();
-}
-
-namespace {
-
-bool needRestart(const FdispatchrcConfig & curr, const FdispatchrcConfig & next)
-{
- if (curr.frtport != next.frtport) {
- LOG(warning, "FRT port has changed from %d to %d.", curr.frtport, next.frtport);
- return true;
- }
- if (curr.ptport != next.ptport) {
- LOG(warning, "PT port has changed from %d to %d.", curr.ptport, next.ptport);
- return true;
- }
- if (curr.healthport != next.healthport) {
- LOG(warning, "Health port has changed from %d to %d.", curr.healthport, next.healthport);
- return true;
- }
- return false;
-}
-
-}
-
-void Fdispatch::configure(std::unique_ptr<FdispatchrcConfig> cfg)
-{
- if (cfg && _config) {
- if ( needRestart(*_config, *cfg) ) {
- LOG(warning, "Will restart by abort now.");
- _needRestart.store(true);
- }
- }
-}
-
-namespace {
-
-CompressionConfig::Type
-convert(InternalFdispatchrcType::Packetcompresstype type)
-{
- switch (type) {
- case InternalFdispatchrcType::Packetcompresstype::LZ4: return CompressionConfig::LZ4;
- default: return CompressionConfig::LZ4;
- }
-}
-
-}
-
-bool
-Fdispatch::Init()
-{
- int maxthreads;
-
- _tempFail = false;
- _FNETLiveCounterDanger = false;
- _FNETLiveCounterWarned = false;
- _FNETLiveCounterFailed = false;
- _lastFNETLiveCounter = 0;
- _timeouts = 0;
- _checkLimit = 60;
-
- FS4PersistentPacketStreamer::Instance.SetCompressionLimit(_config->packetcompresslimit);
- FS4PersistentPacketStreamer::Instance.SetCompressionLevel(_config->packetcompresslevel);
- FS4PersistentPacketStreamer::Instance.SetCompressionType(convert(_config->packetcompresstype));
-
-
- LOG(debug, "Creating FNET transport");
- _transport = std::make_unique<FNET_Transport>(std::make_shared<vespalib::NullCryptoEngine>(), _config->transportthreads); // disable encryption
-
- // grab node slowness limit defaults
-
- FastS_DataSetDesc::SetDefaultSlowQueryLimitFactor(_config->defaultslowquerylimitfactor);
- FastS_DataSetDesc::SetDefaultSlowQueryLimitBias(_config->defaultslowquerylimitbias);
- FastS_DataSetDesc::SetDefaultSlowDocsumLimitFactor(_config->defaultslowdocsumlimitfactor);
- FastS_DataSetDesc::SetDefaultSlowDocsumLimitBias(_config->defaultslowdocsumlimitbias);
-
- maxthreads = _config->maxthreads;
- _mypool = std::make_unique<FastOS_ThreadPool>(256 * 1024, maxthreads);
-
- // Max interval betw read from socket.
- FastS_TimeOut::_val[FastS_TimeOut::maxSockSilent] = _config->maxsocksilent;
-
- if (_transport) {
- _transport->SetIOCTimeOut((uint32_t) (FastS_TimeOut::_val[FastS_TimeOut::maxSockSilent] * 1000.0));
- }
-
- char timestr[40];
- FastS_TimeOut::WriteTime(timestr, sizeof(timestr), FastS_TimeOut::_val[FastS_TimeOut::maxSockSilent]);
- LOG(debug, "VERBOSE: Max time between successful read from a socket: %s", timestr);
-
- FastS_QueryCacheUtil::_systemMaxHits = std::numeric_limits<int>::max();
- LOG(debug, "VERBOSE: maxhits: %d", FastS_QueryCacheUtil::_systemMaxHits);
-
- FastS_QueryCacheUtil::_maxOffset = std::numeric_limits<int>::max();
- const uint32_t linesize = 1;
- if (FastS_QueryCacheUtil::_systemMaxHits < linesize
- && FastS_QueryCacheUtil::_maxOffset < linesize - FastS_QueryCacheUtil::_systemMaxHits) {
- LOG(warning, "maxoffset must be >= %d! (overriding config value)", linesize - FastS_QueryCacheUtil::_systemMaxHits);
- FastS_QueryCacheUtil::_maxOffset = linesize - FastS_QueryCacheUtil::_systemMaxHits;
- }
- LOG(debug, "VERBOSE: maxoffset: %d", FastS_QueryCacheUtil::_maxOffset);
-
- _partition = _config->partition;
-
- int ptportnum = _config->ptport;
-
- LOG(debug, "Using port number %d", ptportnum);
-
- _nodeManager = std::make_unique<FastS_NodeManager>(_componentConfig, this, _partition);
-
- GetFNETTransport()->SetTCPNoDelay(_config->transportnodelay);
-
- if (ptportnum == 0) {
- throw vespalib::IllegalArgumentException("fdispatchrc.ptportnum must be non-zero, most likely an issue with config delivery.");
- }
-
- _engineAdapter = std::make_unique<fdispatch::EngineAdapter>(this, _mypool.get());
- _transportServer = std::make_unique<TransportServer>(*_engineAdapter, *_engineAdapter, *_engineAdapter, ptportnum, search::engine::TransportServer::DEBUG_ALL);
- _transportServer->setTCPNoDelay(_config->transportnodelay);
-
- if (!_transportServer->start()) {
- _transportServer.reset();
- _engineAdapter.reset();
- LOG(error, "CRITICAL: Failed to init upwards FNET transport on port %d", ptportnum);
- return false;
- }
-
- _nodeManager->SubscribePartMap(_configUri);
-
- if (_config->frtport != 0) {
- _rpc = std::make_unique<FastS_fdispatch_RPC>(this);
- if (!_rpc->Init(_config->frtport, _configUri.getConfigId())) {
- LOG(error, "RPC init failed");
- _rpc.reset();
- }
- } else {
- _rpc.reset();
- }
-
- // Kick off fdispatch administrative threads.
- if (_transport) {
- _FNET_adapter.init();
- bool rc = _transport->Start(_mypool.get());
- if (rc) {
- LOG(debug, "Started FNET transport");
- _transportStarted = true;
- } else {
- LOG(error, "Failed to start FNET transport");
- }
- }
- FastOS_Thread::Sleep(1000);
- if (_rpc) {
- _rpc->Start();
- }
- _healthPort = _config->healthport;
- return true;
-}
-
-uint32_t
-Fdispatch::getDispatchLevel()
-{
- return _config->dispatchlevel;
-}
-
-}
diff --git a/searchcore/src/vespa/searchcore/fdispatch/program/fdispatch.h b/searchcore/src/vespa/searchcore/fdispatch/program/fdispatch.h
deleted file mode 100644
index 093308d68d2..00000000000
--- a/searchcore/src/vespa/searchcore/fdispatch/program/fdispatch.h
+++ /dev/null
@@ -1,111 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-
-#pragma once
-
-#include <vespa/fnet/fnet.h>
-#include <vespa/searchcore/fdispatch/common/appcontext.h>
-#include <vespa/searchlib/engine/transportserver.h>
-#include <vespa/searchcore/config/config-fdispatchrc.h>
-#include <vespa/config/subscription/configuri.h>
-#include <vespa/config/helper/ifetchercallback.h>
-#include <vespa/config/helper/configfetcher.h>
-#include <vespa/vespalib/net/simple_component_config_producer.h>
-#include <vespa/vespalib/util/random.h>
-#include <vespa/vespalib/util/threadstackexecutor.h>
-
-class FastS_NodeManager;
-class FastS_fdispatch_RPC;
-
-namespace fdispatch {
-
-class EngineAdapter;
-
-class FastS_FNETAdapter
-{
-private:
- FastS_AppContext *_appCtx;
- FastS_NodeManager *_nodeManager;
- FastS_TimeKeeper *_timeKeeper;
- FNET_Transport *_transport;
- double _last_now; // latency check
- uint32_t _live_counter; // latency check
-
- struct MyTask : FNET_Task {
- FastS_FNETAdapter &self;
- MyTask(FNET_Scheduler *scheduler, FastS_FNETAdapter &self_in)
- : FNET_Task(scheduler), self(self_in) {}
- virtual void PerformTask() override {
- self.perform();
- ScheduleNow();
- }
- };
- std::unique_ptr<MyTask> _task;
-
-public:
- FastS_FNETAdapter(FastS_AppContext *appCtx);
- ~FastS_FNETAdapter();
- void init();
- void perform();
- uint32_t GetLiveCounter() const { return _live_counter; }
- void fini();
-};
-
-
-/**
- * Note: There is only one instance of this.
- */
-class Fdispatch : public FastS_AppContext,
- public config::IFetcherCallback<vespa::config::search::core::FdispatchrcConfig>
-{
-private:
- typedef search::engine::TransportServer TransportServer;
- typedef vespa::config::search::core::FdispatchrcConfig FdispatchrcConfig;
- Fdispatch(const Fdispatch &);
- Fdispatch& operator=(const Fdispatch &);
-
- vespalib::ThreadStackExecutor _executor;
- std::unique_ptr<FastOS_ThreadPool> _mypool;
- std::unique_ptr<EngineAdapter> _engineAdapter;
- std::unique_ptr<TransportServer> _transportServer;
- vespalib::SimpleComponentConfigProducer _componentConfig;
- std::unique_ptr<FastS_NodeManager> _nodeManager;
- std::unique_ptr<FNET_Transport> _transport;
- FastS_FNETAdapter _FNET_adapter;
- std::unique_ptr<FastS_fdispatch_RPC> _rpc;
- std::unique_ptr<FdispatchrcConfig> _config;
- config::ConfigUri _configUri;
- config::ConfigFetcher _fdispatchrcFetcher;
- vespalib::RandomGen _rndGen;
- unsigned int _partition;
- bool _tempFail;
- bool _FNETLiveCounterDanger;
- bool _FNETLiveCounterWarned;
- bool _FNETLiveCounterFailed;
- bool _transportStarted;
- unsigned int _lastFNETLiveCounter;
- FastOS_Time _FNETLiveCounterDangerStart;
- unsigned int _timeouts;
- unsigned int _checkLimit;
- int _healthPort;
- std::atomic<bool> _needRestart;
- void configure(std::unique_ptr<FdispatchrcConfig> cfg) override;
-public:
- // Implements FastS_AppContext
- virtual FNET_Transport *GetFNETTransport() override;
- virtual FNET_Scheduler *GetFNETScheduler() override;
- virtual FastS_NodeManager *GetNodeManager() override;
- virtual FastS_DataSetCollection *GetDataSetCollection() override;
- virtual FastOS_ThreadPool *GetThreadPool() override;
- virtual uint32_t getDispatchLevel() override;
- bool CheckTempFail();
- bool Failed();
- bool Init();
- int getHealthPort() const { return _healthPort; }
- vespalib::SimpleComponentConfigProducer &getComponentConfig() { return _componentConfig; }
-
- Fdispatch(const config::ConfigUri &configUri);
- ~Fdispatch();
-};
-
-}
-
diff --git a/searchcore/src/vespa/searchcore/fdispatch/program/rpc.cpp b/searchcore/src/vespa/searchcore/fdispatch/program/rpc.cpp
deleted file mode 100644
index df8711adb88..00000000000
--- a/searchcore/src/vespa/searchcore/fdispatch/program/rpc.cpp
+++ /dev/null
@@ -1,113 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-
-#include "rpc.h"
-#include <vespa/searchcore/fdispatch/search/engine_base.h>
-#include <vespa/searchcore/fdispatch/search/datasetcollection.h>
-
-void
-FastS_fdispatch_RPC::RegisterMethods(FRT_ReflectionBuilder *rb)
-{
- FastS_RPC::RegisterMethods(rb);
- //------------------------------------------------------------------
- rb->DefineMethod("fs.admin.enableEngine", "s", "i",
- FRT_METHOD(FastS_fdispatch_RPC::RPC_EnableEngine), this);
- rb->MethodDesc("Enable the given engine (clear badness).");
- rb->ParamDesc("name", "engine name");
- rb->ReturnDesc("count", "number of engines affected");
- //------------------------------------------------------------------
- rb->DefineMethod("fs.admin.disableEngine", "s", "i",
- FRT_METHOD(FastS_fdispatch_RPC::RPC_DisableEngine), this);
- rb->MethodDesc("Disable the given engine (mark as admin bad).");
- rb->ParamDesc("name", "engine name");
- rb->ReturnDesc("count", "number of engines affected");
-}
-
-
-void
-FastS_fdispatch_RPC::RPC_GetNodeType(FRT_RPCRequest *req)
-{
- req->GetReturn()->AddString("dispatch");
-}
-
-namespace {
-
-template<class FUN>
-struct ExecuteWhenEqualName_t {
- FUN _successFun;
- const char* _targetName;
- uint32_t _cnt;
-
- ExecuteWhenEqualName_t(const char* targetName, FUN successFun)
- : _successFun(successFun),
- _targetName(targetName),
- _cnt(0)
- {}
-
- void operator()(FastS_EngineBase* engine) {
- if (strcmp(engine->GetName(), _targetName) == 0 ) {
- _cnt++;
- _successFun(engine);
- }
- }
-};
-
-template <class FUN>
-ExecuteWhenEqualName_t<FUN>
-ExecuteWhenEqualName(const char* targetName, FUN successFun) {
- return ExecuteWhenEqualName_t<FUN>(targetName, successFun);
-}
-
-
-} //anonymous namespace
-
-void
-FastS_fdispatch_RPC::RPC_EnableEngine(FRT_RPCRequest *req)
-{
- const char *name = req->GetParams()->GetValue(0)._string._str;
- FastS_DataSetCollection *dsc = GetAppCtx()->GetDataSetCollection();
- uint32_t cnt = 0;
-
- for (uint32_t i = 0; i < dsc->GetMaxNumDataSets(); i++) {
- FastS_DataSetBase *ds;
- FastS_PlainDataSet *ds_plain;
- if ((ds = dsc->PeekDataSet(i)) == NULL ||
- (ds_plain = ds->GetPlainDataSet()) == NULL)
- continue;
-
- cnt += ds_plain->ForEachEngine(
- ExecuteWhenEqualName(name,
- std::mem_fn( &FastS_EngineBase::ClearBad )))
- ._cnt;
- }
-
- dsc->subRef();
- req->GetReturn()->AddInt32(cnt);
-}
-
-
-void
-FastS_fdispatch_RPC::RPC_DisableEngine(FRT_RPCRequest *req)
-{
- const char *name = req->GetParams()->GetValue(0)._string._str;
- FastS_DataSetCollection *dsc = GetAppCtx()->GetDataSetCollection();
- uint32_t cnt = 0;
-
- for (uint32_t i = 0; i < dsc->GetMaxNumDataSets(); i++) {
- FastS_DataSetBase *ds;
- FastS_PlainDataSet *ds_plain;
- if ((ds = dsc->PeekDataSet(i)) == NULL ||
- (ds_plain = ds->GetPlainDataSet()) == NULL)
- continue;
-
- uint32_t badness = FastS_EngineBase::BAD_ADMIN;
- cnt += ds_plain->ForEachEngine(
- ExecuteWhenEqualName(name,
- std::bind(
- std::mem_fn( &FastS_EngineBase::MarkBad ),
- std::placeholders::_1,
- badness)))
- ._cnt;
- }
- dsc->subRef();
- req->GetReturn()->AddInt32(cnt);
-}
diff --git a/searchcore/src/vespa/searchcore/fdispatch/program/rpc.h b/searchcore/src/vespa/searchcore/fdispatch/program/rpc.h
deleted file mode 100644
index 6ca4e020d77..00000000000
--- a/searchcore/src/vespa/searchcore/fdispatch/program/rpc.h
+++ /dev/null
@@ -1,28 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-
-#pragma once
-
-#include <vespa/searchcore/fdispatch/common/rpc.h>
-
-
-class FastS_fdispatch_RPC : public FastS_RPC
-{
-public:
- FastS_fdispatch_RPC(FastS_AppContext *appCtx)
- : FastS_RPC(appCtx) {}
- virtual ~FastS_fdispatch_RPC() {}
-
- // Register RPC Methods
-
- virtual void RegisterMethods(FRT_ReflectionBuilder *rb) override;
-
- // methods registered by superclass
-
- virtual void RPC_GetNodeType(FRT_RPCRequest *req) override;
-
- // methods registered by us
-
- void RPC_EnableEngine(FRT_RPCRequest *req);
- void RPC_DisableEngine(FRT_RPCRequest *req);
-};
-
diff --git a/searchcore/src/vespa/searchcore/fdispatch/program/searchadapter.cpp b/searchcore/src/vespa/searchcore/fdispatch/program/searchadapter.cpp
deleted file mode 100644
index 824688a75f6..00000000000
--- a/searchcore/src/vespa/searchcore/fdispatch/program/searchadapter.cpp
+++ /dev/null
@@ -1,109 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-
-#include "searchadapter.h"
-#include <vespa/searchcore/fdispatch/search/datasetcollection.h>
-#include <vespa/searchcore/fdispatch/search/dataset_base.h>
-#include <vespa/searchcore/fdispatch/search/nodemanager.h>
-
-namespace fdispatch {
-
-void
-SearchAdapter::handleRequest()
-{
- _dsc = _appCtx->GetDataSetCollection();
- FastS_assert(_dsc != NULL);
-
- uint32_t dataset = _dsc->SuggestDataSet();
-
- _search = _dsc->CreateSearch(dataset, _appCtx->GetTimeKeeper());
- FastS_assert(_search != NULL);
-
- _searchInfo = _search->GetSearchInfo();
- _queryResult = _search->GetQueryResult();
- _search->setSearchRequest(_request.get());
- _search->Search(_request->offset, _request->maxhits, /* minhits */ 0);
- _search->ProcessQueryDone();
-}
-
-SearchAdapter::SearchReply::UP
-SearchAdapter::createReply()
-{
- SearchReply::UP reply(new SearchReply());
- SearchReply &r = *reply;
- r.useWideHits = true; // mld
- if (_search->GetErrorCode() != search::engine::ECODE_NO_ERROR) {
- r.errorCode = _search->GetErrorCode();
- r.errorMessage = _search->GetErrorMessage();
- return reply;
- }
-
- uint32_t hitcnt = _queryResult->_hitCount;
- r.offset = _searchInfo->_searchOffset;
- r.totalHitCount = _queryResult->_totalHitCount;
- r.maxRank = _queryResult->_maxRank;
- r.setDistributionKey(_appCtx->GetNodeManager()->GetMldDocstamp());
-
- if (_queryResult->_sortIndex != NULL && hitcnt > 0) {
- r.sortIndex.assign(_queryResult->_sortIndex, _queryResult->_sortIndex + hitcnt + 1);
- r.sortData.assign(_queryResult->_sortData, _queryResult->_sortData + _queryResult->_sortIndex[hitcnt]);
- }
-
- if (_queryResult->_groupResultLen > 0) {
- r.groupResult.assign(_queryResult->_groupResult,
- _queryResult->_groupResult + _queryResult->_groupResultLen);
- }
-
- r.coverage = SearchReply::Coverage(_searchInfo->_activeDocs, _searchInfo->_coverageDocs);
- r.coverage.setSoonActive(_searchInfo->_soonActiveDocs);
- r.coverage.setDegradeReason(_searchInfo->_degradeReason);
- r.coverage.setNodesQueried(_searchInfo->_nodesQueried);
- r.coverage.setNodesReplied(_searchInfo->_nodesReplied);
-
- FastS_hitresult *hitbuf = _queryResult->_hitbuf;
- r.hits.resize(hitcnt);
-
- for (uint32_t cur = 0; cur < hitcnt; cur++) {
- r.hits[cur].gid = hitbuf[cur]._gid;
- r.hits[cur].metric = hitbuf[cur]._metric;
- r.hits[cur].path = hitbuf[cur]._partition;
- r.hits[cur].setDistributionKey(hitbuf[cur].getDistributionKey());
- }
- r.request = _request.release();
- return reply;
-}
-
-void
-SearchAdapter::cleanup()
-{
- if (_search != NULL) {
- _search->Free();
- }
- if (_dsc != NULL) {
- _dsc->subRef();
- }
-}
-
-void
-SearchAdapter::Run(FastOS_ThreadInterface *, void *)
-{
- handleRequest();
- SearchReply::UP reply = createReply();
- cleanup();
- _client.searchDone(std::move(reply));
- delete this;
-}
-
-SearchAdapter::SearchAdapter(FastS_AppContext *appCtx,
- SearchRequest::Source request,
- SearchClient &client)
- : _appCtx(appCtx),
- _request(std::move(request)),
- _client(client),
- _dsc(0),
- _search(0),
- _searchInfo(0),
- _queryResult(0)
-{
-}
-
-} // namespace fdispatch
diff --git a/searchcore/src/vespa/searchcore/fdispatch/program/searchadapter.h b/searchcore/src/vespa/searchcore/fdispatch/program/searchadapter.h
deleted file mode 100644
index 32cadfd0648..00000000000
--- a/searchcore/src/vespa/searchcore/fdispatch/program/searchadapter.h
+++ /dev/null
@@ -1,48 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-
-#pragma once
-
-#include <vespa/searchlib/engine/searchapi.h>
-#include <vespa/searchcore/fdispatch/common/appcontext.h>
-#include <vespa/searchcore/fdispatch/common/search.h>
-#include <vespa/fastos/thread.h>
-
-namespace fdispatch {
-
-/**
- * Implementation of the common search api for the fdispatch server
- * application.
- **/
-class SearchAdapter : public FastOS_Runnable
-{
-public:
- typedef search::engine::SearchRequest SearchRequest;
- typedef search::engine::SearchReply SearchReply;
- typedef search::engine::SearchClient SearchClient;
-
-private:
- FastS_AppContext *_appCtx;
- SearchRequest::Source _request;
- SearchClient &_client;
-
- // internal search related state
- FastS_DataSetCollection *_dsc;
- FastS_ISearch *_search;
- FastS_SearchInfo *_searchInfo;
- FastS_QueryResult *_queryResult;
-
- void handleRequest();
- SearchReply::UP createReply();
- void writeLog();
- void cleanup();
-
- void Run(FastOS_ThreadInterface *, void *) override;
-
-public:
- SearchAdapter(FastS_AppContext *appCtx,
- SearchRequest::Source request,
- SearchClient &client);
-};
-
-} // namespace fdispatch
-
diff --git a/searchcore/src/vespa/searchcore/fdispatch/search/.gitignore b/searchcore/src/vespa/searchcore/fdispatch/search/.gitignore
deleted file mode 100644
index ca1a057edea..00000000000
--- a/searchcore/src/vespa/searchcore/fdispatch/search/.gitignore
+++ /dev/null
@@ -1,3 +0,0 @@
-*.lib
-.depend
-Makefile
diff --git a/searchcore/src/vespa/searchcore/fdispatch/search/CMakeLists.txt b/searchcore/src/vespa/searchcore/fdispatch/search/CMakeLists.txt
deleted file mode 100644
index ec8b7d18143..00000000000
--- a/searchcore/src/vespa/searchcore/fdispatch/search/CMakeLists.txt
+++ /dev/null
@@ -1,21 +0,0 @@
-# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-vespa_add_library(searchcore_fdispatch_search STATIC
- SOURCES
- configdesc.cpp
- dataset_base.cpp
- datasetcollection.cpp
- engine_base.cpp
- fnet_dataset.cpp
- fnet_engine.cpp
- fnet_search.cpp
- mergehits.cpp
- nodemanager.cpp
- plain_dataset.cpp
- query.cpp
- querycacheutil.cpp
- rowstate.cpp
- search_path.cpp
- DEPENDS
- searchcore_fconfig
- searchcore_util
-)
diff --git a/searchcore/src/vespa/searchcore/fdispatch/search/child_info.h b/searchcore/src/vespa/searchcore/fdispatch/search/child_info.h
deleted file mode 100644
index 3568aa384dd..00000000000
--- a/searchcore/src/vespa/searchcore/fdispatch/search/child_info.h
+++ /dev/null
@@ -1,22 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-
-#pragma once
-
-
-#include "poss_count.h"
-
-struct ChildInfo {
- uint32_t maxNodes;
- uint32_t activeNodes;
- uint32_t maxParts;
- uint32_t activeParts;
- PossCount activeDocs;
-
- ChildInfo()
- : maxNodes(0),
- activeNodes(0),
- maxParts(0),
- activeParts(0),
- activeDocs()
- {}
-};
diff --git a/searchcore/src/vespa/searchcore/fdispatch/search/configdesc.cpp b/searchcore/src/vespa/searchcore/fdispatch/search/configdesc.cpp
deleted file mode 100644
index 045f5b20ee0..00000000000
--- a/searchcore/src/vespa/searchcore/fdispatch/search/configdesc.cpp
+++ /dev/null
@@ -1,344 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-
-#include "configdesc.h"
-#include <vespa/searchcore/util/log.h>
-
-#include <vespa/log/log.h>
-LOG_SETUP(".search.configdesc");
-
-//----------------------------------------------------------------------
-
-double FastS_DataSetDesc::_defaultSlowQueryLimitFactor = 0.0;
-double FastS_DataSetDesc::_defaultSlowQueryLimitBias = 100.0;
-double FastS_DataSetDesc::_defaultSlowDocsumLimitFactor = 0.0;
-double FastS_DataSetDesc::_defaultSlowDocsumLimitBias = 100.0;
-
-
-FastS_DataSetDesc::FastS_DataSetDesc(uint32_t datasetid)
- : _id(datasetid),
- _queryDistributionMode(QueryDistributionMode::AUTOMATIC, 100.0, 10000),
- _searchableCopies(1),
- _unitRefCost(0),
- _partBits(6),
- _rowBits(0),
- _numParts(0),
- _firstPart(0),
- _minChildParts(0),
- _maxNodesDownPerFixedRow(0),
- _useRoundRobinForFixedRow(true),
- _maxHitsPerNode(static_cast<uint32_t>(-1)),
- _estimateParts(1),
- _estPartCutoff(1),
- _estimatePartsSet(false),
- _estPartCutoffSet(false),
- _minOurActive(500),
- _maxOurActive(500),
- _cutoffOurActive(1000),
- _minEstActive(500),
- _maxEstActive(1000),
- _cutoffEstActive(1000),
- _queueDrainRate(400.0),
- _queueMaxDrain(40.0),
- _slowQueryLimitFactor(_defaultSlowQueryLimitFactor),
- _slowQueryLimitBias(_defaultSlowQueryLimitBias),
- _slowDocsumLimitFactor(_defaultSlowDocsumLimitFactor),
- _slowDocsumLimitBias(_defaultSlowDocsumLimitBias),
- _monitorInterval(1.0),
- _higherCoverageMaxSearchWait(1.0),
- _higherCoverageMinSearchWait(0.0),
- _higherCoverageBaseSearchWait(0.1),
- _minimalSearchCoverage(100.0),
- _higherCoverageMaxDocSumWait(0.3),
- _higherCoverageMinDocSumWait(0.1),
- _higherCoverageBaseDocSumWait(0.1),
- _minimalDocSumCoverage(100.0),
- _engineCnt(0),
- _enginesHead(NULL),
- _enginesTail(NULL),
- _mpp(1)
-{
-}
-
-
-FastS_DataSetDesc::~FastS_DataSetDesc()
-{
- while (_enginesHead != NULL) {
- FastS_EngineDesc *engine = _enginesHead;
- _enginesHead = engine->GetNext();
- delete engine;
- }
-}
-
-
-FastS_EngineDesc *
-FastS_DataSetDesc::AddEngine(const char *name)
-{
- FastS_EngineDesc *engine = new FastS_EngineDesc(name);
- FastS_assert(engine != NULL);
-
- engine->SetNext(NULL);
- if (_enginesHead == NULL)
- _enginesHead = engine;
- else
- _enginesTail->SetNext(engine);
- _enginesTail = engine;
- _engineCnt++;
- return engine;
-}
-
-
-void
-FastS_DataSetDesc::FinalizeConfig()
-{
- /* assume 1 partition if number of partitions was not specified */
- if (GetNumParts() == 0) {
- LOG(warning,
- "Setting partitions to 1 in dataset %u",
- (unsigned int) GetID());
- SetNumParts(1);
- }
-
- if (!_estPartCutoffSet ||
- _estPartCutoff > _numParts ||
- _estPartCutoff == 0)
- _estPartCutoff = _numParts;
-}
-
-//----------------------------------------------------------------------
-
-bool
-FastS_DataSetCollDesc::CheckIntegrity()
-{
- bool rc = true;
-
- for (uint32_t i = 0; i < _datasets_size; i++) {
- FastS_DataSetDesc *d = _datasets[i];
- if (d != NULL) {
- if (d->GetEngineCnt() == 0) {
- LOG(warning, "plain dataset %d has no engines", d->GetID());
- }
-
- if (d->GetNumParts() == 0) {
- LOG(warning, "plain dataset %d has no partitions", d->GetID());
- }
-
- // check engine configuration
- {
- uint32_t partBits = d->GetPartBits();
- uint32_t rowBits = d->GetRowBits();
- uint32_t minPart = d->GetFirstPart();
- uint32_t maxPart = minPart + (1 << partBits) - 2;
- uint32_t maxRow = (rowBits > 0)? (1 << rowBits) - 1 : 0;
- uint32_t enginePartCnt = 0;
- FastS_assert(partBits > 0);
- bool *partidUsed = new bool[maxPart];
- for (uint32_t j = 0; j < maxPart; j++)
- partidUsed[j] = false;
-
- for (FastS_EngineDesc *engine = d->GetEngineList();
- engine != NULL; engine = engine->GetNext()) {
-
- bool bad = false;
- uint32_t partid = engine->GetConfPartID();
- uint32_t rowid = engine->GetConfRowID();
-
- if (partid != FastS_NoID32() &&
- (partid < minPart || partid > maxPart))
- {
- LOG(error, "engine '%s' in dataset %d has partid %d, legal range is [%d,%d] (partbits = %d)",
- engine->GetName(), d->GetID(), partid,
- minPart, maxPart, partBits);
- bad = true;
- }
-
- if (rowid && rowid != FastS_NoID32()) {
- if (rowBits == 0) {
- LOG(warning, "rowid (%d) on engine '%s' in dataset %d "
- "will be ignored because rowbits is 0",
- rowid, engine->GetName(), d->GetID());
- } else if (rowid > maxRow) {
- LOG(error, "engine '%s' in dataset %d has rowid %d, legal range is [%d,%d] (rowbits = %d)",
- engine->GetName(), d->GetID(), rowid,
- 0, maxRow, rowBits);
- bad = true;
- }
- }
- if (bad) {
- LOG(error, "marking engine '%s' in dataset %d as BAD due to illegal configuration",
- engine->GetName(), d->GetID());
- engine->MarkBad();
- }
-
- if (partid != FastS_NoID32() &&
- (partid >= minPart || partid <= maxPart)) {
- if (!partidUsed[partid]) {
- enginePartCnt++;
- partidUsed[partid] = true;
- }
- } else {
- enginePartCnt++;
- }
- }
- delete [] partidUsed;
- if (d->GetNumParts() < enginePartCnt) {
- LOG(warning,
- "plain dataset %d has "
- "%d engines with different partids, "
- "but only %d partitions",
- d->GetID(),
- enginePartCnt,
- d->GetNumParts());
- }
- }
- }
- }
-
- return rc;
-}
-
-
-
-FastS_DataSetCollDesc::FastS_DataSetCollDesc()
- : _datasets(NULL),
- _datasets_size(0),
- _frozen(false),
- _error(false)
-{
-}
-
-
-FastS_DataSetCollDesc::~FastS_DataSetCollDesc()
-{
- if (_datasets != NULL) {
- for (uint32_t i = 0; i < _datasets_size; i++) {
- if (_datasets[i] != NULL) {
- delete _datasets[i];
- }
- }
- delete [] _datasets;
- }
-}
-
-
-FastS_DataSetDesc *
-FastS_DataSetCollDesc::LookupCreateDataSet(uint32_t datasetid)
-{
- FastS_assert(!_frozen);
-
- if (datasetid >= _datasets_size) {
- uint32_t newSize = datasetid + 1;
-
- FastS_DataSetDesc **newArray = new FastS_DataSetDesc*[newSize];
- FastS_assert(newArray != NULL);
-
- uint32_t i;
- for (i = 0; i < _datasets_size; i++)
- newArray[i] = _datasets[i];
-
- for (; i < newSize; i++)
- newArray[i] = NULL;
-
- delete [] _datasets;
- _datasets = newArray;
- _datasets_size = newSize;
- }
-
- if (_datasets[datasetid] == NULL) {
- _datasets[datasetid] = new FastS_DataSetDesc(datasetid);
- }
-
- return _datasets[datasetid];
-}
-
-
-bool
-FastS_DataSetCollDesc::Freeze()
-{
- if (!_frozen) {
- _frozen = true;
-
- for (uint32_t i = 0; i < _datasets_size; i++)
- if (_datasets[i] != NULL)
- _datasets[i]->FinalizeConfig();
-
- _error = !CheckIntegrity();
- }
- return !_error;
-}
-
-//----------------------------------------------------------------------
-bool
-FastS_DataSetCollDesc::ReadConfig(const PartitionsConfig& partmap)
-{
- FastS_assert(!_frozen);
-
- int datasetcnt = partmap.dataset.size();
-
- if (datasetcnt < 1) {
- LOG(error, "no datasets in partitions config");
- return false;
- }
- for (int i=0; i < datasetcnt; i++) {
- typedef PartitionsConfig::Dataset Dsconfig;
- const Dsconfig dsconfig = partmap.dataset[i];
-
- FastS_DataSetDesc *dataset = LookupCreateDataSet(dsconfig.id);
-
- dataset->setSearchableCopies(dsconfig.searchablecopies);
- dataset->SetUnitRefCost(dsconfig.refcost);
- dataset->SetPartBits(dsconfig.partbits);
- dataset->SetRowBits(dsconfig.rowbits);
- dataset->SetNumParts(dsconfig.numparts);
- dataset->SetMinChildParts(dsconfig.minpartitions);
- dataset->setMaxNodesDownPerFixedRow(dsconfig.maxnodesdownperfixedrow);
- dataset->useRoundRobinForFixedRow(dsconfig.useroundrobinforfixedrow);
- dataset->SetMaxHitsPerNode(dsconfig.maxhitspernode);
- dataset->SetFirstPart(dsconfig.firstpart);
- dataset->SetMinOurActive(dsconfig.minactive);
- dataset->SetMaxOurActive(dsconfig.maxactive);
- dataset->SetCutoffOurActive(dsconfig.cutoffactive);
- dataset->SetMinEstActive(dsconfig.minestactive);
- dataset->SetMaxEstActive(dsconfig.maxestactive);
- dataset->SetCutoffEstActive(dsconfig.cutoffestactive);
- dataset->SetQueueDrainRate(dsconfig.queuedrainrate);
- dataset->SetQueueMaxDrain(dsconfig.queuedrainmax);
- dataset->SetSlowQueryLimitFactor(dsconfig.slowquerylimitfactor);
- dataset->SetSlowQueryLimitBias(dsconfig.slowquerylimitbias);
- dataset->SetSlowDocsumLimitFactor(dsconfig.slowdocsumlimitfactor);
- dataset->SetSlowDocsumLimitBias(dsconfig.slowdocsumlimitbias);
- dataset->setMonitorInterval(dsconfig.monitorinterval);
- dataset->setHigherCoverageMaxSearchWait(dsconfig.higherCoverageMaxsearchwait);
- dataset->setHigherCoverageMinSearchWait(dsconfig.higherCoverageMinsearchwait);
- dataset->setHigherCoverageBaseSearchWait(dsconfig.higherCoverageBasesearchwait);
- dataset->setMinimalSearchCoverage(dsconfig.minimalSearchcoverage);
- dataset->setHigherCoverageMaxDocSumWait(dsconfig.higherCoverageMaxdocsumwait);
- dataset->setHigherCoverageMinDocSumWait(dsconfig.higherCoverageMindocsumwait);
- dataset->setHigherCoverageBaseDocSumWait(dsconfig.higherCoverageBasedocsumwait);
- dataset->setMinimalDocSumCoverage(dsconfig.minimalDocsumcoverage);
- FastS_DataSetDesc::QueryDistributionMode distMode(dsconfig.querydistribution,
- dsconfig.minGroupCoverage,
- dsconfig.latencyDecayRate);
- distMode.setMinActivedocsCoverage(dsconfig.minActivedocsCoverage);
- dataset->SetQueryDistributionMode(distMode);
- dataset->setMPP(dsconfig.mpp);
- if (dsconfig.estparts > 0)
- dataset->SetEstimateParts(dsconfig.estparts);
- if (dsconfig.estpartcutoff > 0)
- dataset->SetEstPartCutoff(dsconfig.estpartcutoff);
-
- int enginecnt = dsconfig.engine.size();
-
- for (int j=0; j < enginecnt; j++) {
- const Dsconfig::Engine& engconfig = dsconfig.engine[j];
-
- FastS_EngineDesc *engine = dataset->AddEngine(engconfig.nameAndPort.c_str());
-
- engine->SetUnitRefCost(engconfig.refcost);
- engine->SetConfRowID(engconfig.rowid);
- engine->SetConfPartID(engconfig.partid);
- if (engconfig.overridepartids)
- engine->SetConfPartIDOverrides();
- }
- }
- return true;
-}
diff --git a/searchcore/src/vespa/searchcore/fdispatch/search/configdesc.h b/searchcore/src/vespa/searchcore/fdispatch/search/configdesc.h
deleted file mode 100644
index e0b0f0d7403..00000000000
--- a/searchcore/src/vespa/searchcore/fdispatch/search/configdesc.h
+++ /dev/null
@@ -1,374 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-
-#pragma once
-
-#include <vespa/searchlib/common/fslimits.h>
-#include <vespa/searchcore/fdispatch/common/stdincl.h>
-#include <vespa/searchcore/config/config-partitions.h>
-#include <cassert>
-
-using vespa::config::search::core::PartitionsConfig;
-
-//-----------------------------------------------------------------------
-
-class FastS_EngineDesc
-{
-private:
- FastS_EngineDesc(const FastS_EngineDesc &);
- FastS_EngineDesc& operator=(const FastS_EngineDesc &);
-
- FastS_EngineDesc *_next;
- std::string _name;
- uint32_t _confPartID;
- uint32_t _confRowID;
- uint32_t _unitrefcost;
- bool _isBad;
- bool _confPartIDOverrides;
-
-public:
- explicit FastS_EngineDesc(const char *name)
- : _next(NULL),
- _name(name),
- _confPartID(FastS_NoID32()),
- _confRowID(FastS_NoID32()),
- _unitrefcost(1),
- _isBad(false),
- _confPartIDOverrides(false)
- { }
-
- void SetNext(FastS_EngineDesc *next) { _next = next; }
- void SetConfPartID(int32_t value) { assert(value >= 0); _confPartID = value; }
- void SetConfPartIDOverrides() { _confPartIDOverrides = true; }
- void SetConfRowID(int32_t value) { assert(value >= 0); _confRowID = value; }
- void SetUnitRefCost(uint32_t value) { _unitrefcost = value; }
- void MarkBad() { _isBad = true; }
- FastS_EngineDesc * GetNext() const { return _next; }
- const char * GetName() const { return _name.c_str(); }
- uint32_t GetConfPartID() const { return _confPartID; }
- bool GetConfPartIDOverrides() const { return _confPartIDOverrides; }
- uint32_t GetConfRowID() const { return _confRowID; }
- uint32_t GetUnitRefCost() const { return _unitrefcost; }
- bool IsBad() const { return _isBad; }
-};
-
-//-----------------------------------------------------------------------
-
-class FastS_DataSetDesc
-{
-private:
- FastS_DataSetDesc(const FastS_DataSetDesc &);
- FastS_DataSetDesc& operator=(const FastS_DataSetDesc &);
-
- static double _defaultSlowQueryLimitFactor;
- static double _defaultSlowQueryLimitBias;
- static double _defaultSlowDocsumLimitFactor;
- static double _defaultSlowDocsumLimitBias;
-
-public:
-
- class QueryDistributionMode {
- public:
- enum Mode {
- RANDOM = static_cast<int>(PartitionsConfig::Dataset::Querydistribution::RANDOM),
- AUTOMATIC = static_cast<int>(PartitionsConfig::Dataset::Querydistribution::AUTOMATIC),
- FIXEDROW = static_cast<int>(PartitionsConfig::Dataset::Querydistribution::FIXEDROW)
- };
-
- QueryDistributionMode(Mode mode, double minGroupCoverage, double latencyDecayRate) :
- _mode(mode),
- _minGroupCoverage(minGroupCoverage),
- _latencyDecayRate(latencyDecayRate),
- _minActivedocsCoverage(0.0)
- { }
-
- QueryDistributionMode(PartitionsConfig::Dataset::Querydistribution mode, double minGroupCoverage, double latencyDecayRate) :
- QueryDistributionMode(static_cast<Mode>(mode), minGroupCoverage, latencyDecayRate)
- {
- }
-
- bool operator==(const QueryDistributionMode & rhs) const {
- return _mode == rhs._mode;
- }
- bool operator == (Mode rhs) const {
- return _mode == rhs;
- }
- double getMinGroupCoverage() const { return _minGroupCoverage; }
- double getLatencyDecayRate() const { return _latencyDecayRate; }
- double getMinActivedocsCoverage() const { return _minActivedocsCoverage; }
-
- void setMinActivedocsCoverage(double val) { _minActivedocsCoverage = val; }
- private:
- Mode _mode;
- double _minGroupCoverage;
- double _latencyDecayRate;
- double _minActivedocsCoverage;
- };
-
- static void SetDefaultSlowQueryLimitFactor(double value)
- { _defaultSlowQueryLimitFactor = value; }
-
- static void SetDefaultSlowQueryLimitBias(double value)
- { _defaultSlowQueryLimitBias = value; }
-
- static void SetDefaultSlowDocsumLimitFactor(double value)
- { _defaultSlowDocsumLimitFactor = value; }
-
- static void SetDefaultSlowDocsumLimitBias(double value)
- { _defaultSlowDocsumLimitBias = value; }
-
-private:
- uint32_t _id;
- QueryDistributionMode _queryDistributionMode;
-
- uint32_t _searchableCopies;
- uint32_t _unitRefCost; // Cost to reference us
- uint32_t _partBits; // # bits used to encode part id
- uint32_t _rowBits; // # bits used to encode row id
- uint32_t _numParts; // Number of partitions
- uint32_t _firstPart; // First partition
- uint32_t _minChildParts; // Minimum partitions live to avoid tempfail
- uint32_t _maxNodesDownPerFixedRow; // max number of nodes down in a row before considering another row.
- bool _useRoundRobinForFixedRow; // Either plain roundrobin or random.
- uint32_t _maxHitsPerNode; // max hits requested from single node
- uint32_t _estimateParts; // number of partitions used for estimate
- uint32_t _estPartCutoff; // First partition not used for estimate
- bool _estimatePartsSet; // has _estimateParts been set ?
- bool _estPartCutoffSet; // has _estimatePartsCutoff been set ?
- uint32_t _minOurActive; // below ==> activate, skip estimates
- uint32_t _maxOurActive; // above ==> queue
- uint32_t _cutoffOurActive; // Above ==> cutoff
- uint32_t _minEstActive; // est below ==> activate
- uint32_t _maxEstActive; // est below ==> queue, est above cutoff > 0%
- uint32_t _cutoffEstActive; // est above ==> cutoff 100%
- double _queueDrainRate; // max queue drain per second
- double _queueMaxDrain; // max queue drain at once
- double _slowQueryLimitFactor;
- double _slowQueryLimitBias;
- double _slowDocsumLimitFactor;
- double _slowDocsumLimitBias;
- double _monitorInterval;
- double _higherCoverageMaxSearchWait;
- double _higherCoverageMinSearchWait;
- double _higherCoverageBaseSearchWait;
- double _minimalSearchCoverage;
- double _higherCoverageMaxDocSumWait;
- double _higherCoverageMinDocSumWait;
- double _higherCoverageBaseDocSumWait;
- double _minimalDocSumCoverage;
-
- uint32_t _engineCnt; // number of search engines in dataset
- FastS_EngineDesc *_enginesHead; // first engine in dataset
- FastS_EngineDesc *_enginesTail; // last engine in dataset
-
- uint32_t _mpp; // Minimum number of engines per partition
-public:
- explicit FastS_DataSetDesc(uint32_t datasetid);
- ~FastS_DataSetDesc();
-
- uint32_t GetID() const { return _id; }
- void SetUnitRefCost(uint32_t value) { _unitRefCost = value; }
- void setSearchableCopies(uint32_t value) { _searchableCopies = value; }
-
- void SetPartBits(uint32_t value) {
- if (value >= MIN_PARTBITS && value <= MAX_PARTBITS)
- _partBits = value;
- }
-
- void SetRowBits(uint32_t value) {
- if (value <= MAX_ROWBITS)
- _rowBits = value;
- }
-
- void SetNumParts(uint32_t value) { _numParts = value; }
- void SetFirstPart(uint32_t value) { _firstPart = value; }
- void SetMinChildParts(uint32_t value) { _minChildParts = value; }
- void setMaxNodesDownPerFixedRow(uint32_t value) { _maxNodesDownPerFixedRow = value; }
- void useRoundRobinForFixedRow(bool value) { _useRoundRobinForFixedRow = value; }
- void SetMaxHitsPerNode(uint32_t value) { _maxHitsPerNode = value; }
- void SetEstimateParts(uint32_t value) {
- _estimateParts = value;
- _estimatePartsSet = true;
- }
-
- void SetEstPartCutoff(uint32_t value) {
- _estPartCutoff = value;
- _estPartCutoffSet = true;
- }
-
- void SetMinOurActive(uint32_t value) { _minOurActive = value; }
- void SetMaxOurActive(uint32_t value) { _maxOurActive = value; }
- void SetCutoffOurActive(uint32_t value) { _cutoffOurActive = value; }
- void SetMinEstActive(uint32_t value) { _minEstActive = value; }
- void SetMaxEstActive(uint32_t value) { _maxEstActive = value; }
- void SetCutoffEstActive(uint32_t value) { _cutoffEstActive = value; }
- void SetQueueDrainRate(double value) { _queueDrainRate = value; }
- void SetQueueMaxDrain(double value) { _queueMaxDrain = value; }
- void SetSlowQueryLimitFactor(double value) { _slowQueryLimitFactor = value; }
- void SetSlowQueryLimitBias(double value) { _slowQueryLimitBias = value; }
- void SetSlowDocsumLimitFactor(double value) { _slowDocsumLimitFactor = value; }
- void SetSlowDocsumLimitBias(double value) { _slowDocsumLimitBias = value; }
-
- void SetQueryDistributionMode(QueryDistributionMode queryDistributionMode) {
- _queryDistributionMode = queryDistributionMode;
- }
-
- QueryDistributionMode GetQueryDistributionMode() { return _queryDistributionMode; }
-
- FastS_EngineDesc * AddEngine(const char *name);
- uint32_t GetUnitRefCost() const { return _unitRefCost; }
- uint32_t GetPartBits() const { return _partBits; }
-
- uint32_t GetRowBits() const { return _rowBits; }
- uint32_t GetNumParts() const { return _numParts; }
- uint32_t GetFirstPart() const { return _firstPart; }
- uint32_t GetMinChildParts() const { return _minChildParts; }
- uint32_t getMaxNodesDownPerFixedRow() const { return _maxNodesDownPerFixedRow; }
- bool useRoundRobinForFixedRow() const { return _useRoundRobinForFixedRow; }
- uint32_t GetMaxHitsPerNode() const { return _maxHitsPerNode; }
- uint32_t GetEstimateParts() const { return _estimateParts; }
- uint32_t GetEstPartCutoff() const { return _estPartCutoff; }
- bool IsEstimatePartsSet() const { return _estimatePartsSet; }
- bool IsEstPartCutoffSet() const { return _estPartCutoffSet; }
- uint32_t getSearchableCopies() const { return _searchableCopies; }
- uint32_t GetMinOurActive() const { return _minOurActive; }
- uint32_t GetMaxOurActive() const { return _maxOurActive; }
- uint32_t GetCutoffOurActive() const { return _cutoffOurActive; }
- uint32_t GetMinEstActive() const { return _minEstActive; }
- uint32_t GetMaxEstActive() const { return _maxEstActive; }
- uint32_t GetCutoffEstActive() const { return _cutoffEstActive; }
- double GetQueueDrainRate() const { return _queueDrainRate; }
- double GetQueueMaxDrain() const { return _queueMaxDrain; }
- double GetSlowQueryLimitFactor() const { return _slowQueryLimitFactor; }
- double GetSlowQueryLimitBias() const { return _slowQueryLimitBias; }
- double GetSlowDocsumLimitFactor() const { return _slowDocsumLimitFactor; }
- double GetSlowDocsumLimitBias() const { return _slowDocsumLimitBias; }
- uint32_t GetEngineCnt() const { return _engineCnt; }
- FastS_EngineDesc * GetEngineList() const { return _enginesHead; }
- void setMPP(uint32_t mpp) { _mpp = mpp; }
- uint32_t getMPP() const { return _mpp; }
-
- void
- setMonitorInterval(double monitorInterval) { _monitorInterval = monitorInterval; }
- double getMonitorInterval() const { return _monitorInterval; }
-
- void
- setHigherCoverageMaxSearchWait(double higherCoverageMaxSearchWait) {
- _higherCoverageMaxSearchWait = higherCoverageMaxSearchWait;
- }
-
- double
- getHigherCoverageMaxSearchWait() const {
- return _higherCoverageMaxSearchWait;
- }
-
- void
- setHigherCoverageMinSearchWait(double higherCoverageMinSearchWait) {
- _higherCoverageMinSearchWait = higherCoverageMinSearchWait;
- }
-
- double
- getHigherCoverageMinSearchWait() const {
- return _higherCoverageMinSearchWait;
- }
-
- void
- setHigherCoverageBaseSearchWait(double higherCoverageBaseSearchWait) {
- _higherCoverageBaseSearchWait = higherCoverageBaseSearchWait;
- }
-
- double
- getHigherCoverageBaseSearchWait() const {
- return _higherCoverageBaseSearchWait;
- }
-
- void
- setMinimalSearchCoverage(double minimalSearchCoverage) {
- _minimalSearchCoverage = minimalSearchCoverage;
- }
-
- double
- getMinimalSearchCoverage() const {
- return _minimalSearchCoverage;
- }
-
- void
- setHigherCoverageMaxDocSumWait(double higherCoverageMaxDocSumWait) {
- _higherCoverageMaxDocSumWait = higherCoverageMaxDocSumWait;
- }
-
- double
- getHigherCoverageMaxDocSumWait() const {
- return _higherCoverageMaxDocSumWait;
- }
-
- void
- setHigherCoverageMinDocSumWait(double higherCoverageMinDocSumWait) {
- _higherCoverageMinDocSumWait = higherCoverageMinDocSumWait;
- }
-
- double
- getHigherCoverageMinDocSumWait() const {
- return _higherCoverageMinDocSumWait;
- }
-
- void
- setHigherCoverageBaseDocSumWait(double higherCoverageBaseDocSumWait) {
- _higherCoverageBaseDocSumWait = higherCoverageBaseDocSumWait;
- }
-
- double
- getHigherCoverageBaseDocSumWait() const {
- return _higherCoverageBaseDocSumWait;
- }
-
- void
- setMinimalDocSumCoverage(double minimalDocSumCoverage) {
- _minimalDocSumCoverage = minimalDocSumCoverage;
- }
-
- double
- getMinimalDocSumCoverage() const {
- return _minimalDocSumCoverage;
- }
-
- void FinalizeConfig();
-};
-
-//-----------------------------------------------------------------------
-
-class FastS_DataSetCollDesc
-{
-private:
- FastS_DataSetCollDesc(const FastS_DataSetCollDesc &);
- FastS_DataSetCollDesc& operator=(const FastS_DataSetCollDesc &);
-
- FastS_DataSetDesc **_datasets;
- uint32_t _datasets_size;
-
- bool _frozen;
- bool _error;
-
- void HandleDeprecatedFPEstPartsOption();
- bool CheckIntegrity();
-
-public:
- FastS_DataSetCollDesc();
- ~FastS_DataSetCollDesc();
-
- FastS_DataSetDesc *LookupCreateDataSet(uint32_t datasetid);
-
- bool Freeze();
-
- uint32_t GetMaxNumDataSets() const { return _datasets_size; }
-
- FastS_DataSetDesc *GetDataSet(uint32_t datasetid) const {
- return (datasetid < _datasets_size)
- ? _datasets[datasetid]
- : NULL;
- }
-
- bool ReadConfig(const PartitionsConfig& partmap);
-};
-
-//-----------------------------------------------------------------------
-
diff --git a/searchcore/src/vespa/searchcore/fdispatch/search/dataset_base.cpp b/searchcore/src/vespa/searchcore/fdispatch/search/dataset_base.cpp
deleted file mode 100644
index 519960bfad0..00000000000
--- a/searchcore/src/vespa/searchcore/fdispatch/search/dataset_base.cpp
+++ /dev/null
@@ -1,294 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-
-#include "dataset_base.h"
-#include "configdesc.h"
-#include "datasetcollection.h"
-#include "engine_base.h"
-#include "nodemanager.h"
-
-//--------------------------------------------------------------------------
-
-FastS_DataSetBase::total_t::total_t()
- : _estimates(0),
- _nTimedOut(0),
- _nOverload(0),
- _normalTimeStat()
-{
- for (uint32_t i = 0; i < _timestatslots; i++)
- _timestats[i] = 0;
-}
-
-//--------------------------------------------------------------------------
-
-FastS_DataSetBase::overload_t::overload_t(FastS_DataSetDesc *desc)
- : _drainRate(desc->GetQueueDrainRate()),
- _drainMax(desc->GetQueueMaxDrain()),
- _minouractive(desc->GetMinOurActive()),
- _maxouractive(desc->GetMaxOurActive()),
- _cutoffouractive(desc->GetCutoffOurActive()),
- _minestactive(desc->GetMinEstActive()),
- _maxestactive(desc->GetMaxEstActive()),
- _cutoffestactive(desc->GetCutoffEstActive())
-{
-}
-
-//--------------------------------------------------------------------------
-
-FastS_DataSetBase::queryQueue_t::queryQueue_t(FastS_DataSetDesc *desc)
- : _head(nullptr),
- _tail(nullptr),
- _queueLen(0),
- _active(0),
- _drainAllowed(0.0),
- _drainStamp(0.0),
- _overload(desc)
-{
-}
-
-
-FastS_DataSetBase::queryQueue_t::~queryQueue_t()
-{
- FastS_assert(_active == 0);
-}
-
-
-void
-FastS_DataSetBase::queryQueue_t::QueueTail(queryQueued_t *newqueued)
-{
- FastS_assert(newqueued->_next == nullptr &&
- _head != newqueued &&
- _tail != newqueued);
- if (_tail != nullptr)
- _tail->_next = newqueued;
- else
- _head = newqueued;
- _tail = newqueued;
- _queueLen++;
-}
-
-
-void
-FastS_DataSetBase::queryQueue_t::DeQueueHead()
-{
- queryQueued_t *queued = _head;
- FastS_assert(_queueLen > 0);
- FastS_assert(queued->_next != nullptr || _tail == queued);
- _head = queued->_next;
- if (queued->_next == nullptr)
- _tail = nullptr;
- queued->_next = nullptr;
- _queueLen--;
-}
-
-//--------------------------------------------------------------------------
-
-FastS_DataSetBase::FastS_DataSetBase(FastS_AppContext *appCtx,
- FastS_DataSetDesc *desc)
- : _appCtx(appCtx),
- _lock(),
- _createtime(),
- _queryQueue(desc),
- _total(),
- _id(desc->GetID()),
- _unitrefcost(desc->GetUnitRefCost()),
- _totalrefcost(0),
- _mldDocStamp(0u),
- _searchableCopies(desc->getSearchableCopies())
-{
- _createtime.SetNow();
-}
-
-
-FastS_DataSetBase::~FastS_DataSetBase()
-{
- FastS_assert(_totalrefcost == 0);
-}
-
-void
-FastS_DataSetBase::ScheduleCheckTempFail()
-{
- _appCtx->GetNodeManager()->ScheduleCheckTempFail(_id);
-}
-
-
-void
-FastS_DataSetBase::DeQueueHeadWakeup_HasLock()
-{
- queryQueued_t *queued;
- queued = _queryQueue.GetFirst();
- FastS_assert(queued->IsQueued());
- auto queuedGuard(queued->getQueuedGuard());
- //SetNowFromMonitor();
- _queryQueue.DeQueueHead();
- queued->UnmarkQueued();
- FNET_Task *dequeuedTask = queued->getDequeuedTask();
- if (dequeuedTask != nullptr) {
- dequeuedTask->ScheduleNow();
- } else {
- queued->SignalCond();
- }
-}
-
-
-void
-FastS_DataSetBase::SetActiveQuery_HasLock()
-{
- _queryQueue.SetActiveQuery();
-}
-
-
-void
-FastS_DataSetBase::SetActiveQuery()
-{
- auto dsGuard(getDsGuard());
- SetActiveQuery_HasLock();
-}
-
-
-void
-FastS_DataSetBase::ClearActiveQuery_HasLock(FastS_TimeKeeper *timeKeeper)
-{
- FastS_assert(_queryQueue._active > 0);
- _queryQueue.ClearActiveQuery();
-
- CheckQueryQueue_HasLock(timeKeeper);
-}
-
-
-void
-FastS_DataSetBase::ClearActiveQuery(FastS_TimeKeeper *timeKeeper)
-{
- auto dsGuard(getDsGuard());
- ClearActiveQuery_HasLock(timeKeeper);
-}
-
-
-void
-FastS_DataSetBase::CheckQueryQueue_HasLock(FastS_TimeKeeper *timeKeeper)
-{
- queryQueued_t *queued;
- unsigned int active;
- unsigned int estactive;
- uint32_t dispatchnodes;
- double delay;
- double fnow;
-
- active = _queryQueue.GetActiveQueries(); // active from us
- estactive = CalculateQueueLens_HasLock(dispatchnodes);// active from us and others
-
- if (dispatchnodes == 0)
- dispatchnodes = 1;
-
- fnow = timeKeeper->GetTime();
- delay = fnow - _queryQueue._drainStamp;
- if (delay >= 0.0) {
- if (delay > 2.0) {
- delay = 2.0;
- if (_queryQueue._drainStamp == 0.0)
- _queryQueue._drainStamp = fnow;
- else
- _queryQueue._drainStamp += 2.0;
- } else
- _queryQueue._drainStamp = fnow;
- } else
- delay = 0.0;
-
- _queryQueue._drainAllowed += delay * _queryQueue._overload._drainRate;
- if (_queryQueue._drainAllowed >=
- _queryQueue._overload._drainMax + dispatchnodes - 1)
- _queryQueue._drainAllowed =
- _queryQueue._overload._drainMax + dispatchnodes - 1;
-
- while (_queryQueue._drainAllowed >= (double) dispatchnodes ||
- active < _queryQueue._overload._minouractive) {
- queued = _queryQueue.GetFirst();
- if (queued == nullptr) {
- return;
- }
-
- if (active >= _queryQueue._overload._maxouractive)
- return; // hard limit for how much we queue
-
- if (active >= _queryQueue._overload._minouractive &&
- estactive >= _queryQueue._overload._minestactive)
- return;
-
- // Dequeue query, count it active and wakeup thread handling query
- SetActiveQuery_HasLock();
- DeQueueHeadWakeup_HasLock();
-
- active++; // one more active from us
- estactive += dispatchnodes; // Assume other nodes do likewise
- if (_queryQueue._drainAllowed >= (double) dispatchnodes)
- _queryQueue._drainAllowed -= dispatchnodes; // Rate limitation
- else
- _queryQueue._drainAllowed = 0.0;
- }
-}
-
-
-void
-FastS_DataSetBase::AbortQueryQueue_HasLock()
-{
- queryQueued_t *queued;
-
- /*
- * Don't allow new queries to be queued.
- * Abort currently queued queries.
- */
- _queryQueue._overload._minouractive = 0;
- _queryQueue._overload._cutoffouractive = 0;
- for (;;) {
- queued = _queryQueue.GetFirst();
- if (queued == nullptr)
- break;
- // Doesn't lock query, but other thread is waiting on queue
- queued->MarkAbort();
- DeQueueHeadWakeup_HasLock();
- }
-}
-
-void
-FastS_DataSetBase::AddCost()
-{
- _totalrefcost += _unitrefcost;
-}
-
-void
-FastS_DataSetBase::SubCost()
-{
- FastS_assert(_totalrefcost >= _unitrefcost);
- _totalrefcost -= _unitrefcost;
-}
-
-void
-FastS_DataSetBase::UpdateSearchTime(double tnow, double elapsed, bool timedout)
-{
- int slot;
- auto dsGuard(getDsGuard());
- slot = (int) (elapsed * 10);
- if (slot >= _total._timestatslots)
- slot = _total._timestatslots - 1;
- else if (slot < 0)
- slot = 0;
- _total._timestats[slot]++;
- _total._normalTimeStat.Update(tnow, elapsed, timedout);
-}
-
-void
-FastS_DataSetBase::UpdateEstimateCount()
-{
- ++_total._estimates;
-}
-
-void
-FastS_DataSetBase::CountTimeout()
-{
- ++_total._nTimedOut;
-}
-
-ChildInfo
-FastS_DataSetBase::getChildInfo() const
-{
- return ChildInfo();
-}
diff --git a/searchcore/src/vespa/searchcore/fdispatch/search/dataset_base.h b/searchcore/src/vespa/searchcore/fdispatch/search/dataset_base.h
deleted file mode 100644
index f4f69285e89..00000000000
--- a/searchcore/src/vespa/searchcore/fdispatch/search/dataset_base.h
+++ /dev/null
@@ -1,227 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-
-#pragma once
-
-#include "child_info.h"
-#include <vespa/searchcore/fdispatch/common/timestat.h>
-#include <vespa/searchcore/util/log.h>
-#include <atomic>
-#include <vespa/fastos/time.h>
-#include <mutex>
-#include <condition_variable>
-
-class FastS_TimeKeeper;
-
-class FastS_DataSetDesc;
-class FastS_EngineDesc;
-class FastS_DataSetCollection;
-class FastS_ISearch;
-class FastS_QueryResult;
-class FastS_PlainDataSet;
-class FastS_FNET_DataSet;
-class FastS_AppContext;
-class FNET_Task;
-
-//---------------------------------------------------------------------------
-
-class FastS_DataSetBase
-{
- friend class FastS_DataSetCollection;
-public:
-
- //----------------------------------------------------------------
- // total query stats
- //----------------------------------------------------------------
-
- class total_t
- {
- public:
- enum {
- _timestatslots = 100
- };
- std::atomic<uint32_t> _estimates;
- std::atomic<uint32_t> _nTimedOut;
- uint32_t _nOverload;
- uint32_t _timestats[_timestatslots];
- FastS_TimeStatHistory _normalTimeStat;
- total_t();
- };
-
- //----------------------------------------------------------------
- // parameters used by query queue
- //----------------------------------------------------------------
-
- class overload_t
- {
- public:
- double _drainRate; // Queue drain rate
- double _drainMax; // Max queue drain at once
- uint32_t _minouractive; // minimum active requests from us
- uint32_t _maxouractive; // maximum active requests from us (queue)
- uint32_t _cutoffouractive; // cutoff active requests
- uint32_t _minestactive; // minimum estimated requests before queueing
- uint32_t _maxestactive; // maximum estimated requests (start earlydrop)
- uint32_t _cutoffestactive; // cutoff estimated requests (end earlydrop)
-
- overload_t(FastS_DataSetDesc *desc);
- };
-
- //----------------------------------------------------------------
- // class used to wait for a query queue
- //----------------------------------------------------------------
-
- class queryQueue_t;
- class queryQueued_t
- {
- friend class queryQueue_t;
- private:
- queryQueued_t(const queryQueued_t &);
- queryQueued_t& operator=(const queryQueued_t &);
-
- std::mutex _queuedLock;
- std::condition_variable _queuedCond;
- queryQueued_t *_next;
- bool _isAborted;
- bool _isQueued;
- FNET_Task *const _deQueuedTask;
- public:
- queryQueued_t(FNET_Task *const deQueuedTask)
- : _queuedLock(),
- _queuedCond(),
- _next(NULL),
- _isAborted(false),
- _isQueued(false),
- _deQueuedTask(deQueuedTask)
- {
- }
-
- ~queryQueued_t()
- {
- FastS_assert(!_isQueued);
- }
- void Wait() {
- std::unique_lock<std::mutex> queuedGuard(_queuedLock);
- while (_isQueued) {
- _queuedCond.wait(queuedGuard);
- }
- }
- bool IsAborted() const { return _isAborted; }
- void MarkAbort() { _isAborted = true; }
- void MarkQueued() { _isQueued = true; }
- void UnmarkQueued() { _isQueued = false; }
- bool IsQueued() const { return _isQueued; }
- std::unique_lock<std::mutex> getQueuedGuard() { return std::unique_lock<std::mutex>(_queuedLock); }
- void SignalCond() { _queuedCond.notify_one(); }
-
- FNET_Task *
- getDequeuedTask() const
- {
- return _deQueuedTask;
- }
- };
-
- //----------------------------------------------------------------
- // per dataset query queue
- //----------------------------------------------------------------
-
- class queryQueue_t
- {
- friend class FastS_DataSetBase;
-
- private:
- queryQueue_t(const queryQueue_t &);
- queryQueue_t& operator=(const queryQueue_t &);
-
- queryQueued_t *_head;
- queryQueued_t *_tail;
- unsigned int _queueLen;
- unsigned int _active;
-
- public:
- double _drainAllowed; // number of drainable request
- double _drainStamp; // stamp of last drain check
- overload_t _overload; // queue parameters
-
- public:
- queryQueue_t(FastS_DataSetDesc *desc);
- ~queryQueue_t();
- void QueueTail(queryQueued_t *newquery);
- void DeQueueHead();
- unsigned int GetQueueLen() const { return _queueLen; }
- unsigned int GetActiveQueries() const { return _active; }
- void SetActiveQuery() { _active++; }
- void ClearActiveQuery() { _active--; }
- queryQueued_t *GetFirst() const { return _head; }
- };
-
- //----------------------------------------------------------------
-
-protected:
- FastS_AppContext *_appCtx;
- std::mutex _lock;
- FastOS_Time _createtime;
- queryQueue_t _queryQueue;
- total_t _total;
- uint32_t _id;
- uint32_t _unitrefcost;
-
- // Total cost as seen by referencing objects
- std::atomic<uint32_t> _totalrefcost;
- uint32_t _mldDocStamp;
-private:
- uint32_t _searchableCopies;
-
-public:
- FastS_DataSetBase(const FastS_DataSetBase &) = delete;
- FastS_DataSetBase& operator=(const FastS_DataSetBase &) = delete;
- FastS_DataSetBase(FastS_AppContext *appCtx, FastS_DataSetDesc *desc);
- virtual ~FastS_DataSetBase();
-
- // locking stuff
- //--------------
- std::unique_lock<std::mutex> getDsGuard() { return std::unique_lock<std::mutex>(_lock); }
-
- // query queue related methods
- //----------------------------
- void SetActiveQuery_HasLock();
- void SetActiveQuery();
- void ClearActiveQuery_HasLock(FastS_TimeKeeper *timeKeeper);
- void ClearActiveQuery(FastS_TimeKeeper *timeKeeper);
- void CheckQueryQueue_HasLock(FastS_TimeKeeper *timeKeeper);
- void AbortQueryQueue_HasLock();
-
- // common dataset methods
- //-----------------------
- uint32_t GetID() { return _id; }
- double Uptime() { return _createtime.MilliSecsToNow() / 1000.0; }
- FastS_AppContext *GetAppContext() const { return _appCtx; }
- void AddCost();
- void SubCost();
- void UpdateSearchTime(double tnow, double elapsed, bool timedout);
- void UpdateEstimateCount();
- void CountTimeout();
- uint32_t getSearchableCopies() const { return _searchableCopies; }
-
- void ScheduleCheckTempFail();
- virtual void DeQueueHeadWakeup_HasLock();
- virtual ChildInfo getChildInfo() const;
- uint32_t GetMldDocStamp() const { return _mldDocStamp; }
- void SetMldDocStamp(uint32_t mldDocStamp) { _mldDocStamp = mldDocStamp; }
-
- // common dataset API
- //-------------------
- virtual uint32_t CalculateQueueLens_HasLock(uint32_t &dispatchnodes) = 0;
- virtual bool AddEngine(FastS_EngineDesc *desc) = 0;
- virtual void ConfigDone(FastS_DataSetCollection *) {}
- virtual void ScheduleCheckBad() {}
- virtual bool AreEnginesReady() = 0;
- virtual FastS_ISearch *CreateSearch(FastS_DataSetCollection *dsc,
- FastS_TimeKeeper *timeKeeper,
- bool async) = 0;
- virtual void Free() = 0;
-
- // typesafe down-cast
- //-------------------
- virtual FastS_PlainDataSet *GetPlainDataSet() { return nullptr; }
- virtual FastS_FNET_DataSet *GetFNETDataSet() { return nullptr; }
-};
diff --git a/searchcore/src/vespa/searchcore/fdispatch/search/datasetcollection.cpp b/searchcore/src/vespa/searchcore/fdispatch/search/datasetcollection.cpp
deleted file mode 100644
index d99b32ac138..00000000000
--- a/searchcore/src/vespa/searchcore/fdispatch/search/datasetcollection.cpp
+++ /dev/null
@@ -1,266 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-
-#include "datasetcollection.h"
-#include "fnet_dataset.h"
-#include <vespa/searchcore/fdispatch/common/search.h>
-#include <vespa/fnet/fnet.h>
-
-#include <vespa/log/log.h>
-LOG_SETUP(".search.datasetcollection");
-
-FastS_DataSetBase *
-FastS_DataSetCollection::CreateDataSet(FastS_DataSetDesc *desc)
-{
- FastS_DataSetBase *ret = nullptr;
-
- FNET_Transport *transport = _appCtx->GetFNETTransport();
- FNET_Scheduler *scheduler = _appCtx->GetFNETScheduler();
- if (transport != nullptr && scheduler != nullptr) {
- ret = new FastS_FNET_DataSet(transport, scheduler, _appCtx, desc);
- } else {
- LOG(error, "Non-available dataset transport: FNET");
- }
- return ret;
-}
-
-
-bool
-FastS_DataSetCollection::AddDataSet(FastS_DataSetDesc *desc)
-{
- uint32_t datasetid = desc->GetID();
-
- if (datasetid >= _datasets_size) {
- uint32_t newSize = datasetid + 1;
-
- FastS_DataSetBase **newArray = new FastS_DataSetBase*[newSize];
- FastS_assert(newArray != nullptr);
-
- uint32_t i;
- for (i = 0; i < _datasets_size; i++)
- newArray[i] = _datasets[i];
-
- for (; i < newSize; i++)
- newArray[i] = nullptr;
-
- delete [] _datasets;
- _datasets = newArray;
- _datasets_size = newSize;
- }
- FastS_assert(_datasets[datasetid] == nullptr);
- FastS_DataSetBase *dataset = CreateDataSet(desc);
- if (dataset == nullptr)
- return false;
- _datasets[datasetid] = dataset;
-
- for (FastS_EngineDesc *engineDesc = desc->GetEngineList();
- engineDesc != nullptr; engineDesc = engineDesc->GetNext()) {
-
- dataset->AddEngine(engineDesc);
- }
- dataset->ConfigDone(this);
- return true;
-}
-
-
-
-FastS_DataSetCollection::FastS_DataSetCollection(FastS_AppContext *appCtx)
- : _nextOld(nullptr),
- _configDesc(nullptr),
- _appCtx(appCtx),
- _datasets(nullptr),
- _datasets_size(0),
- _gencnt(0),
- _frozen(false),
- _error(false)
-{
-}
-
-
-FastS_DataSetCollection::~FastS_DataSetCollection()
-{
- if (_datasets != nullptr) {
- for (uint32_t i = 0; i < _datasets_size; i++) {
- if (_datasets[i] != nullptr) {
- _datasets[i]->Free();
- _datasets[i] = nullptr;
- }
- }
- }
-
- delete [] _datasets;
- delete _configDesc;
-}
-
-
-bool
-FastS_DataSetCollection::Configure(FastS_DataSetCollDesc *cfgDesc,
- uint32_t gencnt)
-{
- bool rc = false;
-
- if (_frozen) {
- delete cfgDesc;
- } else {
- FastS_assert(_configDesc == nullptr);
- if (cfgDesc == nullptr) {
- _configDesc = new FastS_DataSetCollDesc();
- } else {
- _configDesc = cfgDesc;
- }
- _gencnt = gencnt;
- _frozen = true;
- _error = !_configDesc->Freeze();
- rc = !_error;
-
- for (uint32_t i = 0; rc && i < _configDesc->GetMaxNumDataSets(); i++) {
- FastS_DataSetDesc *datasetDesc = _configDesc->GetDataSet(i);
- if (datasetDesc != nullptr) {
- FastS_assert(datasetDesc->GetID() == i);
- rc = AddDataSet(datasetDesc);
- }
- }
-
- _error = !rc;
- }
- return rc;
-}
-
-
-uint32_t
-FastS_DataSetCollection::SuggestDataSet()
-{
- FastS_assert(_frozen);
-
- FastS_DataSetBase *dataset = nullptr;
-
- for (uint32_t i = 0; i < _datasets_size; i++) {
- FastS_DataSetBase *tmp = _datasets[i];
- if (tmp == nullptr || tmp->_unitrefcost == 0)
- continue;
-
- // NB: cost race condition
-
- if (dataset == nullptr ||
- dataset->_totalrefcost + dataset->_unitrefcost >
- tmp->_totalrefcost + tmp->_unitrefcost)
- dataset = tmp;
- }
-
- return (dataset == nullptr)
- ? FastS_NoID32()
- : dataset->GetID();
-}
-
-
-FastS_DataSetBase *
-FastS_DataSetCollection::GetDataSet(uint32_t datasetid)
-{
- FastS_assert(_frozen);
-
- FastS_DataSetBase *dataset =
- (datasetid < _datasets_size) ?
- _datasets[datasetid] : nullptr;
-
- if (dataset != nullptr)
- dataset->AddCost();
-
- return dataset;
-}
-
-
-FastS_DataSetBase *
-FastS_DataSetCollection::GetDataSet()
-{
- FastS_assert(_frozen);
-
- FastS_DataSetBase *dataset = nullptr;
-
- for (uint32_t i = 0; i < _datasets_size; i++) {
- FastS_DataSetBase *tmp = _datasets[i];
- if (tmp == nullptr || tmp->_unitrefcost == 0)
- continue;
-
- // NB: cost race condition
-
- if (dataset == nullptr ||
- dataset->_totalrefcost + dataset->_unitrefcost >
- tmp->_totalrefcost + tmp->_unitrefcost)
- dataset = tmp;
- }
-
- if (dataset != nullptr)
- dataset->AddCost();
-
- return dataset;
-}
-
-
-bool
-FastS_DataSetCollection::AreEnginesReady()
-{
- for (uint32_t datasetidx = 0; datasetidx < GetMaxNumDataSets(); datasetidx++) {
- FastS_DataSetBase *dataset = PeekDataSet(datasetidx);
- if ((dataset != nullptr) && !dataset->AreEnginesReady()) {
- return false;
- }
- }
- return true;
-}
-
-
-FastS_ISearch *
-FastS_DataSetCollection::CreateSearch(uint32_t dataSetID,
- FastS_TimeKeeper *timeKeeper)
-{
- FastS_ISearch *ret = nullptr;
- FastS_DataSetBase *dataset;
-
- if (dataSetID == FastS_NoID32()) {
- dataset = GetDataSet();
- if (dataset != nullptr)
- dataSetID = dataset->GetID();
- } else {
- dataset = GetDataSet(dataSetID);
- }
- if (dataset == nullptr) {
- ret = new FastS_FailedSearch(dataSetID, false,
- search::engine::ECODE_ILLEGAL_DATASET, nullptr);
- } else {
- {
- auto dsGuard(dataset->getDsGuard());
- dataset->SetActiveQuery_HasLock();
- }
- /* XXX: Semantic change: precounted as active in dataset */
- ret = dataset->CreateSearch(this, timeKeeper, /* async = */ false);
- }
- FastS_assert(ret != nullptr);
- return ret;
-}
-
-
-void
-FastS_DataSetCollection::CheckQueryQueues(FastS_TimeKeeper *timeKeeper)
-{
- for (uint32_t datasetidx(0); datasetidx < GetMaxNumDataSets(); datasetidx++) {
- FastS_DataSetBase *dataset = PeekDataSet(datasetidx);
-
- if (dataset != nullptr) {
- auto dsGuard(dataset->getDsGuard());
- dataset->CheckQueryQueue_HasLock(timeKeeper);
- }
- }
-}
-
-
-void
-FastS_DataSetCollection::AbortQueryQueues()
-{
- for (uint32_t datasetidx(0); datasetidx < GetMaxNumDataSets(); datasetidx++) {
- FastS_DataSetBase *dataset = PeekDataSet(datasetidx);
-
- if (dataset != nullptr) {
- auto dsGuard(dataset->getDsGuard());
- dataset->AbortQueryQueue_HasLock();
- }
- }
-}
diff --git a/searchcore/src/vespa/searchcore/fdispatch/search/datasetcollection.h b/searchcore/src/vespa/searchcore/fdispatch/search/datasetcollection.h
deleted file mode 100644
index aed33803c02..00000000000
--- a/searchcore/src/vespa/searchcore/fdispatch/search/datasetcollection.h
+++ /dev/null
@@ -1,87 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-
-#pragma once
-
-#include <vespa/vespalib/util/referencecounter.h>
-#include <vespa/searchcore/fdispatch/common/appcontext.h>
-#include <vespa/searchcore/fdispatch/search/configdesc.h>
-
-class FastS_DataSetBase;
-class FastS_ISearch;
-
-class FastS_DataSetCollection : public vespalib::ReferenceCounter
-{
-private:
- FastS_DataSetCollection(const FastS_DataSetCollection &);
- FastS_DataSetCollection& operator=(const FastS_DataSetCollection &);
-
-public:
- // used by Monitor to service old query queues.
- FastS_DataSetCollection *_nextOld;
-
-private:
- FastS_DataSetCollDesc *_configDesc;
- FastS_AppContext *_appCtx;
-
- FastS_DataSetBase **_datasets;
- uint32_t _datasets_size;
-
- uint32_t _gencnt;
- bool _frozen;
- bool _error;
-
- FastS_DataSetBase *CreateDataSet(FastS_DataSetDesc *desc);
- bool AddDataSet(FastS_DataSetDesc *desc);
-
-public:
- explicit FastS_DataSetCollection(FastS_AppContext *appCtx);
- virtual ~FastS_DataSetCollection();
-
- /**
- * Configure this dataset collection. Note that the given config
- * description is handed over to this object when this method is
- * called. Also note that this method replaces the old methods used
- * to add datasets and engines as well as the Freeze method. In
- * other words; this method uses the given config description to
- * create a new node setup and then freezing it. Using a NULL
- * pointer for the config description is legal; it denotes the empty
- * configuration.
- *
- * @return true(ok)/false(fail)
- * @param cfgDesc configuration description
- * @param gencnt the generation of this node setup
- **/
- bool Configure(FastS_DataSetCollDesc *cfgDesc, uint32_t gencnt);
-
- /**
- * This method may be used to verify that this dataset collection
- * has been successfully configured. See @ref Configure.
- *
- * @return true if successfully configured
- **/
- bool IsValid() { return (_frozen && !_error); }
-
- FastS_DataSetCollDesc *GetConfigDesc() { return _configDesc; }
-
- FastS_AppContext *GetAppContext() { return _appCtx; }
-
- uint32_t GetMaxNumDataSets() { return _datasets_size; }
-
- FastS_DataSetBase *PeekDataSet(uint32_t datasetid)
- { return (datasetid < _datasets_size) ? _datasets[datasetid] : NULL; }
-
- uint32_t SuggestDataSet();
- FastS_DataSetBase *GetDataSet(uint32_t datasetid);
- FastS_DataSetBase *GetDataSet();
-
- bool AreEnginesReady();
-
- // create search
- FastS_ISearch *CreateSearch(uint32_t dataSetID, FastS_TimeKeeper *timeKeeper);
-
- // handle old query queues
- bool IsLastRef() { return (refCount() == 1); }
- void CheckQueryQueues(FastS_TimeKeeper *timeKeeper);
- void AbortQueryQueues();
-};
-
diff --git a/searchcore/src/vespa/searchcore/fdispatch/search/engine_base.cpp b/searchcore/src/vespa/searchcore/fdispatch/search/engine_base.cpp
deleted file mode 100644
index 24668db6024..00000000000
--- a/searchcore/src/vespa/searchcore/fdispatch/search/engine_base.cpp
+++ /dev/null
@@ -1,417 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-
-#include "engine_base.h"
-#include "configdesc.h"
-#include "plain_dataset.h"
-
-#include <vespa/log/log.h>
-LOG_SETUP(".search.engine_base");
-
-//---------------------------------------------------------------------------
-
-FastS_EngineBase::stats_t::stats_t()
- : _fliptime(),
- _floptime(),
- _slowQueryCnt(0),
- _slowDocsumCnt(0),
- _slowQuerySecs(0.0),
- _slowDocsumSecs(0.0),
- _queueLenSampleAcc(0),
- _queueLenSampleCnt(0),
- _activecntSampleAcc(0),
- _activecntSampleCnt(0),
- _queueLenAcc(0.0),
- _activecntAcc(0.0),
- _queueLenIdx(0),
- _queueLenValid(0)
-{
- uint32_t i;
-
- _fliptime.SetNow();
- _floptime.SetNow();
- for (i = 0; i < _queuestatsize; i++) {
- _queueLens[i]._queueLen = 0.0;
- _queueLens[i]._activecnt = 0.0;
- }
-}
-
-//---------------------------------------------------------------------------
-
-FastS_EngineBase::reported_t::reported_t()
- : _queueLen(0),
- _dispatchers(0),
- _mld(false),
- _reportedPartID(FastS_NoID32()),
- _actNodes(0),
- _maxNodes(0),
- _actParts(0),
- _maxParts(0),
- _activeDocs(),
- _docstamp(FastS_EngineBase::NoDocStamp())
-{
- _activeDocs.valid = true;
-}
-
-
-FastS_EngineBase::reported_t::~reported_t()
-{
-}
-
-//---------------------------------------------------------------------------
-
-FastS_EngineBase::config_t::config_t(FastS_EngineDesc *desc)
- : _name(NULL),
- _unitrefcost(desc->GetUnitRefCost()),
- _confPartID(desc->GetConfPartID()),
- _confRowID(desc->GetConfRowID()),
- _confPartIDOverrides(desc->GetConfPartIDOverrides())
-{
- _name = strdup(desc->GetName());
- FastS_assert(_name != NULL);
-}
-
-
-FastS_EngineBase::config_t::~config_t()
-{
- free(_name);
-}
-
-//---------------------------------------------------------------------------
-
-FastS_EngineBase::FastS_EngineBase(FastS_EngineDesc *desc,
- FastS_PlainDataSet *dataset)
- : _stats(),
- _reported(),
- _config(desc),
- _isUp(false),
- _badness(BAD_NOT),
- _partid(FastS_NoID32()),
- _totalrefcost(0),
- _activecnt(0),
- _dataset(dataset),
- _nextds(NULL),
- _prevpart(NULL),
- _nextpart(NULL),
- _lock()
-{
- FastS_assert(_dataset != NULL);
-}
-
-
-FastS_EngineBase::~FastS_EngineBase()
-{
- FastS_assert(_nextds == NULL);
- FastS_assert(_prevpart == NULL);
- FastS_assert(_nextpart == NULL);
- FastS_assert(_totalrefcost == 0);
- FastS_assert(_activecnt == 0);
-}
-
-
-void
-FastS_EngineBase::SlowQuery(double limit, double secs, bool silent)
-{
- {
- std::lock_guard<std::mutex> engineGuard(_lock);
- _stats._slowQueryCnt++;
- _stats._slowQuerySecs += secs;
- }
- if (!silent)
- LOG(warning,
- "engine %s query slow by %.3fs + %.3fs",
- _config._name, limit, secs);
-}
-
-
-void
-FastS_EngineBase::SlowDocsum(double limit, double secs)
-{
- {
- std::lock_guard<std::mutex> engineGuard(_lock);
- _stats._slowDocsumCnt++;
- _stats._slowDocsumSecs += secs;
- }
- LOG(warning,
- "engine %s docsum slow by %.3fs + %.3fs",
- _config._name, limit, secs);
-}
-
-
-void
-FastS_EngineBase::AddCost()
-{
- _totalrefcost += _config._unitrefcost;
- ++_activecnt;
-}
-
-
-void
-FastS_EngineBase::SubCost()
-{
- FastS_assert(_totalrefcost >= _config._unitrefcost);
- _totalrefcost -= _config._unitrefcost;
- FastS_assert(_activecnt >= 1);
- --_activecnt;
-}
-
-
-void
-FastS_EngineBase::SaveQueueLen_NoLock(uint32_t queueLen, uint32_t dispatchers)
-{
- _reported._queueLen = queueLen;
- _reported._dispatchers = dispatchers;
- _stats._queueLenSampleAcc += queueLen;
- _stats._queueLenSampleCnt++;
- _stats._activecntSampleAcc += _activecnt;
- _stats._activecntSampleCnt++;
-}
-
-
-void
-FastS_EngineBase::SampleQueueLens()
-{
- double queueLen;
- double activecnt;
-
- std::lock_guard<std::mutex> engineGuard(_lock);
- if (_stats._queueLenSampleCnt > 0)
- queueLen = (double) _stats._queueLenSampleAcc / (double) _stats._queueLenSampleCnt;
- else
- queueLen = 0;
- if (_stats._activecntSampleCnt > 0)
- activecnt = (double) _stats._activecntSampleAcc / (double) _stats._activecntSampleCnt;
- else
- activecnt = 0;
-
- _stats._queueLenSampleAcc = 0;
- _stats._queueLenSampleCnt = 0;
- _stats._activecntSampleAcc = 0;
- _stats._activecntSampleCnt = 0;
-
- _stats._queueLenAcc -= _stats._queueLens[_stats._queueLenIdx]._queueLen;
- _stats._queueLens[_stats._queueLenIdx]._queueLen = queueLen;
- _stats._queueLenAcc += queueLen;
-
- _stats._activecntAcc -= _stats._queueLens[_stats._queueLenIdx]._activecnt;
- _stats._queueLens[_stats._queueLenIdx]._activecnt = activecnt;
- _stats._activecntAcc += activecnt;
-
- _stats._queueLenIdx++;
- if (_stats._queueLenIdx >= _stats._queuestatsize)
- _stats._queueLenIdx = 0;
- if (_stats._queueLenValid < _stats._queuestatsize)
- _stats._queueLenValid++;
-}
-
-void
-FastS_EngineBase::UpdateSearchTime(double tnow, double elapsed, bool timedout)
-{
- (void) tnow;
- (void) elapsed;
- (void) timedout;
-}
-
-void
-FastS_EngineBase::MarkBad(uint32_t badness)
-{
- bool worse = false;
-
- {
- std::lock_guard<std::mutex> engineGuard(_lock);
- if (badness > _badness) {
- _badness = badness;
- worse = true;
- }
- }
-
- if (worse) {
- if (badness <= BAD_NOT) {
- } else {
- _dataset->ScheduleCheckBad();
- }
- }
-}
-
-
-void
-FastS_EngineBase::ClearBad()
-{
- {
- std::unique_lock<std::mutex> engineGuard(_lock);
- if (_badness >= BAD_CONFIG) {
- engineGuard.unlock();
- LOG(warning,
- "engine %s still bad due to illegal config",
- _config._name);
- return;
- }
- _badness = BAD_NOT;
- }
- HandleClearedBad();
-}
-
-
-void
-FastS_EngineBase::HandlePingResponse(uint32_t partid,
- time_t docstamp,
- bool mld,
- uint32_t maxnodes,
- uint32_t nodes,
- uint32_t maxparts,
- uint32_t parts,
- PossCount activeDocs)
-{
- // ignore really bad nodes
- if (IsRealBad())
- return;
-
- _reported._reportedPartID = partid;
-
- // override reported partid ?
-
- if (_config._confPartIDOverrides && _config._confPartID != FastS_NoID32()) {
- LOG(debug, "Partid(%d) overridden by config(%d)", partid, _config._confPartID);
- partid = _config._confPartID;
- }
-
- // bad partid ?
-
- if ((partid != _config._confPartID && _config._confPartID != FastS_NoID32()) ||
- (partid < _dataset->GetFirstPart()) ||
- (partid >= _dataset->GetLastPart()) ||
- (partid >= _dataset->GetFirstPart() + (1 << _dataset->GetPartBits())))
- {
- LOG(warning, "Partid(%d) overridden to %d since it was bad: _confPartID(%d) dataset.first(%d), last(%d), (1 << bits)(%d)", partid, FastS_NoID32(), _config._confPartID, _dataset->GetFirstPart(), _dataset->GetLastPart(), (1 << _dataset->GetPartBits()));
- partid = FastS_NoID32();
- }
-
- // what happened ?
-
- bool onlined = !IsUp();
- bool bigchange = (!onlined &&
- (partid != _partid ||
- docstamp != _reported._docstamp));
- bool changed = (!onlined &&
- (bigchange ||
- mld != _reported._mld ||
- maxnodes != _reported._maxNodes ||
- nodes != _reported._actNodes ||
- maxparts != _reported._maxParts ||
- activeDocs != _reported._activeDocs ||
- parts != _reported._actParts));
-
- // nothing happened ?
-
-#if 0
- LOG(info,
- "HandlePingResponse: "
- "engine %s (partid %d) docstamp %d, "
- "onlined %s, changed %s",
- _config._name,
- static_cast<int>(partid),
- static_cast<int>(docstamp),
- onlined ? "true" : "false",
- changed ? "true" : "false");
-#endif
- if (!onlined && !changed)
- return;
-
- // report stuff
-
- if (onlined) {
- LOG(debug,
- "Search node %s up, partition %d, docstamp %d",
- _config._name, partid, (uint32_t) docstamp);
- } else if (bigchange) {
- if (partid != _partid) {
- LOG(debug,
- "Search node %s changed partid %u -> %u",
- _config._name, _partid, partid);
- }
- if (docstamp != _reported._docstamp) {
- LOG(debug,
- "Search node %s changed docstamp %u -> %u",
- _config._name,
- (uint32_t)_reported._docstamp,
- (uint32_t)docstamp);
- if (docstamp == 0) {
- LOG(warning, "Search node %s (partid %d) went bad (docstamp 0)",
- _config._name, partid);
- }
- }
- }
-
- {
- auto dsGuard(_dataset->getDsGuard());
- if (changed)
- _dataset->LinkOutPart_HasLock(this);
-
- _partid = partid;
- if (docstamp != _reported._docstamp) {
- _reported._docstamp = docstamp;
- }
- _reported._mld = mld;
- _reported._maxNodes = maxnodes;
- _reported._actNodes = nodes;
- _reported._maxParts = maxparts;
- _reported._actParts = parts;
- if (_reported._activeDocs != activeDocs) {
- _dataset->updateActiveDocs_HasLock(GetConfRowID(), activeDocs, _reported._activeDocs);
- _reported._activeDocs = activeDocs;
- }
- _isUp = true;
-
- _dataset->LinkInPart_HasLock(this);
-
- }
- _dataset->ScheduleCheckTempFail();
-
- if (onlined) {
- HandleUp();
- }
-
- // detect flipflop badness
-
- // NB: fliphistory race with clearbad...
-
- if (onlined || bigchange) {
- _stats._fliptime.SetNow();
- }
-}
-
-
-void
-FastS_EngineBase::HandleLostConnection()
-{
- if (IsUp()) {
- _isUp = false;
- _stats._floptime.SetNow();
- LOG(warning, "Search node %s down", _config._name);
-
- {
- auto dsGuard(_dataset->getDsGuard());
- _dataset->LinkOutPart_HasLock(this);
- PossCount noDocs;
- noDocs.valid = true;
- _dataset->updateActiveDocs_HasLock(GetConfRowID(), noDocs, _reported._activeDocs);
- _reported._activeDocs = noDocs;
- }
- _dataset->ScheduleCheckTempFail();
- HandleDown(); // classic: NotifyVirtualConnsDown
- }
-}
-
-
-void
-FastS_EngineBase::HandleNotOnline(int seconds)
-{
- LOG(warning, "Search node %s still not up after %d seconds",
- _config._name, seconds);
-}
-
-
-void
-FastS_EngineBase::Ping()
-{
- SampleQueueLens();
-}
diff --git a/searchcore/src/vespa/searchcore/fdispatch/search/engine_base.h b/searchcore/src/vespa/searchcore/fdispatch/search/engine_base.h
deleted file mode 100644
index 7c109cb99c0..00000000000
--- a/searchcore/src/vespa/searchcore/fdispatch/search/engine_base.h
+++ /dev/null
@@ -1,195 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-
-#pragma once
-
-#include <vespa/searchcore/fdispatch/common/timestat.h>
-#include "plain_dataset.h"
-#include "poss_count.h"
-#include <atomic>
-
-class FastS_FNET_DataSet;
-class FastS_DataSetInfo;
-
-class FastS_FNET_Engine;
-class FastS_RPC_Engine;
-
-class FastS_EngineBase
-{
- friend class FastS_FNET_Engine;
- friend class FastS_RPC_Engine;
- friend class FastS_PlainDataSet;
- friend class FastS_FNET_DataSet;
- friend class FastS_PartitionMap;
- friend class FastS_DataSetInfo;
-
-private:
- FastS_EngineBase(const FastS_EngineBase &);
- FastS_EngineBase& operator=(const FastS_EngineBase &);
-
-public:
-
- //----------------------------------------------------------------
- // class holding various statistics for a search node
- //----------------------------------------------------------------
- class stats_t
- {
- public:
- enum {
- _queuestatsize = 100
- };
-
- // the node goes up and down...
- FastOS_Time _fliptime; // When state changed last to UP or big chg
- FastOS_Time _floptime; // When state changed last from UP
-
- // search/docsum slowness
- uint32_t _slowQueryCnt;
- uint32_t _slowDocsumCnt;
- double _slowQuerySecs;
- double _slowDocsumSecs;
-
- // active cnt + queue len sampling
- uint32_t _queueLenSampleAcc; // sum of reported queue lengths
- uint32_t _queueLenSampleCnt; // number of reported queue lengths
- uint32_t _activecntSampleAcc; // sum of our "load"
- uint32_t _activecntSampleCnt; // number of our "load" samples
-
- // sampled active cnt + queue len
- struct {
- double _queueLen;
- double _activecnt;
- } _queueLens[_queuestatsize];
- double _queueLenAcc;
- double _activecntAcc;
- uint32_t _queueLenIdx;
- uint32_t _queueLenValid;
-
- stats_t();
-
- };
-
- //----------------------------------------------------------------
- // class holding values reported from the node below
- //----------------------------------------------------------------
- class reported_t
- {
- private:
- reported_t(const reported_t &);
- reported_t& operator=(const reported_t &);
-
- public:
- uint32_t _queueLen; // queue len on search node
- uint32_t _dispatchers; // # dispatchers using search node
-
- bool _mld;
- uint32_t _reportedPartID; // Partid reported from node below
- uint32_t _actNodes; // From _MLD_MON. # active nodes, or 1
- uint32_t _maxNodes; // From _MLD_MON. total # nodes, or 1
- uint32_t _actParts; // From _MLD_MON. # active parts, or 1
- uint32_t _maxParts; // From _MLD_MON. total # parts, or 1
- PossCount _activeDocs;
- time_t _docstamp;
-
- reported_t();
- ~reported_t();
- };
-
- //----------------------------------------------------------------
- // class holding config values
- //----------------------------------------------------------------
- class config_t
- {
- private:
- config_t(const config_t &);
- config_t& operator=(const config_t &);
-
- public:
- char *_name;
- uint32_t _unitrefcost; // Cost to reference us
- uint32_t _confPartID; // Partid configured in partitions file
- uint32_t _confRowID; // What row this engine belongs to
- bool _confPartIDOverrides; // Ignore lower partid and use our conf value
- config_t(FastS_EngineDesc *desc);
- ~config_t();
- };
-
- // engine badness enum
- enum {
- BAD_NOT,
- BAD_ADMIN,
- BAD_CONFIG
- };
-
-protected:
- stats_t _stats;
- reported_t _reported;
- config_t _config;
-
- bool _isUp; // is this engine up ?
- uint32_t _badness; // engine badness indicator
- uint32_t _partid; // Partid we actually use
-
- // Total cost as seen by referencing objects
- std::atomic<uint32_t> _totalrefcost;
- std::atomic<uint32_t> _activecnt; // Our "load" on search node
-
- FastS_PlainDataSet *_dataset; // dataset for this engine
-
- FastS_EngineBase *_nextds; // list of engines in dataset
- FastS_EngineBase *_prevpart; // list of engines in partition
- FastS_EngineBase *_nextpart; // list of engines in partition
- std::mutex _lock;
-
-public:
- FastS_EngineBase(FastS_EngineDesc *desc, FastS_PlainDataSet *dataset);
- virtual ~FastS_EngineBase();
-
- // common engine methods
- //----------------------
- static time_t NoDocStamp() { return static_cast<time_t>(-1); }
- const char *GetName() const { return _config._name; }
- FastS_EngineBase *GetNextDS() const { return _nextds; }
- uint32_t GetQueueLen() const { return _reported._queueLen; }
- uint32_t GetDispatchers() const { return _reported._dispatchers; }
- FastS_PlainDataSet *GetDataSet() const { return _dataset; }
- uint32_t GetConfRowID() const { return _config._confRowID; }
- uint32_t GetPartID() const { return _partid; }
-
- time_t GetTimeStamp() const { return _reported._docstamp; }
- bool IsMLD() const { return _reported._mld; }
-
- bool IsUp() const { return _isUp; }
- bool IsRealBad() const { return (_badness > BAD_NOT); }
- bool isAdminBad() const { return _badness == BAD_ADMIN; }
-
- bool IsReady() const { return (IsUp() || IsRealBad()); }
- void SlowQuery(double limit, double secs, bool silent);
- void SlowDocsum(double limit, double secs);
- void AddCost();
- void SubCost();
- void SaveQueueLen_NoLock(uint32_t queueLen, uint32_t dispatchers);
- void SampleQueueLens();
- void UpdateSearchTime(double tnow, double elapsed, bool timedout);
- void NotifyFailure();
- void MarkBad(uint32_t badness);
- void ClearBad();
- void HandlePingResponse(uint32_t partid, time_t docstamp, bool mld,
- uint32_t maxnodes, uint32_t nodes,
- uint32_t maxparts, uint32_t parts,
- PossCount activeDocs);
- void HandleLostConnection();
- void HandleNotOnline(int seconds);
-
- // common engine API
- //------------------
- virtual void Ping();
- virtual void HandleClearedBad() {}
- virtual void HandleUp() {}
- virtual void HandleDown() {}
-
- // typesafe "down"-cast
- //---------------------
- virtual FastS_FNET_Engine *GetFNETEngine() { return NULL; }
- virtual FastS_RPC_Engine *GetRPCEngine() { return NULL; }
-};
-
diff --git a/searchcore/src/vespa/searchcore/fdispatch/search/fnet_dataset.cpp b/searchcore/src/vespa/searchcore/fdispatch/search/fnet_dataset.cpp
deleted file mode 100644
index 081d5e7da34..00000000000
--- a/searchcore/src/vespa/searchcore/fdispatch/search/fnet_dataset.cpp
+++ /dev/null
@@ -1,144 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-
-#include "fnet_dataset.h"
-#include "fnet_engine.h"
-#include "fnet_search.h"
-#include "datasetcollection.h"
-
-#include <vespa/log/log.h>
-LOG_SETUP(".search.fnet_dataset");
-
-//--------------------------------------------------------------------------
-
-void
-FastS_FNET_DataSet::PingTask::PerformTask()
-{
- _dataset->Ping();
- Schedule(_delay);
-}
-
-//--------------------------------------------------------------------------
-
-FastS_FNET_DataSet::FastS_FNET_DataSet(FNET_Transport *transport,
- FNET_Scheduler *scheduler,
- FastS_AppContext *appCtx,
- FastS_DataSetDesc *desc)
- : FastS_PlainDataSet(appCtx, desc),
- _transport(transport),
- _pingTask(scheduler, this, getMonitorInterval()),
- _failedRowsBitmask(0)
-{
-}
-
-
-FastS_FNET_DataSet::~FastS_FNET_DataSet() = default;
-
-bool
-FastS_FNET_DataSet::AddEngine(FastS_EngineDesc *desc)
-{
- FastS_FNET_Engine *engine = new FastS_FNET_Engine(desc, this);
-
- InsertEngine(engine);
-
- if (desc->IsBad()) {
- engine->MarkBad(FastS_EngineBase::BAD_CONFIG);
- }
- return true;
-}
-
-
-namespace {
-struct ConnectFNETEngine {
- void operator()(FastS_EngineBase* engine) {
- FastS_FNET_Engine* fnet_engine = engine->GetFNETEngine();
- FastS_assert(fnet_engine != nullptr);
- fnet_engine->ScheduleConnect(0.0);
- fnet_engine->StartWarnTimer();
- }
-};
-}
-
-void
-FastS_FNET_DataSet::ConfigDone(FastS_DataSetCollection *)
-{
- ForEachEngine( ConnectFNETEngine() );
- _pingTask.ScheduleNow();
-}
-
-
-void
-FastS_FNET_DataSet::ScheduleCheckBad()
-{
- _pingTask.ScheduleNow();
-}
-
-
-FastS_ISearch *
-FastS_FNET_DataSet::CreateSearch(FastS_DataSetCollection *dsc,
- FastS_TimeKeeper *timeKeeper,
- bool async)
-{
- return (async)
- ? (FastS_ISearch *) new FastS_FNET_Search(dsc, this, timeKeeper)
- : (FastS_ISearch *) new FastS_Sync_FNET_Search(dsc, this, timeKeeper);
-}
-
-
-void
-FastS_FNET_DataSet::Free()
-{
- _pingTask.Kill();
-
- for (FastS_EngineBase *engine = ExtractEngine();
- engine != nullptr; engine = ExtractEngine())
- {
- FastS_assert(engine->GetFNETEngine() != nullptr);
- delete engine;
- }
-
- delete this;
-}
-
-bool
-FastS_FNET_DataSet::isGoodRow(uint32_t rowId)
-{
- auto dsGuard(getDsGuard());
- uint64_t rowBit = 1ul << rowId;
- bool wasBad = ((_failedRowsBitmask & rowBit) != 0);
- bool isBad = false;
- uint64_t candDocs = _stateOfRows.getRowState(rowId).activeDocs();
- // demand: (candidate row active docs >= p% of average active docs)
- // where p = min activedocs coverage
- double p = _queryDistributionMode.getMinActivedocsCoverage() / 100.0;
- p = std::min(p, 0.999); // max demand: 99.9 %
- uint64_t restDocs = _stateOfRows.sumActiveDocs() - candDocs;
- uint64_t restRows = _stateOfRows.numRowStates() - 1;
- double restAvg = (restRows > 0) ? (restDocs / (double)restRows) : 0;
- if (_stateOfRows.activeDocsValid() && (candDocs < (p * restAvg))) {
- isBad = true;
- if (!wasBad) {
- _failedRowsBitmask |= rowBit;
- LOG(warning, "Not enough active docs in group %d (only %" PRIu64 " docs, average is %g)",
- rowId, candDocs, restAvg);
- }
- }
- size_t nodesUp = countNodesUpInRow_HasLock(rowId);
- size_t configuredParts = getNumPartitions(rowId);
- size_t nodesAllowedDown =
- getMaxNodesDownPerFixedRow() +
- (configuredParts*(100.0 - getMinGroupCoverage()))/100.0;
- if (nodesUp + nodesAllowedDown < configuredParts) {
- isBad = true;
- if (!wasBad) {
- _failedRowsBitmask |= rowBit;
- LOG(warning, "Coverage of group %d is only %ld/%ld (requires %ld)",
- rowId, nodesUp, configuredParts, configuredParts-nodesAllowedDown);
- }
- }
- if (wasBad && !isBad) {
- _failedRowsBitmask &= ~rowBit;
- LOG(info, "Group %d is now good again (%" PRIu64 "/%g active docs, coverage %ld/%ld)",
- rowId, candDocs, restAvg, nodesUp, configuredParts);
- }
- return !isBad;
-}
diff --git a/searchcore/src/vespa/searchcore/fdispatch/search/fnet_dataset.h b/searchcore/src/vespa/searchcore/fdispatch/search/fnet_dataset.h
deleted file mode 100644
index e51166a8456..00000000000
--- a/searchcore/src/vespa/searchcore/fdispatch/search/fnet_dataset.h
+++ /dev/null
@@ -1,66 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-
-#pragma once
-
-#include "plain_dataset.h"
-
-class FNET_Transport;
-
-class FastS_FNET_DataSet : public FastS_PlainDataSet
-{
-public:
-
- //----------------------------------------------------------------
- // class used to schedule periodic dataset pinging
- //----------------------------------------------------------------
-
- class PingTask : public FNET_Task
- {
- private:
- FastS_FNET_DataSet *_dataset;
- double _delay;
-
- public:
- PingTask(const PingTask &) = delete;
- PingTask& operator=(const PingTask &) = delete;
- PingTask(FNET_Scheduler *scheduler,
- FastS_FNET_DataSet *dataset,
- double delay)
- : FNET_Task(scheduler),
- _dataset(dataset),
- _delay(delay)
- {}
- void PerformTask() override;
- };
-
-
-private:
- FNET_Transport *_transport;
- PingTask _pingTask;
- uint64_t _failedRowsBitmask;
-
-public:
- FastS_FNET_DataSet(const FastS_FNET_DataSet &) = delete;
- FastS_FNET_DataSet& operator=(const FastS_FNET_DataSet &) = delete;
- FastS_FNET_DataSet(FNET_Transport *transport,
- FNET_Scheduler *scheduler,
- FastS_AppContext *appCtx,
- FastS_DataSetDesc *desc);
- ~FastS_FNET_DataSet() override;
-
- FNET_Transport *GetTransport() { return _transport; }
-
- // typesafe down-cast
- FastS_FNET_DataSet *GetFNETDataSet() override { return this; }
-
- // common dataset API
- bool AddEngine(FastS_EngineDesc *desc) override;
- void ConfigDone(FastS_DataSetCollection *) override;
- void ScheduleCheckBad() override;
- FastS_ISearch *CreateSearch(FastS_DataSetCollection *dsc,
- FastS_TimeKeeper *timeKeeper,
- bool async) override;
- void Free() override;
-
- bool isGoodRow(uint32_t rowId);
-};
diff --git a/searchcore/src/vespa/searchcore/fdispatch/search/fnet_engine.cpp b/searchcore/src/vespa/searchcore/fdispatch/search/fnet_engine.cpp
deleted file mode 100644
index afa0379c06f..00000000000
--- a/searchcore/src/vespa/searchcore/fdispatch/search/fnet_engine.cpp
+++ /dev/null
@@ -1,231 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-
-#include "fnet_dataset.h"
-#include "datasetcollection.h"
-#include "fnet_engine.h"
-#include <vespa/searchcore/fdispatch/common/search.h>
-#include <vespa/fnet/transport.h>
-#include <vespa/fnet/connection.h>
-
-using namespace search::fs4transport;
-
-//----------------------------------------------------------------------
-
-void
-FastS_StaticMonitorQuery::Free()
-{
- if (_refcnt-- == 1) {
- delete this;
- }
-}
-
-
-FastS_StaticMonitorQuery::FastS_StaticMonitorQuery()
- : FS4Packet_MONITORQUERYX(),
- _refcnt(1)
-{ }
-
-
-FastS_StaticMonitorQuery::~FastS_StaticMonitorQuery()
-{
- FastS_assert(_refcnt == 0);
-}
-
-//----------------------------------------------------------------------
-
-void
-FastS_FNET_Engine::WarnTask::PerformTask()
-{
- _engine->HandleNotOnline(DELAY);
-}
-
-//----------------------------------------------------------------------
-
-void
-FastS_FNET_Engine::ConnectTask::PerformTask()
-{
- _engine->Connect();
-}
-
-//----------------------------------------------------------------------
-
-void
-FastS_FNET_Engine::Connect()
-{
- if (_conn == nullptr ||
- _conn->GetState() >= FNET_Connection::FNET_CLOSING)
- {
- FNET_Connection *newConn =
- _transport->Connect(_spec.c_str(),
- &FS4PersistentPacketStreamer::Instance,
- this);
- FNET_Connection *oldConn;
- {
- auto dsGuard(getDsGuard());
- oldConn = _conn;
- _conn = newConn;
- }
- if (oldConn != nullptr)
- oldConn->SubRef();
- if (newConn == nullptr && !IsRealBad())
- ScheduleConnect(2.9);
- }
-}
-
-
-void
-FastS_FNET_Engine::Disconnect()
-{
- if (_conn != nullptr) {
- _conn->CloseAdminChannel();
- FNET_Connection *conn;
- {
- auto dsGuard(getDsGuard());
- conn = _conn;
- _conn = nullptr;
- }
- _transport->Close(conn, /* needref = */ false);
- }
-}
-
-
-FastS_FNET_Engine::FastS_FNET_Engine(FastS_EngineDesc *desc,
- FastS_FNET_DataSet *dataset)
- : FastS_EngineBase(desc, dataset),
- _spec(),
- _transport(dataset->GetTransport()),
- _conn(nullptr),
- _warnTask(dataset->GetAppContext()->GetFNETScheduler(), this),
- _connectTask(dataset->GetAppContext()->GetFNETScheduler(), this),
- _monitorQuery(nullptr)
-{
- if (strncmp(_config._name, "tcp/", 4) == 0) {
- _spec = _config._name;
- } else {
- _spec = "tcp/";
- _spec += _config._name;
- }
-}
-
-
-FastS_FNET_Engine::~FastS_FNET_Engine()
-{
- _warnTask.Kill();
- _connectTask.Kill();
- Disconnect();
- if (IsUp()) {
- auto dsGuard(getDsGuard());
- _dataset->LinkOutPart_HasLock(this);
- }
- if (_monitorQuery != nullptr) {
- _monitorQuery->Free();
- _monitorQuery = nullptr;
- }
-}
-
-
-void
-FastS_FNET_Engine::StartWarnTimer()
-{
- _warnTask.Schedule(_warnTask.DELAY);
-}
-
-
-void
-FastS_FNET_Engine::ScheduleConnect(double delay)
-{
- if (delay == 0.0) {
- _connectTask.ScheduleNow();
- } else {
- _connectTask.Schedule(delay);
- }
-}
-
-
-FNET_Channel *
-FastS_FNET_Engine::OpenChannel_HasDSLock(FNET_IPacketHandler *handler)
-{
- return (_conn != nullptr) ? _conn->OpenChannel(handler, FNET_Context()) : nullptr;
-}
-
-
-FNET_IPacketHandler::HP_RetCode
-FastS_FNET_Engine::HandlePacket(FNET_Packet *packet, FNET_Context)
-{
- HP_RetCode ret = FNET_KEEP_CHANNEL;
- uint32_t pcode = packet->GetPCODE();
-
- if (packet->IsChannelLostCMD()) {
-
- HandleLostConnection();
- ret = FNET_FREE_CHANNEL;
- if (!IsRealBad()) {
- ScheduleConnect(2.9);
- }
-
- } else if (pcode == search::fs4transport::PCODE_MONITORRESULTX) {
-
- FS4Packet_MONITORRESULTX *mr = (FS4Packet_MONITORRESULTX *) packet;
-
- PossCount activeDocs;
- activeDocs.valid = ((mr->_features & search::fs4transport::MRF_ACTIVEDOCS) != 0);
- activeDocs.count = mr->_activeDocs;
- if ((mr->_features & search::fs4transport::MRF_MLD) != 0) {
- HandlePingResponse(mr->_partid, mr->_timestamp, true,
- mr->_totalNodes, mr->_activeNodes,
- mr->_totalParts, mr->_activeParts,
- activeDocs);
- } else {
- HandlePingResponse(mr->_partid, mr->_timestamp, false, 1, 1, 1, 1, activeDocs);
- }
- }
-
- packet->Free();
- return ret;
-}
-
-
-void
-FastS_FNET_Engine::Ping()
-{
- FastS_EngineBase::Ping();
-
- // handle badness
- if (IsRealBad()) {
- if (_conn != nullptr) {
- Disconnect();
- HandleLostConnection();
- }
- return;
- }
-
- // handle ping
- if ((_conn != nullptr) && (_conn->GetState() < FNET_Connection::FNET_CLOSING)) {
- if (_monitorQuery == nullptr) {
- _monitorQuery = new FastS_StaticMonitorQuery();
- }
- if (_monitorQuery->getBusy()) {
- return;
- }
- _monitorQuery->markBusy();
- uint32_t features = search::fs4transport::MQF_QFLAGS;
- uint32_t qflags = search::fs4transport::MQFLAG_REPORT_ACTIVEDOCS;
- _monitorQuery->_features |= features;
- _monitorQuery->_qflags = qflags;
- _conn->PostPacket(_monitorQuery, FastS_NoID32());
- }
-}
-
-
-void
-FastS_FNET_Engine::HandleClearedBad()
-{
- ScheduleConnect(0.0);
-}
-
-
-void
-FastS_FNET_Engine::HandleUp()
-{
- _warnTask.Unschedule();
-}
diff --git a/searchcore/src/vespa/searchcore/fdispatch/search/fnet_engine.h b/searchcore/src/vespa/searchcore/fdispatch/search/fnet_engine.h
deleted file mode 100644
index 4374deb2642..00000000000
--- a/searchcore/src/vespa/searchcore/fdispatch/search/fnet_engine.h
+++ /dev/null
@@ -1,112 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-
-
-#pragma once
-
-#include "engine_base.h"
-#include <vespa/searchlib/common/packets.h>
-#include <vespa/fnet/ipackethandler.h>
-#include <atomic>
-
-//----------------------------------------------------------------------
-
-using search::fs4transport::FS4Packet_MONITORQUERYX;
-
-class FastS_StaticMonitorQuery : public FS4Packet_MONITORQUERYX
-{
- std::atomic<int> _refcnt;
-public:
- virtual void Free() override;
- bool getBusy() const { return _refcnt > 1; }
- void markBusy() { _refcnt++; }
- FastS_StaticMonitorQuery();
- ~FastS_StaticMonitorQuery();
-};
-
-//----------------------------------------------------------------------
-
-class FastS_FNET_Engine : public FNET_IPacketHandler,
- public FastS_EngineBase
-{
-private:
- FastS_FNET_Engine(const FastS_FNET_Engine &);
- FastS_FNET_Engine& operator=(const FastS_FNET_Engine &);
-
-public:
- class WarnTask : public FNET_Task
- {
- private:
- WarnTask(const WarnTask &);
- WarnTask& operator=(const WarnTask &);
-
- FastS_FNET_Engine *_engine;
-
- public:
- enum { DELAY = 30 };
- WarnTask(FNET_Scheduler *scheduler,
- FastS_FNET_Engine *engine)
- : FNET_Task(scheduler), _engine(engine) {}
- virtual void PerformTask() override;
- };
- friend class FastS_FNET_Engine::WarnTask;
-
- class ConnectTask : public FNET_Task
- {
- private:
- ConnectTask(const ConnectTask &);
- ConnectTask& operator=(const ConnectTask &);
-
- FastS_FNET_Engine *_engine;
-
- public:
- ConnectTask(FNET_Scheduler *scheduler,
- FastS_FNET_Engine *engine)
- : FNET_Task(scheduler), _engine(engine) {}
- virtual void PerformTask() override;
- };
- friend class FastS_FNET_Engine::ConnectTask;
-
-private:
- std::string _hostName;
- int _portNumber;
- std::string _spec;
- FNET_Transport *_transport;
- FNET_Connection *_conn;
- WarnTask _warnTask;
- ConnectTask _connectTask;
- FastS_StaticMonitorQuery *_monitorQuery;
-
- void Connect();
- void Disconnect();
-
-public:
- FastS_FNET_Engine(FastS_EngineDesc *desc,
- FastS_FNET_DataSet *dataset);
- virtual ~FastS_FNET_Engine();
-
- std::unique_lock<std::mutex> getDsGuard() { return _dataset->getDsGuard(); }
-
- void StartWarnTimer();
- void ScheduleConnect(double delay);
- FNET_Channel *OpenChannel_HasDSLock(FNET_IPacketHandler *handler);
-
- // handle FNET admin packets
- //--------------------------
- virtual HP_RetCode HandlePacket(FNET_Packet *packet, FNET_Context) override;
-
- // common engine API
- //------------------
- virtual void Ping() override;
- virtual void HandleClearedBad() override;
- virtual void HandleUp() override;
-
- // typesafe "down"-cast
- //---------------------
- virtual FastS_FNET_Engine *GetFNETEngine() override { return this; }
-
- const char *getHostName() const { return _hostName.c_str(); }
- int getPortNumber() const { return _portNumber; }
-};
-
-//----------------------------------------------------------------------
-
diff --git a/searchcore/src/vespa/searchcore/fdispatch/search/fnet_search.cpp b/searchcore/src/vespa/searchcore/fdispatch/search/fnet_search.cpp
deleted file mode 100644
index f665e25f819..00000000000
--- a/searchcore/src/vespa/searchcore/fdispatch/search/fnet_search.cpp
+++ /dev/null
@@ -1,1572 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-
-#include "datasetcollection.h"
-#include "fnet_dataset.h"
-#include "fnet_engine.h"
-#include "fnet_search.h"
-#include "mergehits.h"
-#include <vespa/searchlib/engine/packetconverter.h>
-#include <vespa/searchlib/engine/searchreply.h>
-#include <vespa/vespalib/util/stringfmt.h>
-#include <xxhash.h>
-
-#include <vespa/log/log.h>
-LOG_SETUP(".fnet_search");
-
-#define IS_MLD_PART(part) ((part) > mldpartidmask)
-#define MLD_PART_TO_PARTID(part) ((part) & mldpartidmask)
-#define ENCODE_MLD_PART(part) (((part) + 1) << partbits)
-#define DECODE_MLD_PART(part) (((part) >> partbits) - 1)
-
-using fdispatch::SearchPath;
-using vespalib::nbostream;
-using vespalib::stringref;
-using namespace search::fs4transport;
-using search::engine::PacketConverter;
-
-//---------------------------------------------------------------------
-//
-
-FastS_FNET_SearchNode::FastS_FNET_SearchNode(FastS_FNET_Search *search, uint32_t partid)
- : _search(search),
- _engine(nullptr),
- _channel(nullptr),
- _partid(partid),
- _rowid(0),
- _stamp(0),
- _qresult(nullptr),
- _queryTime(0.0),
- _flags(),
- _docidCnt(0),
- _pendingDocsums(0),
- _docsumRow(0),
- _docsum_offsets_idx(0),
- _docsumTime(0.0),
- _gdx(nullptr),
- _docsum_offsets(),
- _extraDocsumNodes(),
- _nextExtraDocsumNode(this),
- _prevExtraDocsumNode(this),
- _hit_beg(nullptr),
- _hit_cur(nullptr),
- _hit_end(nullptr),
- _sortDataIterator()
-{
-}
-
-
-FastS_FNET_SearchNode::~FastS_FNET_SearchNode()
-{
- Disconnect();
- if (_qresult != nullptr) {
- _qresult->Free();
- }
- if (_gdx != nullptr) {
- _gdx->Free();
- }
-}
-
-FastS_FNET_SearchNode::FastS_FNET_SearchNode(FastS_FNET_SearchNode &&)
-{
- // These objects are referenced everywhere and must never be either copied nor moved,
- // but as std::vector requires this to exist so we do this little trick.
- LOG_ABORT("should not reach here");
-}
-
-bool
-FastS_FNET_SearchNode::NT_InitMerge(uint32_t *numDocs,
- uint64_t *totalHits,
- search::HitRank *maxRank,
- uint32_t *sortDataDocs)
-{
- uint32_t myNumDocs = 0;
- if (_qresult != nullptr) {
- myNumDocs = _qresult->_numDocs;
- *numDocs += myNumDocs;
- *totalHits += _qresult->_totNumDocs;
- search::HitRank mr = _qresult->_maxRank;
- if (mr > *maxRank)
- *maxRank = mr;
- }
- if (myNumDocs > 0) {
- _hit_beg = _qresult->_hits;
- _hit_cur = _hit_beg;
- _hit_end = _hit_beg + myNumDocs;
- if ((_qresult->_features & search::fs4transport::QRF_SORTDATA) != 0) {
- _sortDataIterator.Init(myNumDocs, _qresult->_sortIndex, _qresult->_sortData);
- *sortDataDocs += myNumDocs;
- }
- return true;
- }
- return false;
-}
-
-
-FastS_EngineBase *
-FastS_FNET_SearchNode::getPartition(const std::unique_lock<std::mutex> &dsGuard, bool userow, FastS_FNET_DataSet *dataset)
-{
- return ((userow)
- ? dataset->getPartitionMLD(dsGuard, getPartID(), _flags._docsumMld, _docsumRow)
- : dataset->getPartitionMLD(dsGuard, getPartID(), _flags._docsumMld));
-}
-
-
-void
-FastS_FNET_SearchNode::
-allocGDX(search::docsummary::GetDocsumArgs *args, const search::engine::PropertiesMap &props)
-{
- FS4Packet_GETDOCSUMSX *gdx = new FS4Packet_GETDOCSUMSX();
-
- gdx->AllocateDocIDs(_docidCnt);
- _gdx = gdx;
- _docsum_offsets.resize(_gdx->_docid.size());
- _docsum_offsets_idx = 0;
- if (args == nullptr)
- return;
-
- if (args->getRankProfile().size() != 0 || args->GetQueryFlags() != 0) {
- gdx->_features |= search::fs4transport::GDF_RANKP_QFLAGS;
- gdx->setRanking(args->getRankProfile());
- gdx->_qflags = args->GetQueryFlags();
- }
- gdx->setTimeout(args->getTimeout());
-
- if (args->getResultClassName().size() > 0) {
- gdx->_features |= search::fs4transport::GDF_RESCLASSNAME;
- gdx->setResultClassName(args->getResultClassName());
- }
-
- if (props.size() > 0) {
- PacketConverter::fillPacketProperties(props, gdx->_propsVector);
- gdx->_features |= search::fs4transport::GDF_PROPERTIES;
- }
-
- if (args->getStackDump().size() > 0) {
- gdx->_features |= search::fs4transport::GDF_QUERYSTACK;
- gdx->_stackItems = args->GetStackItems();
- gdx->setStackDump(args->getStackDump());
- }
-
- if (args->GetLocationLen() > 0) {
- gdx->_features |= search::fs4transport::GDF_LOCATION;
- gdx->setLocation(args->getLocation());
- }
-
- if (args->getFlags() != 0) {
- gdx->_features |= search::fs4transport::GDF_FLAGS;
- gdx->_flags = args->getFlags();
- }
-}
-
-
-void
-FastS_FNET_SearchNode::postGDX(uint32_t *pendingDocsums, uint32_t *docsumNodes)
-{
- FS4Packet_GETDOCSUMSX *gdx = _gdx;
- FastS_assert(gdx->_docid.size() == _docsum_offsets_idx);
- if (_flags._docsumMld) {
- gdx->_features |= search::fs4transport::GDF_MLD;
- }
- if (PostPacket(gdx)) {
- _pendingDocsums = _docsum_offsets_idx;
- *pendingDocsums += _pendingDocsums;
- (*docsumNodes)++;
- }
- _gdx = nullptr; // packet hand-over
- _docsum_offsets_idx = 0;
-}
-
-
-FNET_IPacketHandler::HP_RetCode
-FastS_FNET_SearchNode::HandlePacket(FNET_Packet *packet, FNET_Context context)
-{
- uint32_t pcode = packet->GetPCODE();
- if (LOG_WOULD_LOG(spam)) {
- LOG(spam, "handling packet %p\npacket=%s", packet, packet->Print().c_str());
- context.Print();
- }
- if (packet->IsChannelLostCMD()) {
- _search->LostSearchNode(this);
- } else if (pcode == search::fs4transport::PCODE_QUERYRESULTX) {
- _search->GotQueryResult(this, (FS4Packet_QUERYRESULTX *) packet);
- } else if (pcode == search::fs4transport::PCODE_DOCSUM) {
- _search->GotDocsum(this, (FS4Packet_DOCSUM *) packet);
- } else if (pcode == search::fs4transport::PCODE_ERROR) {
- _search->GotError(this, static_cast<FS4Packet_ERROR *>(packet));
- } else {
- if (pcode == search::fs4transport::PCODE_EOL) {
- _search->GotEOL(this);
- }
- packet->Free();
- }
- return FNET_KEEP_CHANNEL;
-}
-
-
-FastS_FNET_SearchNode *
-FastS_FNET_SearchNode::
-allocExtraDocsumNode(bool mld, uint32_t rowid, uint32_t rowbits)
-{
- if (_extraDocsumNodes.empty()) {
- size_t sz = (1 << (rowbits + 1));
- _extraDocsumNodes.resize(sz);
- }
-
- uint32_t idx = (rowid << 1) + (mld ? 1 : 0);
-
- if (_extraDocsumNodes[idx].get() == nullptr) {
- UP eNode(new FastS_FNET_SearchNode(_search, getPartID()));
- eNode->_docsumRow = rowid;
- eNode->_flags._docsumMld = mld;
-
- eNode->_nextExtraDocsumNode = this;
- eNode->_prevExtraDocsumNode = _prevExtraDocsumNode;
- _prevExtraDocsumNode->_nextExtraDocsumNode = eNode.get();
- _prevExtraDocsumNode = eNode.get();
- _extraDocsumNodes[idx] = std::move(eNode);
- }
- return _extraDocsumNodes[idx].get();
-}
-
-
-//---------------------------------------------------------------------
-
-void
-FastS_FNET_Search::Timeout::PerformTask()
-{
- _search->HandleTimeout();
-}
-
-//---------------------------------------------------------------------
-
-void
-FastS_FNET_Search::reallocNodes(size_t numParts)
-{
- _nodes.clear();
-
- _nodes.reserve(numParts);
-
- for (uint32_t i = 0; i < numParts; i++) {
- _nodes.emplace_back(this, i);
- }
-}
-
-namespace {
-volatile std::atomic<uint64_t> _G_prevFixedRow(0);
-} //anonymous namespace
-
-uint32_t
-FastS_FNET_Search::getFixedRowCandidate()
-{
- uint32_t rowId(_dataset->useRoundRobinForFixedRow()
- ? (_G_prevFixedRow++)
- : _dataset->getRandomWeightedRow());
- return rowId % _dataset->getNumRows();
-}
-
-uint32_t
-FastS_FNET_Search::getNextFixedRow()
-{
- size_t numTries(0);
- uint32_t fixedRow(0);
- size_t maxTries(_dataset->getNumRows());
- if ( ! _dataset->useRoundRobinForFixedRow()) {
- maxTries *= 10;
- }
- for(;numTries < maxTries; numTries++) {
- fixedRow = getFixedRowCandidate();
- if (_dataset->isGoodRow(fixedRow)) {
- break;
- }
- }
- if (numTries == maxTries) {
- fixedRow = getFixedRowCandidate(); // Will roundrobin/random if all rows are incomplete.
- }
- LOG(debug, "FixedRow: selected=%d, numRows=%d, numTries=%ld, _G_prevFixedRow=%" PRIu64, fixedRow, _dataset->getNumRows(), numTries, _G_prevFixedRow.load());
- return fixedRow;
-}
-
-void
-FastS_FNET_Search::connectNodes(const EngineNodeMap & engines)
-{
- for (const auto & pair : engines) {
- if ( ! pair.second->IsConnected() ) {
- // Here we are connecting without having the DataSet lock.
- // This might give a race when nodes go up or down, or there is a config change.
- // However none has ever been detected for as long as the race has existed.
- // The correct fix would be to make the DataSet be constant and be replaced upon changes.
- // And using shared_ptr to them. That would avoid the big global lock all together.
- pair.second->Connect_HasDSLock(pair.first->GetFNETEngine());
- } else {
- pair.first->SubCost();
- }
- }
- _nodesConnected = true;
-}
-
-uint32_t
-FastS_FNET_Search::getHashedRow() const {
- uint32_t hash = XXH32(&_queryArgs->sessionId[0], _queryArgs->sessionId.size(), 0);
- std::vector<uint32_t> rowIds;
- rowIds.reserve(_dataset->getNumRows());
- for (uint32_t rowId(0); rowId < _dataset->getNumRows(); rowId++) {
- rowIds.push_back(rowId);
- }
- while (!rowIds.empty()) {
- uint32_t index = hash % rowIds.size();
- uint32_t fixedRow = rowIds[index];
- if (_dataset->isGoodRow(fixedRow)) {
- return fixedRow;
- }
- rowIds.erase(rowIds.begin() + index);
- }
- return 0;
-}
-void
-FastS_FNET_Search::ConnectQueryNodes()
-{
- FastS_assert( ! _nodes.empty() );
- FastS_assert(!_nodesConnected);
-
- uint32_t fixedRow(0);
- if (_dataset->useFixedRowDistribution()) {
- fixedRow = (_queryArgs->sessionId.empty()) ? getNextFixedRow() : getHashedRow();
- _fixedRow = fixedRow;
- size_t numParts = _dataset->getNumPartitions(fixedRow);
- if (_nodes.size() > numParts) {
- reallocNodes(numParts);
- }
- }
- EngineNodeMap engines;
- engines.reserve(_nodes.size());
- {
- auto dsGuard(_dataset->getDsGuard());
- for (uint32_t i = 0; i < _nodes.size(); i++) {
- FastS_EngineBase *engine = nullptr;
- if (_dataset->useFixedRowDistribution()) {
- engine = _dataset->getPartition(dsGuard, i, fixedRow);
- LOG(debug, "FixedRow: getPartition(part=%u, row=%u) -> engine(%s)", i, fixedRow, (engine != nullptr ? engine->GetName() : "null"));
- } else {
- engine = _dataset->getPartition(dsGuard, i);
- }
- if (engine != nullptr) {
- LOG(debug, "Wanted part=%d, engine={name=%s, row=%d, partid=%d}", i, engine->GetName(), engine->GetConfRowID(), engine->GetPartID());
- if (engine != nullptr) {
- engines.emplace_back(engine, getNode(i));
- }
- } else {
- LOG(debug, "No engine for part %d", i);
- }
- }
- }
- connectNodes(engines);
-}
-
-
-void
-FastS_FNET_Search::ConnectEstimateNodes()
-{
- FastS_assert( ! _nodes.empty() );
- FastS_assert(!_nodesConnected);
-
- uint32_t partid = _util.GetQuery().StackDumpHashKey() % _estPartCutoff;
- uint32_t trycnt = 0;
- uint32_t partcnt = 0;
-
- EngineNodeMap engines;
- {
- auto dsGuard(_dataset->getDsGuard());
- while (partcnt < _dataset->GetEstimateParts() && trycnt < _estPartCutoff) {
- FastS_EngineBase *engine = _dataset->getPartition(dsGuard, partid);
- if (engine != nullptr) {
- engines.emplace_back(engine, getNode(partid));
- partcnt++;
- }
- trycnt++;
- partid = (partid + 1) % _estPartCutoff;
- }
- _estParts = partcnt;
- }
- connectNodes(engines);
-}
-
-
-void FastS_FNET_SearchNode::Connect(FastS_FNET_Engine *engine)
-{
- FastS_assert(_engine == nullptr);
- FastS_assert(_channel == nullptr);
-
- _engine = engine;
- _flags._needSubCost = true;
- auto dsGuard(_engine->getDsGuard());
- _channel = _engine->OpenChannel_HasDSLock(this);
- _rowid = _engine->GetConfRowID();
- _stamp = _engine->GetTimeStamp();
-}
-
-void FastS_FNET_SearchNode::Connect_HasDSLock(FastS_FNET_Engine *engine)
-{
- _engine = engine;
- _flags._needSubCost = true;
- _channel = _engine->OpenChannel_HasDSLock(this);
- _rowid = _engine->GetConfRowID();
- _stamp = _engine->GetTimeStamp();
-}
-
-
-void FastS_FNET_Search::connectSearchPath(const vespalib::string &spec)
-{
- FastS_assert( ! _nodes.empty());
- FastS_assert(!_nodesConnected);
-
- SearchPath searchPath(spec, _nodes.size());
- uint32_t dispatchLevel = _dsc->GetAppContext()->getDispatchLevel();
- LOG(debug, "Looking up searchpath element for dispatch level %u in searchpath '%s' (size=%zu)",
- dispatchLevel, spec.c_str(), searchPath.elements().size());
- if (dispatchLevel < searchPath.elements().size()) {
- connectSearchPath(searchPath.elements()[dispatchLevel], spec, dispatchLevel);
- } else {
- LOG(warning, "Did not find searchpath element for dispatch level "
- "%u in searchpath '%s' (size=%zu). No search nodes will be queried.",
- dispatchLevel, spec.c_str(), searchPath.elements().size());
- }
-}
-
-void FastS_FNET_Search::connectSearchPath(const SearchPath::Element &elem,
- const vespalib::string &spec,
- uint32_t dispatchLevel)
-{
- EngineNodeMap engines;
- {
- auto dsGuard(_dataset->getDsGuard());
- if (!elem.hasRow()) {
- for (size_t partId : elem.nodes()) {
- if (partId < _nodes.size()) {
- FastS_EngineBase *engine = _dataset->getPartition(dsGuard, partId);
- LOG(debug, "searchpath='%s', partId=%ld, dispatchLevel=%u", spec.c_str(), partId, dispatchLevel);
- if (engine != nullptr) {
- engines.emplace_back(engine, getNode(partId));
- }
- }
- }
- } else {
- for (size_t partId : elem.nodes()) {
- if (partId < _nodes.size()) {
- FastS_EngineBase *engine = _dataset->getPartition(dsGuard, partId, elem.row());
- LOG(debug, "searchpath='%s', partId=%ld, row=%ld, dispatchLevel=%u", spec.c_str(), partId, elem.row(), dispatchLevel);
- if (engine != nullptr) {
- engines.emplace_back(engine, getNode(partId));
- }
- }
- }
- }
- }
- connectNodes(engines);
-}
-
-void
-FastS_FNET_Search::ConnectDocsumNodes(bool ignoreRow)
-{
- FastS_assert( ! _nodes.empty());
- if (_nodesConnected)
- return;
-
- bool userow = (_dataset->GetRowBits() > 0) && !ignoreRow;
-
- EngineNodeMap engines;
- {
- auto dsGuard(_dataset->getDsGuard());
- for (auto & node : _nodes) {
- if (node._gdx != nullptr) {
- FastS_EngineBase *engine = node.getPartition(dsGuard, userow, _dataset);
- if (engine != nullptr) {
- engines.emplace_back(engine, &node);
- }
- }
- for (FastS_FNET_SearchNode::ExtraDocsumNodesIter iter(&node); iter.valid(); ++iter) {
- FastS_FNET_SearchNode *eNode = *iter;
- if (eNode->_gdx != nullptr) {
- FastS_EngineBase *engine = eNode->getPartition(dsGuard, userow, _dataset);
- if (engine != nullptr) {
- engines.emplace_back(engine, eNode);
- }
- }
- }
- }
- }
- connectNodes(engines);
-}
-
-void
-FastS_FNET_Search::EncodePartIDs(uint32_t partid, uint32_t rowid, bool mld,
- FS4Packet_QUERYRESULTX::FS4_hit *pt,
- FS4Packet_QUERYRESULTX::FS4_hit *end)
-{
- uint32_t rowbits = _dataset->GetRowBits();
- uint32_t partbits = _dataset->GetPartBits();
-
- if (rowbits > 0) {
- if (mld) {
- for (; pt < end; pt++) {
- pt->_partid = ((ENCODE_MLD_PART(pt->_partid) + partid) << rowbits) + rowid;
- }
- } else {
- for (; pt < end; pt++) {
- pt->_partid = (partid << rowbits) + rowid;
- }
- }
-
- } else { // rowbits == 0
-
- if (mld) {
- for (; pt < end; pt++) {
- pt->_partid = ENCODE_MLD_PART(pt->_partid) + partid;
- }
- } else {
- for (; pt < end; pt++) {
- pt->_partid = partid;
- }
- }
- }
-}
-
-
-FastS_FNET_Search::FastS_FNET_Search(FastS_DataSetCollection *dsc,
- FastS_FNET_DataSet *dataset,
- FastS_TimeKeeper *timeKeeper)
- : FastS_AsyncSearch(dataset->GetID()),
- _lock(),
- _timeKeeper(timeKeeper),
- _startTime(timeKeeper->GetTime()),
- _timeout(dataset->GetAppContext()->GetFNETScheduler(), this),
- _util(),
- _dsc(dsc),
- _dataset(dataset),
- _datasetActiveCostRef(true),
- _nodes(),
- _nodesConnected(false),
- _estParts(0),
- _estPartCutoff(dataset->GetEstimatePartCutoff()),
- _FNET_mode(FNET_NONE),
- _pendingQueries(0),
- _pendingDocsums(0),
- _pendingDocsumNodes(0),
- _requestedDocsums(0),
- _queryNodes(0),
- _queryNodesTimedOut(0),
- _docsumNodes(0),
- _docsumNodesTimedOut(0),
- _docsumsTimedOut(0),
- _queryTimeout(false),
- _docsumTimeout(false),
- _queryStartTime(0.0),
- _queryMinWait(0.0),
- _queryMaxWait(0.0),
- _queryWaitCalculated(false),
- _adjustedQueryTimeOut(0.0),
- _docSumStartTime(0.0),
- _adjustedDocSumTimeOut(0.0),
- _fixedRow(0),
- _resbuf()
-{
- _util.GetQuery().SetDataSet(dataset->GetID());
- _util.SetStartTime(GetTimeKeeper()->GetTime());
- reallocNodes(_dataset->GetPartitions());
-}
-
-
-FastS_FNET_Search::~FastS_FNET_Search()
-{
- _timeout.Kill();
- _nodes.clear();
- _util.DropResult();
- dropDatasetActiveCostRef();
-}
-
-
-void
-FastS_FNET_Search::dropDatasetActiveCostRef()
-{
- if (_datasetActiveCostRef) {
- _dataset->SubCost();
- _dataset->ClearActiveQuery(GetTimeKeeper());
- _datasetActiveCostRef = false;
- }
-}
-
-
-void
-FastS_FNET_Search::GotQueryResult(FastS_FNET_SearchNode *node,
- FS4Packet_QUERYRESULTX *qrx)
-{
- auto searchGuard(BeginFNETWork());
- if (!searchGuard) {
- qrx->Free();
- return;
- }
-
- if (_FNET_mode == FNET_QUERY &&
- node->_flags._pendingQuery) {
- FastS_assert(node->_qresult == nullptr);
- node->_qresult = qrx;
- EncodePartIDs(node->getPartID(), node->GetRowID(),
- (qrx->_features & search::fs4transport::QRF_MLD) != 0,
- qrx->_hits, qrx->_hits + qrx->_numDocs);
- LOG(spam, "Got result from row(%d), part(%d) = hits(%d), numDocs(%" PRIu64 ")", node->GetRowID(), node->getPartID(), qrx->_numDocs, qrx->_totNumDocs);
- node->_flags._pendingQuery = false;
- _pendingQueries--;
- double tnow = GetTimeKeeper()->GetTime();
- node->_queryTime = tnow - _startTime;
- node->GetEngine()->UpdateSearchTime(tnow, node->_queryTime, false);
- adjustQueryTimeout();
- node->dropCost();
- } else {
- qrx->Free();
- }
- EndFNETWork(std::move(searchGuard));
-}
-
-void
-FastS_FNET_Search::GotDocsum(FastS_FNET_SearchNode *node,
- FS4Packet_DOCSUM *docsum)
-{
- auto searchGuard(BeginFNETWork());
- if (!searchGuard) {
- docsum->Free();
- return;
- }
-
- if (_FNET_mode == FNET_DOCSUMS &&
- node->_pendingDocsums > 0) {
- LOG(spam, "Got docsum from row(%d), part(%d) = docsumidx(%d)", node->GetRowID(), node->getPartID(), node->_docsum_offsets_idx);
- uint32_t offset = node->_docsum_offsets[node->_docsum_offsets_idx++];
- docsum->swapBuf(_resbuf[offset]._buf);
- node->_pendingDocsums--;
- _pendingDocsums--;
- if (node->_pendingDocsums == 0) {
- node->_docsumTime = (GetTimeKeeper()->GetTime() - _startTime - node->_queryTime);
- _pendingDocsumNodes--;
- }
- adjustDocsumTimeout();
- }
- docsum->Free();
- EndFNETWork(std::move(searchGuard));
-}
-
-void
-FastS_FNET_Search::LostSearchNode(FastS_FNET_SearchNode *node)
-{
- auto searchGuard(BeginFNETWork());
- if (!searchGuard) {
- return;
- }
-
- if (_FNET_mode == FNET_QUERY && node->_flags._pendingQuery) {
- FastS_assert(_pendingQueries > 0);
- _pendingQueries--;
- node->_flags._pendingQuery = false;
- adjustQueryTimeout();
- node->dropCost();
- } else if (_FNET_mode == FNET_DOCSUMS && node->_pendingDocsums > 0) {
- uint32_t nodePendingDocsums = node->_pendingDocsums;
- FastS_assert(_pendingDocsums >= nodePendingDocsums);
- _pendingDocsums -= nodePendingDocsums;
- node->_pendingDocsums = 0;
- _pendingDocsumNodes--;
- adjustDocsumTimeout();
- }
- EndFNETWork(std::move(searchGuard));
-}
-
-
-void
-FastS_FNET_Search::GotEOL(FastS_FNET_SearchNode *node)
-{
- auto searchGuard(BeginFNETWork());
- if (!searchGuard) {
- return;
- }
-
- LOG(spam, "Got EOL from row(%d), part(%d) = pendingQ(%d) pendingDocsum(%d)", node->GetRowID(), node->getPartID(), node->_flags._pendingQuery, node->_pendingDocsums);
- if (_FNET_mode == FNET_QUERY && node->_flags._pendingQuery) {
- FastS_assert(_pendingQueries > 0);
- _pendingQueries--;
- node->_flags._pendingQuery = false;
- adjustQueryTimeout();
- node->dropCost();
- } else if (_FNET_mode == FNET_DOCSUMS && node->_pendingDocsums > 0) {
- uint32_t nodePendingDocsums = node->_pendingDocsums;
- FastS_assert(_pendingDocsums >= nodePendingDocsums);
- _pendingDocsums -= nodePendingDocsums;
- node->_pendingDocsums = 0;
- _pendingDocsumNodes--;
- adjustDocsumTimeout();
- }
- EndFNETWork(std::move(searchGuard));
-}
-
-
-void
-FastS_FNET_Search::GotError(FastS_FNET_SearchNode *node,
- FS4Packet_ERROR *error)
-{
- auto searchGuard(BeginFNETWork());
- if (!searchGuard) {
- error->Free();
- return;
- }
-
- LOG(spam,
- "Got Error from row(%d), part(%d) = pendingQ(%d) pendingDocsum(%d)",
- node->GetRowID(),
- node->getPartID(),
- node->_flags._pendingQuery,
- node->_pendingDocsums);
-
- if (_FNET_mode == FNET_QUERY && node->_flags._pendingQuery) {
- FastS_assert(_pendingQueries > 0);
- _pendingQueries--;
- node->_flags._pendingQuery = false;
- if (error->_errorCode == search::engine::ECODE_TIMEOUT) {
- node->_flags._queryTimeout = true;
- _queryNodesTimedOut++;
- }
- adjustQueryTimeout();
- } else if (_FNET_mode == FNET_DOCSUMS && node->_pendingDocsums > 0) {
- uint32_t nodePendingDocsums = node->_pendingDocsums;
- FastS_assert(_pendingDocsums >= nodePendingDocsums);
- _pendingDocsums -= nodePendingDocsums;
- node->_pendingDocsums = 0;
- _pendingDocsumNodes--;
- if (error->_errorCode == search::engine::ECODE_TIMEOUT) {
- node->_flags._docsumTimeout = true;
- _docsumNodesTimedOut++;
- _docsumsTimedOut += nodePendingDocsums;
- }
- adjustDocsumTimeout();
- }
- error->Free();
- EndFNETWork(std::move(searchGuard));
-}
-
-
-void
-FastS_FNET_Search::HandleTimeout()
-{
- auto searchGuard(BeginFNETWork());
- if (!searchGuard) {
- return;
- }
-
- if (_FNET_mode == FNET_QUERY) {
- for (FastS_FNET_SearchNode & node : _nodes) {
- if (node._flags._pendingQuery) {
- FastS_assert(_pendingQueries > 0);
- _pendingQueries--;
- node._flags._pendingQuery = false;
- node._flags._queryTimeout = true;
- _queryNodesTimedOut++;
- double tnow = GetTimeKeeper()->GetTime();
- node._queryTime = tnow - _startTime;
- node.GetEngine()->UpdateSearchTime(tnow, node._queryTime, true);
- }
- }
- _queryTimeout = true;
- } else if (_FNET_mode == FNET_DOCSUMS) {
- for (FastS_FNET_SearchNode & node : _nodes) {
- if (node._pendingDocsums > 0) {
- uint32_t nodePendingDocsums = node._pendingDocsums;
- FastS_assert(_pendingDocsums >= nodePendingDocsums);
- _pendingDocsums -= nodePendingDocsums;
- _docsumsTimedOut += nodePendingDocsums;
- node._pendingDocsums = 0;
- node._flags._docsumTimeout = true;
- _docsumNodesTimedOut++;
- _pendingDocsumNodes--;
- }
- for (FastS_FNET_SearchNode::ExtraDocsumNodesIter iter(&node); iter.valid(); ++iter) {
- FastS_FNET_SearchNode *eNode = *iter;
- if (eNode->_pendingDocsums > 0) {
- uint32_t nodePendingDocsums = eNode->_pendingDocsums;
- FastS_assert(_pendingDocsums >= nodePendingDocsums);
- _pendingDocsums -= nodePendingDocsums;
- _docsumsTimedOut += nodePendingDocsums;
- eNode->_pendingDocsums = 0;
- eNode->_flags._docsumTimeout = true;
- _docsumNodesTimedOut++;
- _pendingDocsumNodes--;
- }
- }
- }
- _docsumTimeout = true;
- }
- EndFNETWork(std::move(searchGuard));
-}
-
-std::unique_lock<std::mutex>
-FastS_FNET_Search::BeginFNETWork()
-{
- std::unique_lock<std::mutex> searchGuard(_lock);
- if (_FNET_mode == FNET_NONE) {
- searchGuard.unlock();
- }
- return searchGuard;
-}
-
-void
-FastS_FNET_Search::EndFNETWork(std::unique_lock<std::mutex> searchGuard)
-{
- if (_FNET_mode == FNET_QUERY && _pendingQueries == 0) {
- _FNET_mode = FNET_NONE;
- searchGuard.unlock();
- _searchOwner->DoneQuery(this);
- } else if (_FNET_mode == FNET_DOCSUMS && _pendingDocsums == 0) {
- _FNET_mode = FNET_NONE;
- searchGuard.unlock();
- _searchOwner->DoneDocsums(this);
- }
-}
-
-bool
-FastS_FNET_Search::ShouldLimitHitsPerNode() const
-{
- return (_util.GetAlignedMaxHits() > _dataset->GetMaxHitsPerNode());
-}
-
-
-void
-FastS_FNET_Search::MergeHits()
-{
- FastS_HitMerger<FastS_FNETMerge> merger(this);
- merger.MergeHits();
-
- if (_util.IsEstimate())
- return;
-
- if (ShouldLimitHitsPerNode())
- _dataset->UpdateMaxHitsPerNodeLog(merger.WasIncomplete(), merger.WasFuzzy());
-
- if (!_queryArgs->groupSpec.empty()) {
- _groupMerger.reset(new search::grouping::MergingManager(_dataset->GetPartBits(), _dataset->GetRowBits()));
- for (const FastS_FNET_SearchNode & node : _nodes) {
- if (node._qresult != nullptr) {
- _groupMerger->addResult(node.getPartID(), node.GetRowID(),
- ((node._qresult->_features & search::fs4transport::QRF_MLD) != 0),
- node._qresult->_groupData, node._qresult->_groupDataLen);
- }
- }
- _groupMerger->merge();
- _util.SetGroupResultLen(_groupMerger->getGroupResultLen());
- _util.SetGroupResult(_groupMerger->getGroupResult());
- }
-}
-
-FastS_SearchInfo
-FastS_FNET_Search::computeCoverage(const std::vector<FastS_FNET_SearchNode> & nodes,
- uint32_t numSearchableCopies, bool adaptiveTimeout)
-{
- FastS_SearchInfo si;
- size_t cntNone(0);
- size_t askedButNotAnswered(0);
-
- for (const FastS_FNET_SearchNode & node : nodes) {
- if (node._qresult != nullptr) {
- si._coverageDocs += node._qresult->_coverageDocs;
- si._activeDocs += node._qresult->_activeDocs;
- si._soonActiveDocs += node._qresult->_soonActiveDocs;
- si._degradeReason |= node._qresult->_coverageDegradeReason;
- si._nodesQueried += node._qresult->getNodesQueried();
- si._nodesReplied += node._qresult->getNodesReplied();
- } else {
- si._nodesQueried++;
- cntNone++;
- if (node.IsConnected()) {
- askedButNotAnswered++;
- }
- }
- }
- bool missingReplies = (askedButNotAnswered != 0) || (si._nodesQueried != si._nodesReplied);
- const ssize_t missingParts = cntNone - (numSearchableCopies - 1);
-
- if (missingReplies && adaptiveTimeout) {
- // TODO This will not be correct when using multilevel dispatch and has timeout on anything, but leaf level.
- // We can live with that as leaf level failures are the likely ones.
- if (si._nodesReplied ) {
- si._activeDocs += askedButNotAnswered * si._activeDocs/si._nodesReplied;
- si._soonActiveDocs += askedButNotAnswered * si._soonActiveDocs/si._nodesReplied;
- }
- si._degradeReason |= search::engine::SearchReply::Coverage::ADAPTIVE_TIMEOUT;
- } else if (missingParts > 0) {
- // TODO This is a dirty way of anticipating missing coverage.
- // It should be done differently
- if ((cntNone != nodes.size())) {
- si._activeDocs += missingParts * si._activeDocs/(nodes.size() - cntNone);
- si._soonActiveDocs += missingParts * si._soonActiveDocs/(nodes.size() - cntNone);
- }
- si._degradeReason |= search::engine::SearchReply::Coverage::TIMEOUT;
-
- }
- return si;
-}
-
-void
-FastS_FNET_Search::CheckCoverage()
-{
- FastS_SearchInfo si = computeCoverage(_nodes, _dataset->getSearchableCopies(), useAdaptiveTimeout());
- _util.SetCoverage(si._coverageDocs, si._activeDocs, si._soonActiveDocs,
- si._degradeReason, si._nodesQueried, si._nodesReplied);
-}
-
-
-void
-FastS_FNET_Search::CheckQueryTimes()
-{
- double factor = _dataset->GetSlowQueryLimitFactor();
- double bias = _dataset->GetSlowQueryLimitBias();
- double queryTime = 0.0;
- int queryCnt = 0;
-
- for (const FastS_FNET_SearchNode & node : _nodes) {
- if (node.IsConnected() && node._queryTime > 0.0) {
- queryTime += node._queryTime;
- queryCnt++;
- }
- }
-
- if (queryCnt == 0)
- return;
-
- queryTime = queryTime / (double)queryCnt;
- double maxQueryTime = queryTime * factor + bias;
-
- for (const FastS_FNET_SearchNode & node : _nodes) {
- if (node.IsConnected() && node._queryTime > maxQueryTime) {
- node.GetEngine()->SlowQuery(maxQueryTime, node._queryTime - maxQueryTime, false);
- }
- }
-}
-
-
-void
-FastS_FNET_Search::CheckDocsumTimes()
-{
- double factor = _dataset->GetSlowDocsumLimitFactor();
- double bias = _dataset->GetSlowDocsumLimitBias();
- double docsumTime = 0.0;
- int docsumCnt = 0;
-
- for (const FastS_FNET_SearchNode & node : _nodes) {
- if (node.IsConnected() && node._docsumTime > 0.0) {
- docsumTime += node._docsumTime;
- docsumCnt++;
- }
- }
- if (docsumCnt == 0)
- return;
- docsumTime = docsumTime / (double)docsumCnt;
- double maxDocsumTime = docsumTime * factor + bias;
-
- for (const FastS_FNET_SearchNode & node : _nodes) {
- if (node.IsConnected() && node._docsumTime > maxDocsumTime) {
- node.GetEngine()->SlowDocsum(maxDocsumTime, node._docsumTime - maxDocsumTime);
- }
- for (FastS_FNET_SearchNode::ExtraDocsumNodesIter iter(&node); iter.valid(); ++iter) {
- FastS_FNET_SearchNode *eNode = *iter;
- if (eNode->IsConnected() && eNode->_docsumTime > maxDocsumTime) {
- eNode->GetEngine()->SlowDocsum(maxDocsumTime, eNode->_docsumTime - maxDocsumTime);
- }
- }
- }
-}
-
-
-void
-FastS_FNET_Search::CheckQueryTimeout()
-{
- if (_queryNodes != 0 && _queryNodesTimedOut >= _queryNodes)
- SetError(search::engine::ECODE_TIMEOUT, nullptr);
- if (!_queryTimeout)
- return;
-
- vespalib::string nodeList;
- uint32_t nodeCnt = 0;
- uint32_t printNodes = 10;
- for (const FastS_FNET_SearchNode & node : _nodes) {
- if (node._flags._queryTimeout) {
- if (nodeCnt < printNodes) {
- if (nodeCnt > 0) {
- nodeList.append(", ");
- }
- nodeList.append(node.GetEngine()->GetName());
- }
- ++nodeCnt;
- }
- }
- if (nodeCnt > printNodes) {
- nodeList.append(", ...");
- }
- vespalib::string query = _util.GetQuery().getPrintableQuery();
- LOG(warning, "%u nodes(%s) timed out during query execution (%s)",
- nodeCnt, nodeList.c_str(), query.c_str());
-}
-
-
-void
-FastS_FNET_Search::CheckDocsumTimeout()
-{
- if (_docsumNodes != 0 && _docsumNodesTimedOut >= _docsumNodes)
- SetError(search::engine::ECODE_TIMEOUT, nullptr);
- if (!_docsumTimeout)
- return;
-
- vespalib::string nodeList;
- uint32_t nodeCnt = 0;
- uint32_t printNodes = 10;
- for (const FastS_FNET_SearchNode & node : _nodes) {
- if (node._flags._docsumTimeout) {
- if (nodeCnt < printNodes) {
- if (nodeCnt > 0) {
- nodeList.append(", ");
- }
- nodeList.append(node.GetEngine()->GetName());
- }
- ++nodeCnt;
- }
- for (FastS_FNET_SearchNode::ExtraDocsumNodesIter iter(&node); iter.valid(); ++iter) {
- FastS_FNET_SearchNode *eNode = *iter;
- if (eNode->_flags._docsumTimeout) {
- if (nodeCnt < printNodes) {
- if (nodeCnt > 0) {
- nodeList.append(", ");
- }
- nodeList.append(eNode->GetEngine()->GetName());
- }
- ++nodeCnt;
- }
- }
- }
- if (nodeCnt > printNodes) {
- nodeList.append(", ...");
- }
- double elapsed = GetTimeKeeper()->GetTime() - _docSumStartTime;
- LOG(warning, "%u nodes given %1.6f seconds timeout timed out during docsum fetching after %1.6f seconds (%s)",
- nodeCnt, _adjustedDocSumTimeOut, elapsed, nodeList.c_str());
-}
-
-
-FastS_ISearch::RetCode
-FastS_FNET_Search::Search(uint32_t searchOffset,
- uint32_t maxhits, uint32_t minhits)
-{
- // minhits is never sent down from dispatch...
- (void) minhits; // ignore
-
- _util.setSearchRequest(_queryArgs);
- _util.SetupQuery(maxhits, searchOffset);
- if (_util.IsEstimate())
- _util.InitEstimateMode();
- _util.AdjustSearchParameters(_nodes.size());
- _util.AdjustSearchParametersFinal(_nodes.size());
-
- vespalib::string searchPath;
- const search::fef::Properties & model = _queryArgs->propertiesMap.modelOverrides();
- search::fef::Property searchPathProperty = model.lookup("searchpath");
- if (searchPathProperty.found()) {
- searchPath = searchPathProperty.get();
- }
- _adjustedQueryTimeOut = static_cast<double>(_queryArgs->getTimeLeft().ms()) / 1000.0;
- if ( ! searchPath.empty()) {
- connectSearchPath(searchPath);
- } else if (_util.IsEstimate()) {
- ConnectEstimateNodes();
- } else {
- ConnectQueryNodes();
- }
-
- // we support error packets
- uint32_t qflags = _util.GetQuery().GetQueryFlags();
-
- // propagate drop-sortdata flag only if we have single sub-node
- if (_nodes.size() != 1)
- qflags &= ~search::fs4transport::QFLAG_DROP_SORTDATA;
-
- uint32_t hitsPerNode = ShouldLimitHitsPerNode()
- ? _dataset->GetMaxHitsPerNode()
- : _util.GetAlignedMaxHits();
-
- // set up expected _queryNodes, _pendingQueries and node->_flags._pendingQuery state
- for (FastS_FNET_SearchNode & node : _nodes) {
- if (node.IsConnected()) {
- node._flags._pendingQuery = true;
- _pendingQueries++;
- _queryNodes++;
- }
- }
- size_t num_send_ok = 0; // number of partitions where packet send succeeded
- std::vector<uint32_t> send_failed; // partitions where packet send failed
-
- // allow FNET responses while requests are being sent
- {
- std::lock_guard<std::mutex> searchGuard(_lock);
- ++_pendingQueries; // add Elephant query node to avoid early query done
- ++_queryNodes; // add Elephant query node to avoid early query done
- _FNET_mode = FNET_QUERY;
- _queryStartTime = GetTimeKeeper()->GetTime();
- _timeout.Schedule(_adjustedQueryTimeOut);
- }
- FNET_Packet::SP shared(new FS4Packet_PreSerialized(*setupQueryPacket(hitsPerNode, qflags, _queryArgs->propertiesMap)));
- for (uint32_t i = 0; i < _nodes.size(); i++) {
- FastS_FNET_SearchNode & node = _nodes[i];
- if (node.IsConnected()) {
- FNET_Packet::UP qx(new FS4Packet_Shared(shared));
- LOG(spam, "posting packet to node %d='%s'\npacket=%s", i, node.toString().c_str(), qx->Print(0).c_str());
- if (node.PostPacket(qx.release())) {
- ++num_send_ok;
- } else {
- send_failed.push_back(i);
- LOG(debug, "FAILED posting packet to node %d='%s'\npacket=%s", i, node.toString().c_str(), qx->Print(0).c_str());
- }
- }
- }
-
- // finalize setup and check if query is still in progress
- bool done;
- {
- std::lock_guard<std::mutex> searchGuard(_lock);
- assert(_queryNodes >= _pendingQueries);
- for (uint32_t i: send_failed) {
- // conditional revert of state for failed nodes
- if (_nodes[i]._flags._pendingQuery) {
- _nodes[i]._flags._pendingQuery = false;
- assert(_pendingQueries > 0);
- --_pendingQueries;
- --_queryNodes;
- }
- }
- // revert Elephant query node to allow search to complete
- assert(_pendingQueries > 0);
- --_pendingQueries;
- --_queryNodes;
- done = (_pendingQueries == 0);
- bool all_down = (num_send_ok == 0);
- if (done) {
- _FNET_mode = FNET_NONE;
- if (all_down) {
- SetError(search::engine::ECODE_ALL_PARTITIONS_DOWN, nullptr);
- }
- }
- }
-
- return (done) ? RET_OK : RET_INPROGRESS;
-}
-
-vespalib::string
-FastS_FNET_SearchNode::toString() const
-{
- vespalib::string s;
- s += vespalib::make_string("{ channel=%p={%d, c=%p='%s'}, partId = %d, rowid=%d }",
- _channel, _channel->GetID(),
- _channel->GetConnection(), _channel->GetConnection()->GetSpec(),
- _partid, _rowid);
- return s;
-}
-
-
-FNET_Packet::UP
-FastS_FNET_Search::setupQueryPacket(uint32_t hitsPerNode, uint32_t qflags,
- const search::engine::PropertiesMap &properties)
-{
- FNET_Packet::UP ret(new FS4Packet_QUERYX());
- FS4Packet_QUERYX & qx = static_cast<FS4Packet_QUERYX &>(*ret);
- qx._features = search::fs4transport::QF_PARSEDQUERY | search::fs4transport::QF_RANKP;
- qx._offset = _util.GetAlignedSearchOffset();
- qx._maxhits = hitsPerNode; // capped maxhits
- qx.setQueryFlags(qflags);
- qx.setTimeout(_queryArgs->getTimeLeft());
-
- qx.setRanking(_queryArgs->ranking);
-
- if (!_queryArgs->sortSpec.empty()) {
- qx._features |= search::fs4transport::QF_SORTSPEC;
- qx.setSortSpec(_queryArgs->sortSpec);
- }
-
- if (!_queryArgs->groupSpec.empty()) {
- qx._features |= search::fs4transport::QF_GROUPSPEC;
- qx.setGroupSpec(vespalib::stringref(&_queryArgs->groupSpec[0], _queryArgs->groupSpec.size()));
- }
-
- if (!_queryArgs->sessionId.empty()) {
- qx._features |= search::fs4transport::QF_SESSIONID;
- qx.setSessionId(vespalib::stringref(&_queryArgs->sessionId[0], _queryArgs->sessionId.size()));
- }
-
- if (!_queryArgs->location.empty()) {
- qx._features |= search::fs4transport::QF_LOCATION;
- qx.setLocation(_queryArgs->location);
- }
-
- if (properties.size() > 0) {
- PacketConverter::fillPacketProperties(properties, qx._propsVector);
- qx._features |= search::fs4transport::QF_PROPERTIES;
- }
-
- qx._numStackItems = _queryArgs->stackItems;
- qx.setStackDump(_queryArgs->getStackRef());
- return ret;
-}
-
-
-FastS_ISearch::RetCode
-FastS_FNET_Search::ProcessQueryDone()
-{
- CheckCoverage();
-
- if (_errorCode == search::engine::ECODE_NO_ERROR) {
- MergeHits();
- }
- _queryResult = *_util.GetQueryResult();
- double tnow = GetTimeKeeper()->GetTime();
- _queryResult._queryResultTime = tnow - _startTime;
- if (_errorCode == search::engine::ECODE_NO_ERROR) {
- if (_util.IsEstimate()) {
- _dataset->UpdateEstimateCount();
- } else {
- _dataset->UpdateSearchTime(tnow, _queryResult._queryResultTime, _queryNodesTimedOut != 0);
- }
- if ( _dataset->useFixedRowDistribution() ) {
- _dataset->updateSearchTime(_queryResult._queryResultTime, _fixedRow);
- }
- }
- CheckQueryTimes();
- CheckQueryTimeout();
- dropDatasetActiveCostRef();
- return RET_OK;
-}
-
-
-FastS_ISearch::RetCode
-FastS_FNET_Search::GetDocsums(const FastS_hitresult *hits, uint32_t hitcnt)
-{
- if (hitcnt > 0) {
- _resbuf.resize(hitcnt);
- }
-
- // copy values from query result
-
- uint32_t i;
- for (i = 0; i < hitcnt; i++) {
- _resbuf[i]._docid = 0;
- _resbuf[i]._gid = hits[i]._gid;
- _resbuf[i]._metric = hits[i]._metric;
- _resbuf[i]._partition = hits[i]._partition;
- }
-
- // determine docsum distribution among nodes
-
- const FastS_hitresult *p = hits;
- uint32_t rowbits = _dataset->GetRowBits();
- uint32_t partbits = _dataset->GetPartBits();
- uint32_t mldpartidmask = (1 << partbits) - 1;
- bool ignoreRow = (_docsumArgs->getFlags() &
- search::fs4transport::GDFLAG_IGNORE_ROW) != 0;
- if (rowbits > 0) {
- uint32_t rowmask = (1 << rowbits) - 1;
- for (i = 0; i < hitcnt; i++, p++) {
- FastS_FNET_SearchNode *node;
- uint32_t partid0 = p->_partition >> rowbits;
- uint32_t row = ignoreRow ? 0u : p->_partition & rowmask;
- if (IS_MLD_PART(partid0)) {
- uint32_t partid = MLD_PART_TO_PARTID(partid0);
- if (partid < _nodes.size()) {
- node = getNode(partid);
- if (node->_docidCnt == 0) {
- node->_flags._docsumMld = true;// Only accept MLD from now on
- node->_docsumRow = row;
- } else if (!node->_flags._docsumMld || row != node->_docsumRow) {
- if (_nodesConnected)
- continue; // Drop (inconsistent)
- node = node->allocExtraDocsumNode(true, row, rowbits);
- }
- node->_docidCnt++;
- }
- } else { // !MLD
- if (partid0 < _nodes.size()) {
- node = getNode(partid0);
- if (node->_docidCnt == 0) {
- node->_docsumRow = row;
- } else if (node->_flags._docsumMld || row != node->_docsumRow) {
- if (_nodesConnected)
- continue; // Drop (inconsistent)
- node = node->allocExtraDocsumNode(false, row, rowbits);
- }
- node->_docidCnt++;
- }
- }
- }
- } else { // rowbits == 0
- for (i = 0; i < hitcnt; i++, p++) {
- FastS_FNET_SearchNode *node;
- if (IS_MLD_PART(p->_partition)) {
- uint32_t partid = MLD_PART_TO_PARTID(p->_partition);
- if (partid < _nodes.size()) {
- node = getNode(partid);
- if (node->_docidCnt == 0) {
- node->_flags._docsumMld = true;// Only accept MLD from now on
- } else if (!node->_flags._docsumMld) {
- if (_nodesConnected)
- continue; // Drop (inconsistent)
- node = node->allocExtraDocsumNode(true, 0, 0);
- }
- node->_docidCnt++;
- }
- } else { // !MLD
- if (p->_partition < _nodes.size()) {
- node = getNode(p->_partition);
- if (node->_docidCnt == 0) {
- } else if (node->_flags._docsumMld) {
- if (_nodesConnected)
- continue; // Drop (inconsistent)
- node = node->allocExtraDocsumNode(false, 0, 0);
- }
- node->_docidCnt++;
- }
- }
- }
- }
- FastS_assert(p == hits + hitcnt);
-
- // allocate docsum requests and insert features
-
- search::docsummary::GetDocsumArgs *args = _docsumArgs;
- for (FastS_FNET_SearchNode & node : _nodes) {
- if (node._docidCnt != 0) {
- node.allocGDX(args, args->propertiesMap());
- }
- for (FastS_FNET_SearchNode::ExtraDocsumNodesIter iter(&node); iter.valid(); ++iter) {
- FastS_FNET_SearchNode *eNode = *iter;
- if (eNode->_docidCnt != 0)
- eNode->allocGDX(args, args->propertiesMap());
- }
- }
-
- // fill docid(/partid/stamp) data into docsum requests
-
- p = hits;
- if (rowbits > 0) {
- uint32_t rowmask = (1 << rowbits) - 1;
- for (i = 0; i < hitcnt; i++, p++) {
- FastS_FNET_SearchNode *node;
- uint32_t partid0 = p->_partition >> rowbits;
- uint32_t row = ignoreRow ? 0u : p->_partition & rowmask;
- if (IS_MLD_PART(partid0)) {
- uint32_t partid = MLD_PART_TO_PARTID(partid0);
- if (partid < _nodes.size()) {
- node = getNode(partid);
- if (!node->_flags._docsumMld || row != node->_docsumRow) {
- if (_nodesConnected)
- continue; // Drop (inconsistent)
- node = node->allocExtraDocsumNode(true, row, rowbits);
- }
-
- FS4Packet_GETDOCSUMSX::FS4_docid &q = node->_gdx->_docid[node->_docsum_offsets_idx];
- q._gid = p->_gid;
- q._partid = DECODE_MLD_PART(partid0);
- node->_docsum_offsets[node->_docsum_offsets_idx++] = i;
- }
- } else { // !MLD
- if (partid0 < _nodes.size()) {
- node = getNode(partid0);
- if (node->_flags._docsumMld || row != node->_docsumRow) {
- if (_nodesConnected)
- continue; // Drop (inconsistent)
- node = node->allocExtraDocsumNode(false, row, rowbits);
- }
-
- FS4Packet_GETDOCSUMSX::FS4_docid &q = node->_gdx->_docid[node->_docsum_offsets_idx];
- q._gid = p->_gid;
- node->_docsum_offsets[node->_docsum_offsets_idx++] = i;
- }
- }
- }
- } else { // rowbits == 0
- for (i = 0; i < hitcnt; i++, p++) {
- FastS_FNET_SearchNode *node;
- if (IS_MLD_PART(p->_partition)) {
- uint32_t partid = MLD_PART_TO_PARTID(p->_partition);
- if (partid < _nodes.size()) {
- node = getNode(partid);
- if (!node->_flags._docsumMld) {
- if (_nodesConnected)
- continue; // Drop (inconsistent)
- node = node->allocExtraDocsumNode(true, 0, 0);
- }
-
- FS4Packet_GETDOCSUMSX::FS4_docid &q = node->_gdx->_docid[node->_docsum_offsets_idx];
- q._gid = p->_gid;
- q._partid = DECODE_MLD_PART(p->_partition);
- node->_docsum_offsets[node->_docsum_offsets_idx++] = i;
- }
- } else { // !MLD
- if (p->_partition < _nodes.size()) {
- node = getNode(p->_partition);
- if (node->_flags._docsumMld) {
- if (_nodesConnected)
- continue; // Drop (inconsistent)
- node = node->allocExtraDocsumNode(false, 0, 0);
- }
-
- FS4Packet_GETDOCSUMSX::FS4_docid &q = node->_gdx->_docid[node->_docsum_offsets_idx];
- q._gid = p->_gid;
- node->_docsum_offsets[node->_docsum_offsets_idx++] = i;
- }
- }
- }
- }
- FastS_assert(p == hits + hitcnt);
-
- ConnectDocsumNodes(ignoreRow);
- bool done;
- {
- std::lock_guard<std::mutex> searchGuard(_lock);
-
- // patch in engine dependent features and send docsum requests
-
- for (FastS_FNET_SearchNode & node : _nodes) {
- if (node._gdx != nullptr)
- node.postGDX(&_pendingDocsums, &_docsumNodes);
- for (FastS_FNET_SearchNode::ExtraDocsumNodesIter iter(&node); iter.valid(); ++iter) {
- FastS_FNET_SearchNode *eNode = *iter;
- if (eNode->_gdx != nullptr)
- eNode->postGDX(&_pendingDocsums, &_docsumNodes);
- }
- }
- _pendingDocsumNodes = _docsumNodes;
- _requestedDocsums = _pendingDocsums;
-
- done = (_pendingDocsums == 0);
- if (!done) {
- _FNET_mode = FNET_DOCSUMS; // FNET; do your thing
-
- _adjustedDocSumTimeOut = args->getTimeout().sec();
- _docSumStartTime = GetTimeKeeper()->GetTime();
- _timeout.Schedule(_adjustedDocSumTimeOut);
- }
- }
-
- return (done) ? RET_OK : RET_INPROGRESS;
-}
-
-
-FastS_ISearch::RetCode
-FastS_FNET_Search::ProcessDocsumsDone()
-{
- _docsumsResult._fullresult = &_resbuf[0];
- _docsumsResult._fullResultCount = _resbuf.size();
- _docsumsResult._queryDocSumTime = GetTimeKeeper()->GetTime() - _startTime;
- CheckDocsumTimes();
- CheckDocsumTimeout();
- dropDatasetActiveCostRef();
- return RET_OK;
-}
-
-bool
-FastS_FNET_Search::useAdaptiveTimeout() const {
- return _dataset->getMinimalSearchCoverage() < 100.0;
-}
-
-void
-FastS_FNET_Search::adjustQueryTimeout()
-{
- uint32_t pendingQueries = getPendingQueries();
-
- if ((pendingQueries == 0) ||
- _util.IsQueryFlagSet(search::fs4transport::QFLAG_DUMP_FEATURES) ||
- ! useAdaptiveTimeout())
- {
- return;
- }
-
- double mincoverage = _dataset->getMinimalSearchCoverage();
- uint32_t wantedAnswers = std::ceil(getRequestedQueries() * mincoverage / 100.0);
- LOG(spam, "Adjusting wanted answers from %u to %u", getRequestedQueries(), wantedAnswers);
- if (getDoneQueries() < wantedAnswers) {
- return;
- }
- if (!_queryWaitCalculated) {
- double timeLeft = _queryArgs->getTimeLeft().sec();
- _queryMinWait = timeLeft * _dataset->getHigherCoverageMinSearchWait();
- _queryMaxWait = timeLeft * _dataset->getHigherCoverageMaxSearchWait();
- _queryWaitCalculated = true;
- }
-
- double basewait = 0.0;
- double minwait = _queryMinWait;
- double maxwait = _queryMaxWait;
-
- double elapsed = GetTimeKeeper()->GetTime() - _queryStartTime;
-
- double missWidth = ((100.0 - mincoverage) * getRequestedQueries()) / 100.0 - 1.0;
-
- double slopedwait = minwait;
-
- if (pendingQueries > 1 && missWidth > 0.0)
- slopedwait += ((maxwait - minwait) * (pendingQueries - 1)) / missWidth;
-
- double newTimeOut = std::max(elapsed, basewait) + slopedwait;
-
-
- if (newTimeOut >= _adjustedQueryTimeOut)
- return;
-
- _adjustedQueryTimeOut = newTimeOut;
- if (newTimeOut > elapsed)
- _timeout.Schedule(newTimeOut - elapsed);
- else
- _timeout.ScheduleNow();
-}
-
-
-void
-FastS_FNET_Search::adjustDocsumTimeout()
-{
- uint32_t pendingDocsums = getPendingDocsums();
-
- if (pendingDocsums == 0 || _util.IsQueryFlagSet(search::fs4transport::QFLAG_DUMP_FEATURES)) {
- return;
- }
-
- double coverage = static_cast<double>(getDoneDocsums() * 100) / getRequestedDocsums();
-
- double mincoverage = _dataset->getMinimalDocSumCoverage();
-
- if (coverage < mincoverage)
- return;
-
- double basewait = _dataset->getHigherCoverageBaseDocSumWait();
- double minwait = _dataset->getHigherCoverageMinDocSumWait();
- double maxwait = _dataset->getHigherCoverageMaxDocSumWait();
-
- double elapsed = GetTimeKeeper()->GetTime() - _docSumStartTime;
-
- double missWidth = ((100.0 - mincoverage) * getRequestedDocsums()) / 100.0 - 1.0;
-
- double slopedwait = minwait;
-
- if (pendingDocsums > 1 && missWidth > 0.0)
- slopedwait += ((maxwait - minwait) * (pendingDocsums - 1)) / missWidth;
-
- double newTimeOut = std::max(elapsed, basewait) + slopedwait;
-
- if (newTimeOut >= _adjustedDocSumTimeOut)
- return;
-
- _adjustedDocSumTimeOut = newTimeOut;
- if (newTimeOut > elapsed)
- _timeout.Schedule(newTimeOut - elapsed);
- else
- _timeout.ScheduleNow();
-}
-
-
-FastS_Sync_FNET_Search::~FastS_Sync_FNET_Search() {}
diff --git a/searchcore/src/vespa/searchcore/fdispatch/search/fnet_search.h b/searchcore/src/vespa/searchcore/fdispatch/search/fnet_search.h
deleted file mode 100644
index a77b984ca28..00000000000
--- a/searchcore/src/vespa/searchcore/fdispatch/search/fnet_search.h
+++ /dev/null
@@ -1,375 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-
-#pragma once
-
-#include <vespa/fnet/fnet.h>
-#include <vespa/vespalib/objects/nbostream.h>
-#include <vespa/searchcore/fdispatch/common/search.h>
-#include <vespa/searchlib/common/sortdata.h>
-#include <vespa/searchcore/grouping/mergingmanager.h>
-#include <vespa/searchcore/fdispatch/search/search_path.h>
-#include <vespa/searchcore/fdispatch/search/querycacheutil.h>
-#include <vespa/searchcore/fdispatch/search/fnet_engine.h>
-
-class FastS_FNET_Engine;
-class FastS_FNET_Search;
-
-using search::fs4transport::FS4Packet_QUERYRESULTX;
-using search::fs4transport::FS4Packet_GETDOCSUMSX;
-using search::fs4transport::FS4Packet_DOCSUM;
-using search::fs4transport::FS4Packet_TRACEREPLY;
-
-//-----------------------------------------------------------------
-
-class FastS_FNET_SearchNode : public FNET_IPacketHandler
-{
-public:
- class ExtraDocsumNodesIter;
- typedef std::unique_ptr<FastS_FNET_SearchNode> UP;
-private:
- friend class ExtraDocsumNodesIter;
-
- FastS_FNET_Search *_search; // we are part of this search
- FastS_FNET_Engine *_engine; // we use this search engine
- FNET_Channel *_channel; // connection with search engine
- uint32_t _partid; // engine partition id
- uint32_t _rowid; // engine row id
- uint32_t _stamp; // engine timestamp
-
-public:
-
- FS4Packet_QUERYRESULTX *_qresult; // query result packet
- double _queryTime;
- struct Flags {
- Flags() :
- _pendingQuery(false),
- _docsumMld(false),
- _queryTimeout(false),
- _docsumTimeout(false),
- _needSubCost(false)
- { }
- bool _pendingQuery; // is query pending ?
- bool _docsumMld;
- bool _queryTimeout;
- bool _docsumTimeout;
- bool _needSubCost;
- };
-
- Flags _flags;
-
-// Docsum related stuff.
- uint32_t _docidCnt;
- uint32_t _pendingDocsums; // how many docsums pending ?
- uint32_t _docsumRow;
- uint32_t _docsum_offsets_idx;
- double _docsumTime;
-
- FS4Packet_GETDOCSUMSX *_gdx;
- std::vector<uint32_t> _docsum_offsets;
-private:
- std::vector<FastS_FNET_SearchNode::UP> _extraDocsumNodes;
- FastS_FNET_SearchNode *_nextExtraDocsumNode;
- FastS_FNET_SearchNode *_prevExtraDocsumNode;
-public:
-
-// Query processing stuff.
- FS4Packet_QUERYRESULTX::FS4_hit *_hit_beg; // hit array start
- FS4Packet_QUERYRESULTX::FS4_hit *_hit_cur; // current hit
- FS4Packet_QUERYRESULTX::FS4_hit *_hit_end; // end boundary
-
- search::common::SortDataIterator _sortDataIterator;
-
-public:
- FastS_FNET_SearchNode(FastS_FNET_Search *search, uint32_t partid);
- // These objects are referenced everywhere and must never be either copied nor moved,
- // but std::vector requires this to exist. If called it will assert.
- FastS_FNET_SearchNode(FastS_FNET_SearchNode && rhs);
- FastS_FNET_SearchNode(const FastS_FNET_SearchNode &) = delete;
- FastS_FNET_SearchNode& operator=(const FastS_FNET_SearchNode &) = delete;
-
- ~FastS_FNET_SearchNode() override;
-
- // Methods needed by mergehits
- bool NT_InitMerge(uint32_t *numDocs, uint64_t *totalHits, search::HitRank *maxRank, uint32_t *sortDataDocs);
- search::common::SortDataIterator *NT_GetSortDataIterator() { return &_sortDataIterator; }
- FS4Packet_QUERYRESULTX::FS4_hit *NT_GetHit() const { return _hit_cur; }
- uint32_t NT_GetNumHitsUsed() const { return (_hit_cur - _hit_beg); }
- uint32_t NT_GetNumHitsLeft() const { return (_hit_end - _hit_cur); }
- uint64_t NT_GetTotalHits() const { return (_qresult != nullptr) ? _qresult->_totNumDocs : 0; }
- uint32_t NT_GetNumHits() const { return (_hit_end - _hit_beg); }
- void NT_NextHit() { _hit_cur++; }
-
- uint32_t getPartID() const { return _partid; }
- uint32_t GetRowID() const { return _rowid; }
-
- FastS_FNET_SearchNode * allocExtraDocsumNode(bool mld, uint32_t rowid, uint32_t rowbits);
-
- FastS_FNET_Engine *GetEngine() const { return _engine; }
-
- bool IsConnected() const { return _channel != nullptr; }
- void Connect(FastS_FNET_Engine *engine);
- void Connect_HasDSLock(FastS_FNET_Engine *engine);
- FastS_EngineBase * getPartition(const std::unique_lock<std::mutex> &dsGuard, bool userow, FastS_FNET_DataSet *dataset);
- void allocGDX(search::docsummary::GetDocsumArgs *args, const search::engine::PropertiesMap &properties);
- void postGDX(uint32_t *pendingDocsums, uint32_t *pendingDocsumNodes);
- vespalib::string toString() const;
-
- void dropCost() {
- if (_engine != nullptr && _flags._needSubCost) {
- _engine->SubCost();
- _flags._needSubCost = false;
- }
- }
-
- void DirtySetChannelOnlyForTesting(FNET_Channel * channel) {
- _channel = channel;
- }
-
-
- void Disconnect()
- {
- if (_channel != nullptr) {
- _channel->CloseAndFree();
- _channel = nullptr;
- }
- if (_engine != nullptr) {
- if (_flags._needSubCost) {
- _engine->SubCost();
- _flags._needSubCost = false;
- }
- _engine = nullptr;
- }
- }
-
-
- bool PostPacket(FNET_Packet *packet) {
- return (_channel == nullptr) ? packet->Free(), false : _channel->Send(packet);
- }
-
- HP_RetCode HandlePacket(FNET_Packet *packet, FNET_Context context) override;
-};
-
-
-class FastS_FNET_SearchNode::ExtraDocsumNodesIter
-{
-private:
- ExtraDocsumNodesIter(const ExtraDocsumNodesIter &other);
- ExtraDocsumNodesIter& operator=(const ExtraDocsumNodesIter &other);
-
- FastS_FNET_SearchNode *_cur;
- const FastS_FNET_SearchNode *_head;
-
-public:
- ExtraDocsumNodesIter(const FastS_FNET_SearchNode *head)
- : _cur(head->_nextExtraDocsumNode),
- _head(head)
- {
- }
-
- ExtraDocsumNodesIter & operator++() {
- _cur = _cur->_nextExtraDocsumNode;
- return *this;
- }
-
- bool valid() const { return _cur != _head; }
- FastS_FNET_SearchNode *operator*() const { return _cur; }
-};
-
-
-//-----------------------------------------------------------------
-
-class FastS_FNET_Search : public FastS_AsyncSearch
-{
-private:
- FastS_FNET_Search(const FastS_FNET_Search &);
- FastS_FNET_Search& operator=(const FastS_FNET_Search &);
-
-public:
-
- class Timeout : public FNET_Task
- {
- private:
- Timeout(const Timeout &);
- Timeout& operator=(const Timeout &);
-
- FastS_FNET_Search *_search;
-
- public:
- Timeout(FNET_Scheduler *scheduler, FastS_FNET_Search *search)
- : FNET_Task(scheduler),
- _search(search) {}
- void PerformTask() override;
- };
-
- enum FNETMode {
- FNET_NONE = 0x00,
- FNET_QUERY = 0x01,
- FNET_DOCSUMS = 0x02
- };
-
-private:
- std::mutex _lock;
- FastS_TimeKeeper *_timeKeeper;
- double _startTime;
- Timeout _timeout;
- FastS_QueryCacheUtil _util;
- std::unique_ptr<search::grouping::MergingManager> _groupMerger;
- FastS_DataSetCollection *_dsc; // owner keeps this alive
- FastS_FNET_DataSet *_dataset;
- bool _datasetActiveCostRef;
- std::vector<FastS_FNET_SearchNode> _nodes;
- bool _nodesConnected;
-
- uint32_t _estParts;
- uint32_t _estPartCutoff;
-
- FNETMode _FNET_mode;
-
- uint32_t _pendingQueries; // # nodes with query left
- uint32_t _pendingDocsums; // # docsums left
- uint32_t _pendingDocsumNodes; // # nodes with docsums left
- uint32_t _requestedDocsums; // # docsums requested
- uint32_t _queryNodes; // #nodes with query
- uint32_t _queryNodesTimedOut; // #nodes with query timeout
- uint32_t _docsumNodes; // #nodes with docsums
- uint32_t _docsumNodesTimedOut; // #nodes with docsum timeout
- uint32_t _docsumsTimedOut;
- bool _queryTimeout;
- bool _docsumTimeout;
-
- double _queryStartTime;
- double _queryMinWait;
- double _queryMaxWait;
- bool _queryWaitCalculated;
- double _adjustedQueryTimeOut;
- double _docSumStartTime;
- double _adjustedDocSumTimeOut;
- uint32_t _fixedRow;
-
- std::vector<FastS_fullresult> _resbuf;
-
- void dropDatasetActiveCostRef();
-
- typedef std::vector<std::pair<FastS_EngineBase *, FastS_FNET_SearchNode *>> EngineNodeMap;
- void connectNodes(const EngineNodeMap & engines);
- void reallocNodes(size_t numParts);
- void ConnectQueryNodes();
- void ConnectEstimateNodes();
- void connectSearchPath(const vespalib::string &spec);
- void connectSearchPath(const fdispatch::SearchPath::Element &elem,
- const vespalib::string &spec, uint32_t dispatchLevel);
- void ConnectDocsumNodes(bool ignoreRow);
- uint32_t getNextFixedRow();
- uint32_t getFixedRowCandidate();
- uint32_t getHashedRow() const;
-
- std::unique_lock<std::mutex> BeginFNETWork();
- void EndFNETWork(std::unique_lock<std::mutex> searchGuard);
-
- void EncodePartIDs(uint32_t partid, uint32_t rowid, bool mld,
- FS4Packet_QUERYRESULTX::FS4_hit *pt,
- FS4Packet_QUERYRESULTX::FS4_hit *end);
-
- FastS_TimeKeeper *GetTimeKeeper() const { return _timeKeeper; }
-
- FastS_FNET_SearchNode * getNode(size_t i) { return &_nodes[i]; }
-public:
- FastS_FNET_Search(FastS_DataSetCollection *dsc, FastS_FNET_DataSet *dataset, FastS_TimeKeeper *timeKeeper);
- virtual ~FastS_FNET_Search();
-
- void GotQueryResult(FastS_FNET_SearchNode *node, FS4Packet_QUERYRESULTX *qrx);
- void GotDocsum(FastS_FNET_SearchNode *node, FS4Packet_DOCSUM *docsum);
- void LostSearchNode(FastS_FNET_SearchNode *node);
- void GotEOL(FastS_FNET_SearchNode *node);
- void GotError(FastS_FNET_SearchNode *node, search::fs4transport::FS4Packet_ERROR *error);
-
- void HandleTimeout();
-
- bool ShouldLimitHitsPerNode() const;
- void MergeHits();
- void CheckCoverage();
- static FastS_SearchInfo computeCoverage(const std::vector<FastS_FNET_SearchNode> & nodes,
- uint32_t numSearchableCopies, bool adaptiveTimeout);
- void CheckQueryTimes();
- void CheckDocsumTimes();
- void CheckQueryTimeout();
- void CheckDocsumTimeout();
-
- // *** API methods -- BEGIN ***
-
- FastS_SearchInfo *GetSearchInfo() override { return _util.GetSearchInfo(); }
-
- RetCode Search(uint32_t searchOffset, uint32_t maxhits, uint32_t minhits = 0) override;
- RetCode ProcessQueryDone() override;
- RetCode GetDocsums(const FastS_hitresult *hits, uint32_t hitcnt) override;
- RetCode ProcessDocsumsDone() override;
-
- // *** API methods -- END ***
-
- // Hit merging methods
-
- FastS_FNET_SearchNode *ST_GetNode(size_t i) { return getNode(i); }
- uint32_t ST_GetNumNodes() const { return _nodes.size(); }
- bool ST_IsEstimate() const { return _util.IsEstimate(); }
- uint32_t ST_GetEstParts() const { return _estParts; }
- uint32_t ST_GetEstPartCutoff() const { return _estPartCutoff; }
- bool ST_ShouldDropSortData() const { return _util.ShouldDropSortData(); }
- bool ST_ShouldLimitHitsPerNode() const { return ShouldLimitHitsPerNode(); }
- void ST_SetNumHits(uint32_t numHits) {
- _util.SetAlignedHitCount(numHits);
- _util.CalcHitCount();
- _util.AllocAlignedHitBuf();
- }
- void ST_AdjustNumHits(uint32_t numHits) {
- _util.SetAlignedHitCount(numHits);
- _util.CalcHitCount();
- }
- uint32_t ST_GetAlignedSearchOffset() const { return _util.GetAlignedSearchOffset(); }
- uint32_t ST_GetAlignedMaxHits() const { return _util.GetAlignedMaxHits(); }
- uint32_t ST_GetAlignedHitCount() const { return _util.GetAlignedHitCount(); }
- FastS_hitresult *ST_GetAlignedHitBuf() { return _util.GetAlignedHitBuf(); }
- FastS_hitresult *ST_GetAlignedHitBufEnd() { return _util.GetAlignedHitBufEnd(); }
- void ST_AllocSortData(uint32_t len) { _util.AllocSortData(len); }
- uint32_t *ST_GetSortIndex() { return _util.GetSortIndex(); }
- char *ST_GetSortData() { return _util.GetSortData(); }
- FastS_QueryResult *ST_GetQueryResult() { return _util.GetQueryResult(); }
-
- bool useAdaptiveTimeout() const;
- void adjustQueryTimeout();
- void adjustDocsumTimeout();
- uint32_t getRequestedQueries() const { return _queryNodes; }
- uint32_t getPendingQueries() const { return _pendingQueries; }
- uint32_t getDoneQueries() const {
- return getRequestedQueries() - getPendingQueries();
- }
- uint32_t getRequestedDocsums() const { return _requestedDocsums; }
- uint32_t getPendingDocsums() const { return _pendingDocsums; }
- uint32_t getDoneDocsums() const {
- return getRequestedDocsums() - getPendingDocsums();
- }
-
- FNET_Packet::UP
- setupQueryPacket(uint32_t hitsPerNode, uint32_t qflags,
- const search::engine::PropertiesMap &properties);
-};
-
-//-----------------------------------------------------------------------------
-
-class FastS_Sync_FNET_Search : public FastS_SyncSearchAdapter
-{
-private:
- FastS_FNET_Search _search;
-
-public:
- FastS_Sync_FNET_Search(FastS_DataSetCollection *dsc, FastS_FNET_DataSet *dataset, FastS_TimeKeeper *timeKeeper) :
- FastS_SyncSearchAdapter(&_search),
- _search(dsc, dataset, timeKeeper)
- {
- _search.SetAsyncArgs(this);
- }
- ~FastS_Sync_FNET_Search() override;
- void Free() override { delete this; }
-};
-
-//-----------------------------------------------------------------
-
diff --git a/searchcore/src/vespa/searchcore/fdispatch/search/mergehits.cpp b/searchcore/src/vespa/searchcore/fdispatch/search/mergehits.cpp
deleted file mode 100644
index 8c4a08a3bbb..00000000000
--- a/searchcore/src/vespa/searchcore/fdispatch/search/mergehits.cpp
+++ /dev/null
@@ -1,283 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-
-#include "mergehits.h"
-#include "querycacheutil.h"
-#include "fnet_dataset.h"
-#include "fnet_search.h"
-#include <vespa/searchcore/util/stlishheap.h>
-#include <vespa/vespalib/stllike/hash_set.h>
-#include <vespa/vespalib/stllike/hash_set.hpp>
-
-#include <vespa/log/log.h>
-LOG_SETUP(".fdispatch.mergehits");
-
-using search::common::SortData;
-using search::common::SortDataIterator;
-
-//-----------------------------------------------------------------------------
-
-template <bool SORTDATA, bool DROP>
-struct FastS_MergeFeatures
-{
- static bool UseSortData() { return SORTDATA; }
- static bool DropSortData() { return DROP; }
-};
-
-
-template <typename T, typename F>
-bool
-FastS_MergeCompare(typename T::NodeType *a,
- typename T::NodeType *b)
-{
- bool prefer_b = (b->NT_GetNumHitsUsed() < a->NT_GetNumHitsUsed());
- if (F::UseSortData()) {
- return b->NT_GetSortDataIterator()->Before(a->NT_GetSortDataIterator(), prefer_b);
- } else {
- search::HitRank rank_a = a->NT_GetHit()->HT_GetMetric();
- search::HitRank rank_b = b->NT_GetHit()->HT_GetMetric();
- return ((rank_b > rank_a) || ((rank_b == rank_a) && prefer_b));
- }
-}
-
-
-template <typename T>
-inline void
-FastS_MergeCopySortData(typename T::NodeType *node,
- SortData::Ref *dst,
- uint32_t &sortDataLen)
-{
- if (dst == NULL)
- return;
-
- SortDataIterator *src = node->NT_GetSortDataIterator();
- dst->_buf = src->GetBuf();
- dst->_len = src->GetLen();
- sortDataLen += dst->_len;
-}
-
-
-template <typename T>
-inline void
-FastS_MergeCopyHit(typename T::HitType *src,
- FastS_hitresult *dst)
-{
- dst->HT_SetGlobalID(src->HT_GetGlobalID());
- dst->HT_SetMetric(src->HT_GetMetric());
- dst->HT_SetPartID(src->HT_GetPartID());
- dst->setDistributionKey(src->getDistributionKey());
-}
-
-struct GlobalIdHasher {
- vespalib::hash_set<document::GlobalId, document::GlobalId::hash> seenSet;
- bool insert(const document::GlobalId & g_id) {
- return seenSet.insert(g_id).second;
- }
- GlobalIdHasher(size_t expected_size) : seenSet(expected_size * 3) {}
-};
-
-
-template <typename T, typename F>
-size_t
-FastS_InternalMergeHits(FastS_HitMerger<T> *merger)
-{
- typename T::SearchType *search = merger->GetSearch();
- typename T::NodeType **heap = merger->GetHeap();
- uint32_t heapSize = merger->GetHeapSize();
- typename T::NodeType *node = NULL;
- FastS_hitresult *beg = search->ST_GetAlignedHitBuf();
- FastS_hitresult *end = search->ST_GetAlignedHitBufEnd();
- FastS_hitresult *pt = beg;
-
- // multi-level sorting related variables
- SortData::Ref *sortRef = NULL;
- SortData::Ref *sortItr = NULL;
- uint32_t sortDataLen = 0;
-
- if (F::UseSortData() && !F::DropSortData()) {
- sortRef = merger->AllocSortRef(end - beg);
- sortItr = sortRef;
- }
-
- GlobalIdHasher seenGids(end - beg);
-
- FastS_make_heap(heap, heapSize, FastS_MergeCompare<T, F>);
-
- while ((pt < end) && (heapSize > 0)) {
- node = *heap;
- bool useHit = seenGids.insert(node->NT_GetHit()->HT_GetGlobalID());
- if (F::UseSortData()) {
- if (!F::DropSortData() && useHit) {
- FastS_MergeCopySortData<T>(node, sortItr++, sortDataLen);
- }
- node->NT_GetSortDataIterator()->Next();
- }
- if (useHit) {
- FastS_MergeCopyHit<T>(node->NT_GetHit(), pt++);
- }
- node->NT_NextHit();
- if (node->NT_GetNumHitsLeft() > 0) {
- FastS_pop_push_heap(heap, heapSize, node, FastS_MergeCompare<T, F>);
- } else {
- FastS_pop_heap(heap, heapSize--, FastS_MergeCompare<T, F>);
- }
- }
- if (pt != end) {
- LOG(warning, "Duplicate removal lead to %zd missing hits (wanted %zd, got %zd)",
- end - pt, end - beg, pt - beg);
- }
- merger->SetLastNode(node); // source of last hit
- if (F::UseSortData()) {
- FastS_assert(F::DropSortData() || sortItr == sortRef + (pt - beg));
- }
-
- // generate merged sort data
- if (F::UseSortData() && sortDataLen > 0) {
-
- FastS_assert(!F::DropSortData());
- search->ST_AllocSortData(sortDataLen);
-
- uint32_t offset = 0;
- uint32_t *sortIdx = search->ST_GetSortIndex();
- char *sortData = search->ST_GetSortData();
-
- sortItr = sortRef;
- for (uint32_t residue = (pt - beg); residue > 0; residue--) {
- *sortIdx++ = offset;
- memcpy(sortData + offset, sortItr->_buf, sortItr->_len);
- offset += sortItr->_len;
- sortItr++;
- }
- *sortIdx = offset;
- FastS_assert(sortItr == sortRef + (pt - beg));
- FastS_assert(offset == sortDataLen);
- }
- return (pt - beg);
-}
-
-//-----------------------------------------------------------------------------
-
-template <typename T>
-typename FastS_HitMerger<T>::NODE **
-FastS_HitMerger<T>::AllocHeap(uint32_t maxNodes)
-{
- FastS_assert(_heap == NULL);
- _heap = new NODE*[maxNodes];
- _heapSize = 0;
- _heapMax = maxNodes;
- return _heap;
-}
-
-
-template <typename T>
-SortData::Ref *
-FastS_HitMerger<T>::AllocSortRef(uint32_t size)
-{
- FastS_assert(_sortRef == NULL);
- _sortRef = new SortData::Ref[size];
- return _sortRef;
-}
-
-
-template <typename T>
-void
-FastS_HitMerger<T>::MergeHits()
-{
- uint32_t numNodes = _search->ST_GetNumNodes();
- bool dropSortData = _search->ST_ShouldDropSortData();
- bool useSortData = false;
- uint32_t numDocs = 0;
- uint64_t totalHits = 0;
- search::HitRank maxRank =
- std::numeric_limits<search::HitRank>::is_integer ?
- std::numeric_limits<search::HitRank>::min() :
- - std::numeric_limits<search::HitRank>::max();
- uint32_t sortDataDocs = 0;
-
- FastS_QueryResult *result = _search->ST_GetQueryResult();
-
- // just set totalHitCount for estimates
- if (_search->ST_IsEstimate()) {
- for (uint32_t i = 0; i < numNodes; i++) {
- _search->ST_GetNode(i)->NT_InitMerge(&numDocs, &totalHits,
- &maxRank, &sortDataDocs);
- }
- result->_totalHitCount = (_search->ST_GetEstParts() == 0) ? 0
- : (uint64_t) (((double)totalHits
- * (double)_search->ST_GetEstPartCutoff())
- / (double)_search->ST_GetEstParts());
- return;
- }
-
- // prepare nodes for merging
- NODE **heap = AllocHeap(numNodes);
- for (uint32_t i = 0; i < numNodes; i++) {
- if (_search->ST_GetNode(i)->NT_InitMerge(&numDocs, &totalHits,
- &maxRank, &sortDataDocs))
- {
- heap[_heapSize++] = _search->ST_GetNode(i);
- }
- }
-
- // check if we should use sort data for sorting
- if (sortDataDocs > 0) {
- if (sortDataDocs == numDocs) {
- useSortData = true;
- } else {
- LOG(warning, "Some results are missing sort data, sorting by rank instead");
- }
- }
-
- // set some result variables
- result->_totalHitCount = totalHits;
- result->_maxRank = maxRank;
-
- // allocate some needed structures
- _search->ST_SetNumHits(numDocs); // NB: allocs result buffer
-
- // do actual merging by invoking templated function
- if (useSortData) {
- if (dropSortData) {
- numDocs = FastS_InternalMergeHits
- <T, FastS_MergeFeatures<true, true> >(this);
- } else {
- numDocs = FastS_InternalMergeHits
- <T, FastS_MergeFeatures<true, false> >(this);
- }
- } else {
- numDocs = FastS_InternalMergeHits
- <T, FastS_MergeFeatures<false, false> >(this);
- }
- _search->ST_AdjustNumHits(numDocs);
-
- // detect incomplete/fuzzy results
- if (_search->ST_ShouldLimitHitsPerNode()) {
- if (_search->ST_GetAlignedHitCount() < _search->ST_GetAlignedMaxHits() &&
- result->_totalHitCount > (_search->ST_GetAlignedSearchOffset() +
- _search->ST_GetAlignedHitCount()))
- {
- _incomplete = true;
- }
-
- NODE *lastNode = GetLastNode();
- for (size_t i(0), m(_search->ST_GetNumNodes()); i < m; i++) {
- NODE *node(_search->ST_GetNode(i));
- if (node == lastNode ||
- node->NT_GetTotalHits() == 0)
- continue;
- if (node->NT_GetNumHitsLeft() == 0 &&
- node->NT_GetTotalHits() > (_search->ST_GetAlignedSearchOffset() +
- node->NT_GetNumHits()))
- {
- _fuzzy = true;
- break;
- }
- }
- }
-}
-
-//-----------------------------------------------------------------------------
-
-template class FastS_HitMerger<FastS_MergeHits_DummyMerge>; // for API check
-template class FastS_HitMerger<FastS_FNETMerge>;
-
-//-----------------------------------------------------------------------------
diff --git a/searchcore/src/vespa/searchcore/fdispatch/search/mergehits.h b/searchcore/src/vespa/searchcore/fdispatch/search/mergehits.h
deleted file mode 100644
index 306229b4730..00000000000
--- a/searchcore/src/vespa/searchcore/fdispatch/search/mergehits.h
+++ /dev/null
@@ -1,158 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-
-#pragma once
-
-#include <vespa/searchlib/common/sortdata.h>
-#include <vespa/searchlib/common/packets.h>
-#include <vespa/document/base/globalid.h>
-
-//-----------------------------------------------------------------------------
-
-class FastS_hitresult;
-class FastS_QueryResult;
-class FastS_FNET_Search;
-class FastS_FNET_SearchNode;
-
-// T::HitType API
-
-struct FastS_MergeHits_DummyHit
-{
- document::GlobalId _emptyGid;
- uint32_t HT_GetDocID() { return 0; }
- const document::GlobalId & HT_GetGlobalID() { return _emptyGid; }
- search::HitRank HT_GetMetric() { return 0; }
- uint32_t HT_GetPartID() { return 0; }
- uint32_t getDistributionKey() { return 0; }
- void HT_SetDocID(uint32_t val) { (void) val; }
- void HT_SetGlobalID(const document::GlobalId & val) { (void) val; }
- void HT_SetMetric(search::HitRank val) { (void) val; }
- void HT_SetPartID(uint32_t val) { (void) val; }
- void setDistributionKey(uint32_t val) { (void) val; }
-};
-
-// T::NodeType API
-
-struct FastS_MergeHits_DummyNode
-{
- bool NT_InitMerge(uint32_t *numDocs, uint64_t *totalHits,
- search::HitRank *maxRank, uint32_t *sortDataDocs)
- {
- (void) numDocs;
- (void) totalHits;
- (void) maxRank;
- (void) sortDataDocs;
- return false;
- }
- search::common::SortDataIterator *NT_GetSortDataIterator() { return NULL; }
- FastS_MergeHits_DummyHit *NT_GetHit() { return NULL; }
- uint32_t NT_GetNumHitsUsed() { return 0; }
- uint32_t NT_GetNumHitsLeft() { return 0; }
- uint64_t NT_GetTotalHits() { return 0; }
- uint32_t NT_GetNumHits() { return 0; }
- void NT_NextHit() { }
-};
-
-// T::SearchType API
-
-struct FastS_MergeHits_DummySearch
-{
- FastS_MergeHits_DummyNode *ST_GetNode(size_t i) { (void) i; return NULL; }
- uint32_t ST_GetNumNodes() { return 0; }
- bool ST_IsEstimate() { return false; }
- uint32_t ST_GetEstParts() { return 0; }
- uint32_t ST_GetEstPartCutoff() { return 0; }
- bool ST_ShouldDropSortData() { return false; }
- bool ST_ShouldLimitHitsPerNode() { return false; }
- void ST_SetNumHits(uint32_t numHits) { (void) numHits; }
- void ST_AdjustNumHits(uint32_t nH) { (void) nH; }
- uint32_t ST_GetAlignedSearchOffset() { return 0; }
- uint32_t ST_GetAlignedMaxHits() { return 0; }
- uint32_t ST_GetAlignedHitCount() { return 0; }
- FastS_hitresult *ST_GetAlignedHitBuf() { return NULL; }
- FastS_hitresult *ST_GetAlignedHitBufEnd() { return NULL; }
- void ST_AllocSortData(uint32_t len) { (void) len; }
- uint32_t *ST_GetSortIndex() { return NULL; }
- char *ST_GetSortData() { return NULL; }
- FastS_QueryResult *ST_GetQueryResult() { return NULL; }
-};
-
-// T (Merge Type) API
-
-struct FastS_MergeHits_DummyMerge
-{
- typedef FastS_MergeHits_DummyHit HitType;
- typedef FastS_MergeHits_DummyNode NodeType;
- typedef FastS_MergeHits_DummySearch SearchType;
-};
-
-//-----------------------------------------------------------------------------
-
-struct FastS_FNETMerge
-{
- typedef search::fs4transport::FS4Packet_QUERYRESULTX::FS4_hit HitType;
- typedef FastS_FNET_SearchNode NodeType;
- typedef FastS_FNET_Search SearchType;
-};
-
-//-----------------------------------------------------------------------------
-
-template <typename T>
-class FastS_HitMerger
-{
-private:
- FastS_HitMerger(const FastS_HitMerger &);
- FastS_HitMerger& operator=(const FastS_HitMerger &);
-
-
- typedef typename T::NodeType NODE;
- typedef typename T::SearchType SEARCH;
-
- // owning search object
- SEARCH *_search;
-
- // sorting heap
- NODE **_heap;
- uint32_t _heapSize;
- uint32_t _heapMax;
-
- // temporary array for merging sortdata
- search::common::SortData::Ref *_sortRef;
-
- // limit hits per node effect variables
- NODE *_lastNode;
- bool _incomplete;
- bool _fuzzy;
-
-public:
- FastS_HitMerger(SEARCH *search) : _search(search),
- _heap(NULL),
- _heapSize(0),
- _heapMax(0),
- _sortRef(NULL),
- _lastNode(NULL),
- _incomplete(false),
- _fuzzy(false)
- {}
-
- ~FastS_HitMerger()
- {
- delete [] _heap;
- delete [] _sortRef;
- }
-
- NODE **AllocHeap(uint32_t maxNodes);
- search::common::SortData::Ref *AllocSortRef(uint32_t size);
- void SetLastNode(NODE *lastNode) { _lastNode = lastNode; }
-
- SEARCH *GetSearch() const { return _search; }
- NODE **GetHeap() const { return _heap; }
- uint32_t GetHeapSize() const { return _heapSize; }
- uint32_t GetHeapMax() const { return _heapMax; }
- NODE *GetLastNode() const { return _lastNode; }
- bool WasIncomplete() const { return _incomplete; }
- bool WasFuzzy() const { return _fuzzy; }
-
- search::common::SortData::Ref *GetSortRef() const { return _sortRef; }
-
- void MergeHits();
-};
diff --git a/searchcore/src/vespa/searchcore/fdispatch/search/nodemanager.cpp b/searchcore/src/vespa/searchcore/fdispatch/search/nodemanager.cpp
deleted file mode 100644
index abe37011850..00000000000
--- a/searchcore/src/vespa/searchcore/fdispatch/search/nodemanager.cpp
+++ /dev/null
@@ -1,438 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-
-
-#include "nodemanager.h"
-#include "datasetcollection.h"
-#include "plain_dataset.h"
-#include "engine_base.h"
-#include <vespa/config/common/exceptions.h>
-#include <vespa/fastos/thread.h>
-#include <set>
-
-#include <vespa/log/log.h>
-LOG_SETUP(".search.nodemanager");
-
-void
-FastS_NodeManager::configure(std::unique_ptr<PartitionsConfig> cfg)
-{
- LOG(config, "configuring datasetcollection from '%s'",
- _configUri.getConfigId().c_str());
- SetPartMap(*cfg, _waitUpMS);
- _componentConfig.addConfig(
- vespalib::ComponentConfigProducer::Config("fdispatch.nodemanager",
- _fetcher->getGeneration(),
- "will not update generation unless config has changed"));
-}
-
-
-class AdminBadEngines
-{
- std::set<vespalib::string> _bad;
-public:
- void addAdminBad(const vespalib::string &name) {
- _bad.insert(name);
- }
-
- bool isAdminBad(const vespalib::string &name) const {
- return _bad.find(name) != _bad.end();
- }
-};
-
-class CollectAdminBadEngines
-{
- AdminBadEngines &_adminBadEngines;
-
-public:
-
- CollectAdminBadEngines(AdminBadEngines &adminBadEngines)
- : _adminBadEngines(adminBadEngines)
- {
- }
-
- void operator()(FastS_EngineBase* engine)
- {
- if (engine->isAdminBad()) {
- _adminBadEngines.addAdminBad(engine->GetName());
- }
- }
-};
-
-
-class PropagateAdminBadEngines
-{
- const AdminBadEngines &_adminBadEngines;
-
-public:
-
- PropagateAdminBadEngines(const AdminBadEngines &adminBadEngines)
- : _adminBadEngines(adminBadEngines)
- {
- }
-
- void operator()(FastS_EngineBase* engine)
- {
- if (_adminBadEngines.isAdminBad(engine->GetName())) {
- engine->MarkBad(FastS_EngineBase::BAD_ADMIN);
- }
- }
-};
-
-
-FastS_NodeManager::FastS_NodeManager(vespalib::SimpleComponentConfigProducer &componentConfig,
- FastS_AppContext *appCtx,
- uint32_t partition)
- : _componentConfig(componentConfig),
- _managerLock(),
- _configLock(),
- _appCtx(appCtx),
- _mldPartit(partition),
- _mldDocStamp(0),
- _mldDocStampMin(0),
- _gencnt(0),
- _fetcher(),
- _configUri(config::ConfigUri::createEmpty()),
- _lastPartMap(NULL),
- _datasetCollection(NULL),
- _oldDSCList(NULL),
- _tempFail(false),
- _failed(false),
- _hasDsc(false),
- _checkTempFailScheduled(false),
- _shutdown(false),
- _waitUpMS(20000)
-{
- _datasetCollection = new FastS_DataSetCollection(_appCtx);
- FastS_assert(_datasetCollection != NULL);
- _datasetCollection->Configure(NULL, 0);
- FastOS_Time now;
- now.SetNow();
- _mldDocStamp = now.GetSeconds();
- _mldDocStampMin = _mldDocStamp;
-}
-
-
-FastS_NodeManager::~FastS_NodeManager()
-{
- free(_lastPartMap);
- FastS_assert(_datasetCollection != NULL);
- _datasetCollection->subRef();
-}
-
-void
-FastS_NodeManager::CheckTempFail()
-{
- bool tempfail;
-
- _checkTempFailScheduled = false;
- tempfail = false;
- {
- std::lock_guard<std::mutex> mangerGuard(_managerLock);
- FastS_DataSetCollection *dsc = PeekDataSetCollection();
- for (unsigned int i = 0; i < dsc->GetMaxNumDataSets(); i++) {
- FastS_DataSetBase *ds;
- FastS_PlainDataSet *ds_plain;
- if ((ds = dsc->PeekDataSet(i)) != NULL &&
- (ds_plain = ds->GetPlainDataSet()) != NULL &&
- ds_plain->GetTempFail()) {
- tempfail = true;
- break;
- }
- }
- }
- _tempFail = tempfail;
-}
-
-void
-FastS_NodeManager::SubscribePartMap(const config::ConfigUri & configUri)
-{
- vespalib::string configId(configUri.getConfigId());
- LOG(debug, "loading new datasetcollection from %s", configId.c_str());
- try {
- _configUri = configUri;
- uint32_t oldWaitTime = _waitUpMS;
- _waitUpMS = 100;
- _fetcher.reset(new config::ConfigFetcher(_configUri.getContext()));
- _fetcher->subscribe<PartitionsConfig>(configId, this);
- _fetcher->start();
- _waitUpMS = oldWaitTime;
- if (_gencnt == 0) {
- throw new config::InvalidConfigException("failure during initial configuration: bad partition map");
- }
- } catch (std::exception &ex) {
- LOG(error, "Runtime exception: %s", (const char *) ex.what());
- EV_STOPPING("", "bad partitions config");
- exit(1);
- }
-}
-
-
-uint32_t
-FastS_NodeManager::SetPartMap(const PartitionsConfig& partmap,
- unsigned int waitms)
-{
- std::lock_guard<std::mutex> configGuard(_configLock);
- FastS_DataSetCollDesc *configDesc = new FastS_DataSetCollDesc();
- if (!configDesc->ReadConfig(partmap)) {
- LOG(error, "NodeManager::SetPartMap: Failed to load configuration");
- delete configDesc;
- return 0;
- }
- int retval = SetCollDesc(configDesc, waitms);
- return retval;
-}
-
-
-uint32_t
-FastS_NodeManager::SetCollDesc(FastS_DataSetCollDesc *configDesc,
- unsigned int waitms)
-{
- FastS_DataSetCollection *newCollection;
- uint32_t gencnt;
-
- if (_shutdown) return 0;
-
- AdminBadEngines adminBad;
-
- {
- CollectAdminBadEngines adminBadCollect(adminBad);
- FastS_DataSetCollection *dsc = GetDataSetCollection();
- for (uint32_t i = 0; i < dsc->GetMaxNumDataSets(); i++) {
- FastS_DataSetBase *ds;
- FastS_PlainDataSet *ds_plain;
- if ((ds = dsc->PeekDataSet(i)) == NULL ||
- (ds_plain = ds->GetPlainDataSet()) == NULL)
- continue;
-
- ds_plain->ForEachEngine(adminBadCollect);
- }
- dsc->subRef();
- }
-
-
- newCollection = new FastS_DataSetCollection(_appCtx);
- if (!newCollection->Configure(configDesc, _gencnt + 1)) {
- LOG(error, "NodeManager::SetPartMap: Inconsistent configuration");
- newCollection->subRef();
- return 0;
- }
-
- {
- PropagateAdminBadEngines adminBadPropagate(adminBad);
- for (uint32_t i = 0; i < newCollection->GetMaxNumDataSets(); i++) {
- FastS_DataSetBase *ds;
- FastS_PlainDataSet *ds_plain;
- if ((ds = newCollection->PeekDataSet(i)) == NULL ||
- (ds_plain = ds->GetPlainDataSet()) == NULL)
- continue;
-
- ds_plain->ForEachEngine(adminBadPropagate);
- }
- }
-
- if (waitms > 0) {
- FastOS_Time last;
- unsigned int rwait;
- bool allup;
- last.SetNow();
- while (1) {
- allup = newCollection->AreEnginesReady();
- rwait = (unsigned int) last.MilliSecsToNow();
- if (rwait >= waitms || allup)
- break;
- FastOS_Thread::Sleep(10);
- };
- if (allup) {
- LOG(debug, "All new engines up after %d ms", rwait);
- } else {
- LOG(debug, "Some new engines still down after %d ms", rwait);
- }
- }
-
- gencnt = SetDataSetCollection(newCollection);
-
- ScheduleCheckTempFail(FastS_NoID32());
- return gencnt;
-}
-
-
-
-/**
- * When calling this method, a single reference on the 'dsc' parameter
- * is passed to the monitor object.
- *
- * @return generation count, or 0 on fail.
- * @param dsc new dataset collection. A single reference is passed
- * to the monitor when this method is invoked.
- **/
-uint32_t
-FastS_NodeManager::SetDataSetCollection(FastS_DataSetCollection *dsc)
-{
- if (dsc == NULL)
- return 0;
-
- uint32_t gencnt = 0;
- FastS_DataSetCollection *old_dsc = NULL;
-
- if (!dsc->IsValid()) {
- LOG(error, "NodeManager::SetDataSetCollection: Inconsistent configuration");
- dsc->subRef();
-
- } else {
- {
- std::lock_guard<std::mutex> managerGuard(_managerLock);
- _gencnt++;
- gencnt = _gencnt;
-
- old_dsc = _datasetCollection;
- _datasetCollection = dsc;
-
- // put old config on service list
- FastS_assert(old_dsc != NULL);
- if (!old_dsc->IsLastRef()) {
- old_dsc->_nextOld = _oldDSCList;
- _oldDSCList = old_dsc;
- old_dsc = NULL;
- }
- _hasDsc = true;
- }
-
- if (old_dsc != NULL)
- old_dsc->subRef();
- }
- return gencnt;
-}
-
-
-FastS_DataSetCollection *
-FastS_NodeManager::GetDataSetCollection()
-{
- FastS_DataSetCollection *ret;
-
- std::lock_guard<std::mutex> managerGuard(_managerLock);
- ret = _datasetCollection;
- FastS_assert(ret != NULL);
- ret->addRef();
-
- return ret;
-}
-
-
-void
-FastS_NodeManager::ShutdownConfig()
-{
- FastS_DataSetCollection *dsc;
- FastS_DataSetCollection *old_dsc;
-
- {
- std::lock_guard<std::mutex> configGuard(_configLock);
- std::lock_guard<std::mutex> managerGuard(_managerLock);
- _shutdown = true; // disallow SetPartMap
- dsc = _datasetCollection;
- _datasetCollection = new FastS_DataSetCollection(_appCtx);
- _datasetCollection->Configure(NULL, 0);
- old_dsc = _oldDSCList;
- _oldDSCList = NULL;
- }
- dsc->AbortQueryQueues();
- dsc->subRef();
- while (old_dsc != NULL) {
- dsc = old_dsc;
- old_dsc = old_dsc->_nextOld;
- dsc->_nextOld = NULL;
- dsc->AbortQueryQueues();
- dsc->subRef();
- }
-}
-
-ChildInfo
-FastS_NodeManager::getChildInfo()
-{
- ChildInfo r;
- r.activeDocs.valid = true;
- FastS_DataSetCollection *dsc = GetDataSetCollection();
-
- for (unsigned int i = 0; i < dsc->GetMaxNumDataSets(); i++) {
- FastS_DataSetBase *ds;
- FastS_PlainDataSet *ds_plain;
- if ((ds = dsc->PeekDataSet(i)) == NULL ||
- (ds_plain = ds->GetPlainDataSet()) == NULL)
- continue;
- r.maxNodes += ds_plain->_partMap._childmaxnodesSinceReload;
- r.activeNodes += ds_plain->_partMap._childnodes;
- r.maxParts += ds_plain->_partMap._childmaxpartsSinceReload;
- r.activeParts += ds_plain->_partMap._childparts;
- PossCount rowActive = ds_plain->getActiveDocs();
- if (rowActive.valid) {
- r.activeDocs.count += rowActive.count;
- } else {
- r.activeDocs.valid = false;
- }
- }
-
- dsc->subRef();
- return r;
-}
-
-void
-FastS_NodeManager::CheckEvents(FastS_TimeKeeper *timeKeeper)
-{
- // CHECK SCHEDULED OPERATIONS
-
- if (_checkTempFailScheduled)
- CheckTempFail();
-
- // CHECK QUERY QUEUES
-
- FastS_DataSetCollection *dsc = GetDataSetCollection();
-
- dsc->CheckQueryQueues(timeKeeper);
- dsc->subRef();
-
- // check old query queues and discard old configs
-
- FastS_DataSetCollection *old_dsc;
- FastS_DataSetCollection *prev = NULL;
- FastS_DataSetCollection *tmp;
-
- {
- std::lock_guard<std::mutex> managerGuard(_managerLock);
- old_dsc = _oldDSCList;
- }
-
- while (old_dsc != NULL) {
- if (old_dsc->IsLastRef()) {
- if (prev == NULL) {
- std::unique_lock<std::mutex> managerGuard(_managerLock);
- if (_oldDSCList == old_dsc) {
- _oldDSCList = old_dsc->_nextOld;
- } else {
- prev = _oldDSCList;
- managerGuard.unlock();
- while (prev->_nextOld != old_dsc)
- prev = prev->_nextOld;
-
- prev->_nextOld = old_dsc->_nextOld;
- }
- } else {
- prev->_nextOld = old_dsc->_nextOld;
- }
- tmp = old_dsc;
- old_dsc = old_dsc->_nextOld;
- tmp->subRef();
-
- } else {
-
- old_dsc->CheckQueryQueues(timeKeeper);
- prev = old_dsc;
- old_dsc = old_dsc->_nextOld;
- }
- }
-}
-
-uint32_t
-FastS_NodeManager::GetMldDocstamp()
-{
- if (!_hasDsc)
- return 0;
- return _mldDocStamp;
-}
diff --git a/searchcore/src/vespa/searchcore/fdispatch/search/nodemanager.h b/searchcore/src/vespa/searchcore/fdispatch/search/nodemanager.h
deleted file mode 100644
index 21180014995..00000000000
--- a/searchcore/src/vespa/searchcore/fdispatch/search/nodemanager.h
+++ /dev/null
@@ -1,89 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-
-#pragma once
-
-#include "child_info.h"
-#include "configdesc.h"
-#include <vespa/config/helper/configfetcher.h>
-#include <vespa/vespalib/net/simple_component_config_producer.h>
-#include <vespa/config/subscription/configuri.h>
-#include <vespa/vespalib/util/executor.h>
-#include <mutex>
-
-using vespa::config::search::core::PartitionsConfig;
-
-class FastS_DataSetBase;
-class FastS_AppContext;
-class FastS_DataSetCollection;
-class FastS_TimeKeeper;
-
-class FastS_NodeManager : public config::IFetcherCallback<PartitionsConfig>
-{
-private:
- FastS_NodeManager(const FastS_NodeManager &);
- FastS_NodeManager& operator=(const FastS_NodeManager &);
-
- vespalib::SimpleComponentConfigProducer &_componentConfig;
-
- std::mutex _managerLock;
- std::mutex _configLock;
- FastS_AppContext *_appCtx;
- uint32_t _mldPartit;
- uint32_t _mldDocStamp; // Bumped for all cache flushes
- uint32_t _mldDocStampMin; // Bumped for global cache flush
- uint32_t _gencnt;
-
-
-
- std::unique_ptr<config::ConfigFetcher> _fetcher;
- config::ConfigUri _configUri;
-
- char *_lastPartMap;
- FastS_DataSetCollection *_datasetCollection; // current node config
- FastS_DataSetCollection *_oldDSCList; // list of old node configs
-
- bool _tempFail;
- bool _failed;
- bool _hasDsc;
-
- volatile bool _checkTempFailScheduled;
- volatile bool _shutdown;
- volatile uint32_t _waitUpMS;
-
-protected:
-
- void configure(std::unique_ptr<PartitionsConfig> cfg) override;
-
-public:
- FastS_NodeManager(vespalib::SimpleComponentConfigProducer &componentConfig,
- FastS_AppContext *appCtx,
- uint32_t partition);
- ~FastS_NodeManager();
-
- void SubscribePartMap(const config::ConfigUri & configUri);
-
- uint32_t GetMldPartition() const { return _mldPartit; }
- uint32_t GetMldDocstamp();
-
- bool Failed() const { return _failed; }
- bool GetTempFail() const { return _tempFail; }
-
- void ScheduleCheckTempFail(uint32_t datasetid) {
- (void) datasetid;
- _checkTempFailScheduled = true;
- }
-
- FastS_DataSetCollection *PeekDataSetCollection()
- { return _datasetCollection; }
-
- void CheckTempFail();
- uint32_t SetPartMap(const PartitionsConfig& partmap, unsigned int waitms);
- uint32_t SetCollDesc(FastS_DataSetCollDesc *configDesc, unsigned int waitms);
- uint32_t SetDataSetCollection(FastS_DataSetCollection *dsc);
- FastS_DataSetCollection *GetDataSetCollection();
- ChildInfo getChildInfo();
- void ShutdownConfig();
-
- void CheckEvents(FastS_TimeKeeper *timeKeeper); // invoked by FNET thread
-};
-
diff --git a/searchcore/src/vespa/searchcore/fdispatch/search/plain_dataset.cpp b/searchcore/src/vespa/searchcore/fdispatch/search/plain_dataset.cpp
deleted file mode 100644
index cd68b208ff4..00000000000
--- a/searchcore/src/vespa/searchcore/fdispatch/search/plain_dataset.cpp
+++ /dev/null
@@ -1,560 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-
-#include "plain_dataset.h"
-#include "datasetcollection.h"
-#include "engine_base.h"
-#include "nodemanager.h"
-#include <vespa/searchcore/fdispatch/common/search.h>
-#include <vespa/vespalib/util/host_name.h>
-#include <iomanip>
-
-#include <vespa/log/log.h>
-LOG_SETUP(".search.plain_dataset");
-
-//--------------------------------------------------------------------------
-
-static inline int imax(int a, int b) { return (a > b) ? a : b; }
-
-//--------------------------------------------------------------------------
-
-FastS_PartitionMap::Partition::Partition()
- : _engines(nullptr),
- _maxnodesNow(0),
- _maxnodesSinceReload(0),
- _nodes(0),
- _maxpartsNow(0),
- _maxpartsSinceReload(0),
- _parts(0)
-{
-}
-
-
-FastS_PartitionMap::Partition::~Partition()
-{
- FastS_assert(_engines == nullptr);
- FastS_assert(_nodes == 0);
- FastS_assert(_parts == 0);
-}
-
-//--------------------------------------------------------------------------
-
-FastS_PartitionMap::FastS_PartitionMap(FastS_DataSetDesc *desc)
- : _partitions(nullptr),
- _partBits(desc->GetPartBits()),
- _rowBits(desc->GetRowBits()),
- _num_partitions(desc->GetNumParts()),
- _first_partition(desc->GetFirstPart()),
- _minchildparts(desc->GetMinChildParts()),
- _maxNodesDownPerFixedRow(desc->getMaxNodesDownPerFixedRow()),
- _useRoundRobinForFixedRow(desc->useRoundRobinForFixedRow()),
- _childnodes(0),
- _childmaxnodesNow(0),
- _childmaxnodesSinceReload(0),
- _childparts(0),
- _childmaxpartsNow(0),
- _childmaxpartsSinceReload(0),
- _mpp(desc->getMPP()),
- _maxRows(0)
-{
- // finalize config settings
- if (_num_partitions > (1U << _partBits)) {
- LOG(error, "Too many partitions %d constrained by partbits %d", _num_partitions, _partBits);
- _num_partitions = (1U << _partBits);
- }
-
- if (_num_partitions > 0) {
- _partitions = new Partition[_num_partitions];
- FastS_assert(_partitions != nullptr);
- }
- for (FastS_EngineDesc *curr = desc->GetEngineList(); curr != nullptr; curr = curr->GetNext()) {
- _maxRows = std::max(_maxRows, curr->GetConfRowID());
- }
- _numPartitions = std::vector<uint32_t>(getNumRows(), 0);
- for (FastS_EngineDesc *curr = desc->GetEngineList(); curr != nullptr; curr = curr->GetNext()) {
- size_t rowId(curr->GetConfRowID());
- _numPartitions[rowId] = std::max(_numPartitions[rowId], curr->GetConfPartID()+1);
- }
-}
-
-
-FastS_PartitionMap::~FastS_PartitionMap()
-{
- delete [] _partitions;
-}
-
-
-void
-FastS_PartitionMap::RecalcPartCnt(uint32_t partid)
-{
- uint32_t maxparts = 0;
- uint32_t parts = 0;
- for (FastS_EngineBase * engine = _partitions[partid]._engines;
- engine != nullptr; engine = engine->_nextpart) {
- maxparts = imax(engine->_reported._maxParts, maxparts);
- parts = imax(engine->_reported._actParts, parts);
- }
- if (_partitions[partid]._maxpartsNow != maxparts) {
- _childmaxpartsNow += maxparts - _partitions[partid]._maxpartsNow;
- _partitions[partid]._maxpartsNow = maxparts;
- if (_childmaxpartsNow > _childmaxpartsSinceReload)
- _childmaxpartsSinceReload = _childmaxpartsNow;
- }
- if (_partitions[partid]._parts != parts) {
- _childparts += parts - _partitions[partid]._parts;
- _partitions[partid]._parts = parts;
- }
-}
-
-
-void
-FastS_PartitionMap::LinkIn(FastS_EngineBase *engine)
-{
- uint32_t partid = engine->_partid - _first_partition;
-
- FastS_assert(partid < GetSize());
- FastS_assert(engine->_nextpart == nullptr);
- FastS_assert(engine->_prevpart == nullptr);
- FastS_PartitionMap::Partition & part = _partitions[partid];
- engine->_nextpart = part._engines;
- if (engine->_nextpart != nullptr)
- engine->_nextpart->_prevpart = engine;
- part._engines = engine;
- part._maxnodesNow += engine->_reported._maxNodes;
- part._maxnodesSinceReload = std::max(part._maxnodesSinceReload, part._maxnodesNow);
- part._nodes += engine->_reported._actNodes;
- _childmaxnodesNow += engine->_reported._maxNodes;
- _childmaxnodesSinceReload = std::max(_childmaxnodesSinceReload, _childmaxnodesNow);
- _childnodes += engine->_reported._actNodes;
- if (part._maxpartsNow <= engine->_reported._maxParts) {
- _childmaxpartsNow += engine->_reported._maxParts - part._maxpartsNow;
- _childmaxpartsSinceReload += std::max(_childmaxpartsSinceReload, _childmaxpartsNow);
- part._maxpartsNow = engine->_reported._maxParts;
- }
- if (part._parts < engine->_reported._actParts) {
- _childparts += engine->_reported._actParts - part._parts;
- part._parts = engine->_reported._actParts;
- }
-}
-
-
-void
-FastS_PartitionMap::LinkOut(FastS_EngineBase *engine)
-{
- uint32_t partid = engine->_partid - _first_partition;
-
- FastS_assert(partid < GetSize());
- if (engine->_nextpart != nullptr)
- engine->_nextpart->_prevpart = engine->_prevpart;
- if (engine->_prevpart != nullptr)
- engine->_prevpart->_nextpart = engine->_nextpart;
- if (_partitions[partid]._engines == engine)
- _partitions[partid]._engines = engine->_nextpart;
-
- _partitions[partid]._maxnodesNow -= engine->_reported._maxNodes;
- _partitions[partid]._nodes -= engine->_reported._actNodes;
- _childmaxnodesNow -= engine->_reported._maxNodes;
- _childnodes -= engine->_reported._actNodes;
- if (_partitions[partid]._maxpartsNow <= engine->_reported._maxParts ||
- _partitions[partid]._parts <= engine->_reported._actParts)
- RecalcPartCnt(partid);
-
- engine->_nextpart = nullptr;
- engine->_prevpart = nullptr;
-}
-
-//--------------------------------------------------------------------------
-
-FastS_PlainDataSet::MHPN_log_t::MHPN_log_t()
- : _cnt(0),
- _incompleteCnt(0),
- _fuzzyCnt(0)
-{
-}
-
-//--------------------------------------------------------------------------
-
-void
-FastS_PlainDataSet::InsertEngine(FastS_EngineBase *engine)
-{
- _enginesArray.push_back(engine);
-}
-
-FastS_EngineBase *
-FastS_PlainDataSet::ExtractEngine()
-{
- if (_enginesArray.size() > 0) {
- FastS_EngineBase *ret = _enginesArray.back();
- _enginesArray.pop_back();
- return ret;
- } else {
- return nullptr;
- }
-}
-
-FastS_PlainDataSet::FastS_PlainDataSet(FastS_AppContext *appCtx,
- FastS_DataSetDesc *desc)
- : FastS_DataSetBase(appCtx, desc),
- _partMap(desc),
- _stateOfRows(_partMap.getNumRows(), 0.001, desc->GetQueryDistributionMode().getLatencyDecayRate()),
- _MHPN_log(),
- _slowQueryLimitFactor(desc->GetSlowQueryLimitFactor()),
- _slowQueryLimitBias(desc->GetSlowQueryLimitBias()),
- _slowDocsumLimitFactor(desc->GetSlowDocsumLimitFactor()),
- _slowDocsumLimitBias(desc->GetSlowDocsumLimitBias()),
- _monitorInterval(desc->getMonitorInterval()),
- _higherCoverageMaxSearchWait(desc->getHigherCoverageMaxSearchWait()),
- _higherCoverageMinSearchWait(desc->getHigherCoverageMinSearchWait()),
- _higherCoverageBaseSearchWait(desc->getHigherCoverageBaseSearchWait()),
- _minimalSearchCoverage(desc->getMinimalSearchCoverage()),
- _higherCoverageMaxDocSumWait(desc->getHigherCoverageMaxDocSumWait()),
- _higherCoverageMinDocSumWait(desc->getHigherCoverageMinDocSumWait()),
- _higherCoverageBaseDocSumWait(desc->getHigherCoverageBaseDocSumWait()),
- _minimalDocSumCoverage(desc->getMinimalDocSumCoverage()),
- _maxHitsPerNode(desc->GetMaxHitsPerNode()),
- _estimateParts(desc->GetEstimateParts()),
- _estimatePartCutoff(desc->GetEstPartCutoff()),
- _queryDistributionMode(desc->GetQueryDistributionMode()),
- _randState()
-{
- uint32_t seed = 0;
- const char *hostname = vespalib::HostName::get().c_str();
- unsigned const char *p = reinterpret_cast<unsigned const char *>(hostname);
-
- if (p != nullptr) {
- while (*p != '\0') {
- seed = (seed << 7) + *p + (seed >> 25);
- p++;
- }
- }
- seed ^= _createtime.GetSeconds();
- seed ^= _createtime.GetMicroSeconds();
- _randState.srand48(seed);
-}
-
-
-FastS_PlainDataSet::~FastS_PlainDataSet() = default;
-
-void
-FastS_PlainDataSet::UpdateMaxHitsPerNodeLog(bool incomplete, bool fuzzy)
-{
- auto dsGuard(getDsGuard());
- _MHPN_log._cnt++;
- if (incomplete)
- _MHPN_log._incompleteCnt++;
- if (fuzzy)
- _MHPN_log._fuzzyCnt++;
-}
-
-
-bool
-FastS_PlainDataSet::RefCostUseNewEngine(FastS_EngineBase *oldEngine,
- FastS_EngineBase *newEngine,
- unsigned int *oldCount)
-{
- if (oldEngine->_totalrefcost + oldEngine->_config._unitrefcost >
- newEngine->_totalrefcost + newEngine->_config._unitrefcost) {
- *oldCount = 1;
- return true;
- }
- if (oldEngine->_totalrefcost + oldEngine->_config._unitrefcost <
- newEngine->_totalrefcost + newEngine->_config._unitrefcost)
- return false;
- /* Use random generator for tie breaker */
- (*oldCount)++;
- return ((_randState.lrand48() % *oldCount) == 0);
-}
-
-void
-FastS_PlainDataSet::updateSearchTime(double searchTime, uint32_t rowId)
-{
- auto dsGuard(getDsGuard());
- _stateOfRows.updateSearchTime(searchTime, rowId);
-}
-
-uint32_t
-FastS_PlainDataSet::getRandomWeightedRow() const
-{
- return _stateOfRows.getRandomWeightedRow();
-}
-
-
-bool
-FastS_PlainDataSet::UseNewEngine(FastS_EngineBase *oldEngine,
- FastS_EngineBase *newEngine,
- unsigned int *oldCount)
-{
- /*
- * If old engine has used _indexSwitchMinSearchGrace seconds
- * of grace period then select new engine if it has used
- * less grace period.
- */
- if (!EngineDocStampOK(oldEngine->_reported._docstamp) &&
- (EngineDocStampOK(newEngine->_reported._docstamp)))
- {
- *oldCount = 1;
- return true;
- }
-
- /*
- * If new engine has used _indexSwitchMinSearchGrace seconds
- * of grace period then select old engine if it has used
- * less grace period.
- */
- if (!EngineDocStampOK(newEngine->_reported._docstamp) &&
- (EngineDocStampOK(oldEngine->_reported._docstamp)))
- {
- return false;
- }
-
- return RefCostUseNewEngine(oldEngine, newEngine, oldCount);
-}
-
-FastS_EngineBase *
-FastS_PlainDataSet::getPartition(const std::unique_lock<std::mutex> &dsGuard, uint32_t partindex, uint32_t rowid)
-{
- (void) dsGuard;
- FastS_EngineBase* ret = nullptr;
-
- if (IsValidPartIndex_HasLock(partindex)) {
- for (FastS_EngineBase* iter = _partMap._partitions[partindex]._engines;
- iter != nullptr && ret == nullptr;
- iter = iter->_nextpart) {
-
- // NB: cost race condition
-
- if (!iter->IsRealBad() &&
- EngineDocStampOK(iter->_reported._docstamp) &&
- iter->_config._confRowID == rowid) {
- ret = iter;
- }
- }
- }
-
- if (ret != nullptr) {
- ret->AddCost();
- }
- return ret;
-}
-
-size_t
-FastS_PlainDataSet::countNodesUpInRow_HasLock(uint32_t rowid)
-{
- size_t count = 0;
- const size_t numParts = _partMap.GetSize();
- for (size_t partindex = 0; partindex < numParts; ++partindex) {
- for (FastS_EngineBase* iter = _partMap._partitions[partindex]._engines;
- iter != nullptr;
- iter = iter->_nextpart)
- {
- if (!iter->IsRealBad() &&
- EngineDocStampOK(iter->_reported._docstamp) &&
- iter->_config._confRowID == rowid)
- {
- ++count;
- break;
- }
- }
- }
- return count;
-}
-
-FastS_EngineBase *
-FastS_PlainDataSet::getPartition(const std::unique_lock<std::mutex> &dsGuard, uint32_t partindex)
-{
- (void) dsGuard;
- FastS_EngineBase* ret = nullptr;
- unsigned int oldCount = 1;
- unsigned int engineCount = 0;
-
- if (IsValidPartIndex_HasLock(partindex)) {
- for (FastS_EngineBase* iter = _partMap._partitions[partindex]._engines;
- iter != nullptr;
- iter = iter->_nextpart) {
-
- // NB: cost race condition
-
- if (!iter->IsRealBad() &&
- (iter->_config._unitrefcost > 0) &&
- EngineDocStampOK(iter->_reported._docstamp))
- {
- engineCount++;
- if (ret == nullptr || UseNewEngine(ret, iter, &oldCount))
- ret = iter;
- }
- }
- }
-
- if (engineCount < getMPP()) {
- ret = nullptr;
- }
- if (ret != nullptr) {
- ret->AddCost();
- }
- return ret;
-}
-
-FastS_EngineBase *
-FastS_PlainDataSet::getPartitionMLD(const std::unique_lock<std::mutex> &dsGuard, uint32_t partindex, bool mld)
-{
- (void) dsGuard;
- FastS_EngineBase* ret = nullptr;
- unsigned int oldCount = 1;
- if (partindex < _partMap._num_partitions) {
- FastS_EngineBase* iter;
- for (iter = _partMap._partitions[partindex]._engines; iter != nullptr; iter = iter->_nextpart) {
- // NB: cost race condition
-
- if (!iter->IsRealBad() &&
- iter->_reported._mld == mld &&
- (iter->_config._unitrefcost > 0) &&
- EngineDocStampOK(iter->_reported._docstamp) &&
- (ret == nullptr || UseNewEngine(ret, iter, &oldCount)))
- {
- ret = iter;
- }
- }
- } else {
- LOG(error, "Couldn't fetch partition data: Partition ID too big, partindex=%x _partMap._num_partitions=%x", partindex, _partMap._num_partitions);
- }
- if (ret != nullptr) {
- ret->AddCost();
- }
- return ret;
-}
-
-FastS_EngineBase *
-FastS_PlainDataSet::getPartitionMLD(const std::unique_lock<std::mutex> &dsGuard, uint32_t partindex, bool mld, uint32_t rowid)
-{
- (void) dsGuard;
- FastS_EngineBase* ret = nullptr;
- unsigned int oldCount = 1;
-
- if (partindex < _partMap._num_partitions) {
- FastS_EngineBase* iter;
- for (iter = _partMap._partitions[partindex]._engines; iter != nullptr; iter = iter->_nextpart) {
- // NB: cost race condition
- if (!iter->IsRealBad() &&
- (iter->_reported._mld == mld) &&
- (iter->_config._confRowID == rowid) &&
- EngineDocStampOK(iter->_reported._docstamp) &&
- (ret == nullptr || UseNewEngine(ret, iter, &oldCount)))
- {
- ret = iter;
- }
- }
- } else {
- LOG(error, "Couldn't fetch partition data: Partition ID too big, partindex=%x _partMap._num_partitions=%x", partindex, _partMap._num_partitions);
- }
- if (ret != nullptr) {
- ret->AddCost();
- }
- return ret;
-}
-
-void
-FastS_PlainDataSet::LinkInPart_HasLock(FastS_EngineBase *engine)
-{
- if (engine->GetPartID() == FastS_NoID32())
- return;
-
- _partMap.LinkIn(engine);
-}
-
-
-void
-FastS_PlainDataSet::LinkOutPart_HasLock(FastS_EngineBase *engine)
-{
- if (engine->GetPartID() == FastS_NoID32())
- return;
-
- _partMap.LinkOut(engine);
-}
-
-
-uint32_t
-FastS_PlainDataSet::CalculateQueueLens_HasLock(uint32_t &dispatchnodes)
-{
- uint32_t partindex;
- uint32_t equeueLen;
- uint32_t pqueueLen;
- FastS_EngineBase *eng;
- uint32_t pdispatchnodes;
- uint32_t dupnodes;
-
- uint32_t queueLen = 0;
- dispatchnodes = 1;
- for (partindex = 0; partindex < _partMap._num_partitions ; partindex++) {
- eng = _partMap._partitions[partindex]._engines;
- if (eng != nullptr) {
- pqueueLen = eng->GetQueueLen();
- pdispatchnodes = eng->GetDispatchers();
- dupnodes = 1;
- eng = eng->_nextpart;
- while (eng != nullptr) {
- equeueLen = eng->GetQueueLen();
- if (equeueLen < pqueueLen)
- pqueueLen = equeueLen;
- pdispatchnodes += eng->GetDispatchers();
- dupnodes++;
- eng = eng->_nextpart;
- }
- if (pqueueLen > queueLen)
- queueLen = pqueueLen;
- if (dispatchnodes * dupnodes < pdispatchnodes)
- dispatchnodes = pdispatchnodes / dupnodes;
- }
- }
- return queueLen;
-}
-
-namespace {
-struct CheckReady {
- bool allReady;
- CheckReady() : allReady(true) {}
-
- inline void operator()(FastS_EngineBase* engine) {
- allReady &= engine->IsReady();
- }
-};
-
-} //anonymous namespace
-
-bool
-FastS_PlainDataSet::AreEnginesReady()
-{
-
- // We don't need to lock things here, since the engine list
- // is non-mutable during datasetcollection lifetime.
- return ForEachEngine( CheckReady() ).allReady;
-}
-
-void
-FastS_PlainDataSet::Ping()
-{
- for (FastS_EngineBase* engine : _enginesArray) {
- engine->Ping();
- }
-}
-
-
-ChildInfo
-FastS_PlainDataSet::getChildInfo() const
-{
- ChildInfo r;
- r.maxNodes = _partMap._childmaxnodesSinceReload;
- r.activeNodes = _partMap._childnodes;
- r.maxParts = _partMap._childmaxpartsSinceReload;
- r.activeParts = _partMap._childparts;
- r.activeDocs = getActiveDocs();
- return r;
-}
-
-bool
-FastS_PlainDataSet::IsValidPartIndex_HasLock(uint32_t partindex) {
- if (partindex < _partMap._num_partitions) {
- return true;
- } else {
- LOG(error, "Couldn't fetch partition data: Partition ID too big, partindex=%x _partMap._num_partitions=%x", partindex, _partMap._num_partitions);
- return false;
- }
-}
diff --git a/searchcore/src/vespa/searchcore/fdispatch/search/plain_dataset.h b/searchcore/src/vespa/searchcore/fdispatch/search/plain_dataset.h
deleted file mode 100644
index 8ce6b3adb75..00000000000
--- a/searchcore/src/vespa/searchcore/fdispatch/search/plain_dataset.h
+++ /dev/null
@@ -1,216 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-
-#pragma once
-
-#include <list>
-
-#include "child_info.h"
-#include <vespa/searchcore/fdispatch/search/dataset_base.h>
-#include <vespa/searchlib/util/rand48.h>
-#include <vespa/searchcore/fdispatch/search/configdesc.h>
-#include <vespa/searchcore/fdispatch/search/rowstate.h>
-#include <vespa/fnet/task.h>
-
-class FastS_EngineBase;
-
-//----------------------------------------------------------------
-// class holding information about a set of partitions
-//----------------------------------------------------------------
-class FastS_PartitionMap
-{
-public:
-
- //----------------------------------------------------------------
- // class holding information about a single partition
- //----------------------------------------------------------------
- class Partition
- {
-
- public:
- FastS_EngineBase *_engines;
- uint32_t _maxnodesNow;
- uint32_t _maxnodesSinceReload;
- uint32_t _nodes;
- uint32_t _maxpartsNow;
- uint32_t _maxpartsSinceReload;
- uint32_t _parts;
-
- public:
- Partition();
- ~Partition();
- private:
- Partition(const Partition &);
- Partition& operator=(const Partition &);
- };
-
-
-public:
- Partition *_partitions;
- uint32_t _partBits;
- uint32_t _rowBits;
- uint32_t _num_partitions; // Number of partitions (active)
- uint32_t _first_partition; // From partitions-file 'firstpart' (active)
- uint32_t _minchildparts; // Minimum partitions live to avoid tempfail
- uint32_t _maxNodesDownPerFixedRow;
- bool _useRoundRobinForFixedRow;
- uint32_t _childnodes;
- uint32_t _childmaxnodesNow;
- uint32_t _childmaxnodesSinceReload;
- uint32_t _childparts;
- uint32_t _childmaxpartsNow;
- uint32_t _childmaxpartsSinceReload;
- uint32_t _mpp; // Number of engines needed per partition
-
- std::vector<uint32_t> _numPartitions;
-
-public:
- FastS_PartitionMap(FastS_DataSetDesc *desc);
- ~FastS_PartitionMap();
-
- void RecalcPartCnt(uint32_t partid);
- void LinkIn(FastS_EngineBase *engine);
- void LinkOut(FastS_EngineBase *engine);
-
- uint32_t GetSize() { return _num_partitions; }
-
- uint32_t getNumRows() const { return _maxRows + 1; }
- uint32_t getNumPartitions(size_t rowId) { return _numPartitions[rowId]; }
-private:
- FastS_PartitionMap(const FastS_PartitionMap &);
- FastS_PartitionMap& operator=(const FastS_PartitionMap &);
- uint32_t _maxRows;
-
-};
-
-//---------------------------------------------------------------------------
-
-class FastS_PlainDataSet : public FastS_DataSetBase
-{
- friend class FastS_NodeManager;
-
-public:
-
- //----------------------------------------------------------------
- // Max Hits Per Node Stats
- //----------------------------------------------------------------
-
- class MHPN_log_t
- {
- public:
- uint32_t _cnt; // # times maxHitsPerNode affected # hits requested
- uint32_t _incompleteCnt; // # times maxHitsPerNode caused too few hits
- uint32_t _fuzzyCnt; // # times maxHitsPerNode may have caused wrong hits
-
- MHPN_log_t();
- };
-
-protected:
- FastS_PartitionMap _partMap;
- fdispatch::StateOfRows _stateOfRows;
- MHPN_log_t _MHPN_log;
- double _slowQueryLimitFactor;
- double _slowQueryLimitBias;
- double _slowDocsumLimitFactor;
- double _slowDocsumLimitBias;
- double _monitorInterval;
- double _higherCoverageMaxSearchWait;
- double _higherCoverageMinSearchWait;
- double _higherCoverageBaseSearchWait;
- double _minimalSearchCoverage;
- double _higherCoverageMaxDocSumWait;
- double _higherCoverageMinDocSumWait;
- double _higherCoverageBaseDocSumWait;
- double _minimalDocSumCoverage;
- uint32_t _maxHitsPerNode; // Max hits requested from single node
- uint32_t _estimateParts; // number of partitions used for estimate
- uint32_t _estimatePartCutoff; // First partition not used for estimate
-
- FastS_DataSetDesc::QueryDistributionMode _queryDistributionMode;
- //all engines in this dataset
- std::vector<FastS_EngineBase *> _enginesArray;
- search::Rand48 _randState;
-
- void InsertEngine(FastS_EngineBase *engine);
- FastS_EngineBase *ExtractEngine();
- bool RefCostUseNewEngine(FastS_EngineBase *oldEngine, FastS_EngineBase *newEngine, unsigned int *oldCount);
- bool UseNewEngine(FastS_EngineBase *oldEngine, FastS_EngineBase *newEngine, unsigned int *oldCount);
-
- bool IsValidPartIndex_HasLock(uint32_t partindex);
-public:
- FastS_PlainDataSet(FastS_AppContext *appCtx, FastS_DataSetDesc *desc);
- ~FastS_PlainDataSet() override;
-
- bool useFixedRowDistribution() const {
- return _queryDistributionMode == FastS_DataSetDesc::QueryDistributionMode::FIXEDROW;
- }
- uint32_t getNumRows() const { return _partMap.getNumRows(); }
- uint32_t getNumPartitions(size_t rowId) { return _partMap.getNumPartitions(rowId); }
- uint32_t GetRowBits() const { return _partMap._rowBits; }
- uint32_t GetPartBits() const { return _partMap._partBits; }
- uint32_t GetFirstPart() const { return _partMap._first_partition; }
- uint32_t GetLastPart() const {
- return _partMap._first_partition + _partMap._num_partitions;
- }
- uint32_t GetPartitions() const { return _partMap._num_partitions; }
- uint32_t GetEstimateParts() const { return _estimateParts; }
- uint32_t GetEstimatePartCutoff() const { return _estimatePartCutoff; }
- uint32_t GetMaxHitsPerNode() const { return _maxHitsPerNode; }
- double GetSlowQueryLimitFactor() const { return _slowQueryLimitFactor; }
- double GetSlowQueryLimitBias() const { return _slowQueryLimitBias; }
- double GetSlowDocsumLimitFactor() const { return _slowDocsumLimitFactor; }
- double GetSlowDocsumLimitBias() const { return _slowDocsumLimitBias; }
- bool GetTempFail() const { return _partMap._childparts < _partMap._minchildparts; }
- void UpdateMaxHitsPerNodeLog(bool incomplete, bool fuzzy);
- uint32_t getMaxNodesDownPerFixedRow() const { return _partMap._maxNodesDownPerFixedRow; }
- uint32_t useRoundRobinForFixedRow() const { return _partMap._useRoundRobinForFixedRow; }
- double getMinGroupCoverage() const { return _queryDistributionMode.getMinGroupCoverage(); }
- void updateSearchTime(double searchTime, uint32_t rowId);
- void updateActiveDocs_HasLock(uint32_t rowId, PossCount newVal, PossCount oldVal) {
- _stateOfRows.updateActiveDocs(rowId, newVal, oldVal);
- }
- PossCount getActiveDocs() const { return _stateOfRows.getActiveDocs(); }
- uint32_t getRandomWeightedRow() const;
-
- FastS_EngineBase * getPartition(const std::unique_lock<std::mutex> &dsGuard, uint32_t partid);
- FastS_EngineBase * getPartition(const std::unique_lock<std::mutex> &dsGuard, uint32_t partid, uint32_t rowid);
-
- size_t countNodesUpInRow_HasLock(uint32_t rowid);
-
- FastS_EngineBase * getPartitionMLD(const std::unique_lock<std::mutex> &dsGuard, uint32_t partid, bool mld);
- FastS_EngineBase * getPartitionMLD(const std::unique_lock<std::mutex> &dsGuard, uint32_t partid, bool mld, uint32_t rowid);
-
- void LinkInPart_HasLock(FastS_EngineBase *engine);
- void LinkOutPart_HasLock(FastS_EngineBase *engine);
-
- ChildInfo getChildInfo() const override;
-
- uint32_t getMPP() const { return _partMap._mpp; }
- double getMonitorInterval() const { return _monitorInterval; }
- double getHigherCoverageMaxSearchWait() const { return _higherCoverageMaxSearchWait; }
- double getHigherCoverageMinSearchWait() const { return _higherCoverageMinSearchWait; }
- double getMinimalSearchCoverage() const { return _minimalSearchCoverage; }
- double getHigherCoverageMaxDocSumWait() const { return _higherCoverageMaxDocSumWait; }
- double getHigherCoverageMinDocSumWait() const { return _higherCoverageMinDocSumWait; }
- double getHigherCoverageBaseDocSumWait() const { return _higherCoverageBaseDocSumWait; }
- double getMinimalDocSumCoverage() const { return _minimalDocSumCoverage; }
-
- // API
- //----
- uint32_t CalculateQueueLens_HasLock(uint32_t &dispatchnodes) override;
- bool AreEnginesReady() override;
- virtual void Ping();
-
- // Downcast
- //---------
- FastS_PlainDataSet * GetPlainDataSet() override { return this; }
-
- template <class FUN>
- FUN ForEachEngine(FUN fun) {
- for (FastS_EngineBase *ptr : _enginesArray) {
- fun(ptr);
- }
- return fun;
- }
-
- static bool EngineDocStampOK(time_t haveDocStamp) { return (haveDocStamp != 0); }
-};
diff --git a/searchcore/src/vespa/searchcore/fdispatch/search/poss_count.h b/searchcore/src/vespa/searchcore/fdispatch/search/poss_count.h
deleted file mode 100644
index b5075116da2..00000000000
--- a/searchcore/src/vespa/searchcore/fdispatch/search/poss_count.h
+++ /dev/null
@@ -1,16 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-
-#pragma once
-
-#include <cstdint>
-
-struct PossCount {
- bool valid;
- uint64_t count;
-
- PossCount() : valid(false), count(0) {}
-
- bool operator != (const PossCount& other) {
- return (valid != other.valid) || (count != other.count);
- }
-};
diff --git a/searchcore/src/vespa/searchcore/fdispatch/search/query.cpp b/searchcore/src/vespa/searchcore/fdispatch/search/query.cpp
deleted file mode 100644
index d50a2aed46a..00000000000
--- a/searchcore/src/vespa/searchcore/fdispatch/search/query.cpp
+++ /dev/null
@@ -1,122 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-
-#include "query.h"
-#include <vespa/searchlib/parsequery/simplequerystack.h>
-
-/** Marks as empty
- */
-FastS_query::FastS_query()
- : _dataset(0),
- _flags(0),
- _stackDump(),
- _sortSpec(),
- _groupSpec(),
- _location(),
- _rankProperties(),
- _featureOverrides()
-{
-};
-
-FastS_query::FastS_query(const search::docsummary::GetDocsumArgs &docsumArgs)
- : _dataset(0), // not known
- _flags(docsumArgs.GetQueryFlags()),
- _stackDump(docsumArgs.getStackDump()),
- _sortSpec(), // not known
- _groupSpec(), // not known
- _location(),
- _rankProperties(docsumArgs.rankProperties()),
- _featureOverrides(docsumArgs.featureOverrides())
-{
- // _query = search::SimpleQueryStack::StackbufToString(docsumArgs.getStackDump());
- if (docsumArgs.getLocation().size() > 0) {
- _location = strdup(docsumArgs.getLocation().c_str());
- }
-}
-
-
-void
-FastS_query::SetStackDump(vespalib::stringref stackRef)
-{
- _stackDump = stackRef;
-}
-
-const char *
-FastS_query::getPrintableQuery()
-{
- if (_printableQuery.empty()) {
- _printableQuery = search::SimpleQueryStack::StackbufToString(_stackDump);
- }
- return _printableQuery.c_str();
-}
-
-FastS_query::~FastS_query()
-{
-}
-
-
-void
-FastS_query::SetDataSet(uint32_t dataset)
-{
- _dataset = dataset;
-}
-
-unsigned int
-FastS_query::StackDumpHashKey() const
-{
- unsigned int res = 0;
- const unsigned char *p;
- const unsigned char *e;
- p = (const unsigned char *) _stackDump.begin();
- e = (const unsigned char *) _stackDump.end();
- while (p != e) {
- res = (res << 7) + (res >> 25) + *p;
- p++;
- }
- return res;
-}
-
-namespace
-{
-
-// This is ugly, somebody please find a better way.
-
-class SizeCollector : public search::fef::IPropertiesVisitor
-{
- static const size_t _stringFuzz = 15; // Compensate for malloc() waste
- static const size_t _vectorFuzz = 15;
- static const size_t _mapFuzz = 15;
- size_t _size;
-public:
- SizeCollector()
- : _size(0)
- {
- }
-
- virtual void
- visitProperty(const search::fef::Property::Value &key,
- const search::fef::Property &values) override
- {
- // Account for std::map element size
- _size += _mapFuzz;
- // Account for key string size
- _size += key.size() + _stringFuzz;
- size_t numValues = values.size();
- // Account for value vector size
- if (numValues > 0) {
- _size += numValues * sizeof(search::fef::Property::Value) + _vectorFuzz;
- for (size_t i = 0; i < numValues; ++i) {
- // Account for string sizes in value vector
- const search::fef::Property::Value &str = values.getAt(i);
- _size += str.size() + _stringFuzz;
- }
- }
- }
-
- size_t
- getSize() const
- {
- return _size;
- }
-};
-
-}
diff --git a/searchcore/src/vespa/searchcore/fdispatch/search/query.h b/searchcore/src/vespa/searchcore/fdispatch/search/query.h
deleted file mode 100644
index b6949e70d8f..00000000000
--- a/searchcore/src/vespa/searchcore/fdispatch/search/query.h
+++ /dev/null
@@ -1,75 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-
-#pragma once
-
-#include <vespa/searchlib/fef/properties.h>
-#include <vespa/searchsummary/docsummary/getdocsumargs.h>
-
-class FastS_query
-{
-public:
- uint32_t _dataset;
- uint32_t _flags;
- vespalib::string _stackDump;
- vespalib::string _printableQuery;
- vespalib::string _sortSpec;
- std::vector<char> _groupSpec; // this is binary
- vespalib::string _location;
- search::fef::Properties _rankProperties;
- search::fef::Properties _featureOverrides;
-
- FastS_query(const FastS_query &other);
- FastS_query &operator=(const FastS_query &other);
-public:
- FastS_query();
- FastS_query(const search::docsummary::GetDocsumArgs &docsumArgs);
- ~FastS_query();
-
- void SetStackDump(vespalib::stringref stackDump);
- void SetSortSpec(const char *spec) { _sortSpec = spec; }
- void SetLocation(const char *loc) { _location = loc; }
- void SetRankProperties(const search::fef::Properties &rp) { _rankProperties = rp; }
- void SetFeatureOverrides(const search::fef::Properties &fo) { _featureOverrides = fo; }
- void SetDataSet(uint32_t dataset);
- void SetQueryFlags(uint32_t flags) { _flags = flags; }
- void SetFlag(uint32_t flag) { _flags |= flag; }
- void ClearFlag(uint32_t flag) { _flags &= ~flag; }
- const vespalib::string &getStackDump() const { return _stackDump; }
- const char *GetSortSpec() const { return _sortSpec.c_str(); }
- const char *GetLocation() const { return _location.c_str(); }
- const search::fef::Properties &GetRankProperties() const { return _rankProperties; }
- const search::fef::Properties &GetFeatureOverrides() const { return _featureOverrides; }
-
- uint32_t GetQueryFlags() const { return _flags; }
- const char *getPrintableQuery();
- bool IsFlagSet(uint32_t flag) const { return (_flags & flag) != 0; }
-
- unsigned int StackDumpHashKey() const;
-
-
-private:
- static unsigned int hash_str_check(const unsigned char *pt)
- {
- if (pt == NULL)
- return 0;
-
- unsigned int res = 0;
- for (; *pt != 0; pt++)
- res = (res << 7) + (res >> 25) + *pt;
- return res;
- }
- static bool cmp_str_check(const char *a, const char *b)
- {
- if (a == NULL && b == NULL)
- return true;
- if (a == NULL || b == NULL)
- return false;
- return (strcmp(a, b) == 0);
- }
- static bool cmp_str_ref(vespalib::stringref a,
- vespalib::stringref b)
- {
- return (a.size() == b.size() &&
- memcmp(a.data(), b.data(), a.size()) == 0);
- }
-};
diff --git a/searchcore/src/vespa/searchcore/fdispatch/search/querycacheutil.cpp b/searchcore/src/vespa/searchcore/fdispatch/search/querycacheutil.cpp
deleted file mode 100644
index 5a2fd5a95c3..00000000000
--- a/searchcore/src/vespa/searchcore/fdispatch/search/querycacheutil.cpp
+++ /dev/null
@@ -1,156 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-
-#include "querycacheutil.h"
-#include <vespa/searchlib/parsequery/simplequerystack.h>
-#include <vespa/searchlib/common/sortdata.h>
-
-#include <vespa/log/log.h>
-LOG_SETUP(".search.querycacheutil");
-
-using search::common::SortData;
-
-uint32_t FastS_QueryCacheUtil::_systemMaxHits;
-uint32_t FastS_QueryCacheUtil::_maxOffset = 4000;
-
-
-FastS_QueryCacheUtil::FastS_QueryCacheUtil()
- : _startTime(),
- _userMaxHits(0),
- _alignedMaxHits(0),
- _alignedSearchOffset(0),
- _ranking(),
- _dateTime(0),
- _query(),
- _queryResult(),
- _docsumsResult(),
- _searchInfo(),
- _alignedHitBuf(NULL),
- _hitbuf_needfree(false),
- _alignedHitCount(0),
- _sortIndex(NULL),
- _sortData(NULL),
- _sortdata_needfree(false)
-{
- _searchInfo._maxHits = 10;
-}
-
-FastS_QueryCacheUtil::~FastS_QueryCacheUtil()
-{
-}
-
-void
-FastS_QueryCacheUtil::setSearchRequest(const search::engine::SearchRequest * request)
-{
- _ranking = request->ranking;
-
- _query.SetQueryFlags(request->queryFlags);
-
- _query.SetStackDump(request->getStackRef());
- _query.SetSortSpec(request->sortSpec.c_str());
- _query._groupSpec = request->groupSpec;
- _query.SetLocation(request->location.c_str());
- _query.SetRankProperties(request->propertiesMap.rankProperties());
- _query.SetFeatureOverrides(request->propertiesMap.featureOverrides());
-}
-
-
-void
-FastS_QueryCacheUtil::SetupQuery(uint32_t maxhits,
- uint32_t offset)
-{
- FastS_assert(_queryResult._hitbuf == NULL);
- FastS_assert(_alignedHitBuf == NULL);
- FastS_assert(!_hitbuf_needfree);
- FastS_assert(_queryResult._hitCount == 0);
- FastS_assert(_docsumsResult._fullResultCount == 0);
- FastS_assert(_alignedHitCount == 0);
- FastS_assert(_queryResult._totalHitCount == 0);
- FastS_assert(_alignedMaxHits == 0);
- FastS_assert(_alignedSearchOffset == 0);
- FastS_assert(_docsumsResult._fullresult == NULL);
- _searchInfo._searchOffset = offset;
- _searchInfo._maxHits = maxhits;
-}
-
-
-void
-FastS_QueryCacheUtil::AdjustSearchParameters(uint32_t partitions)
-{
- bool strict = (partitions > 1);
-
- if (_searchInfo._maxHits == 0) {
- _searchInfo._searchOffset = 0;
- }
-
- _searchInfo._maxHits = std::min(_searchInfo._maxHits, _maxOffset + _systemMaxHits);
- if (strict) {
- _searchInfo._searchOffset = std::min(_searchInfo._searchOffset, _maxOffset);
- _searchInfo._maxHits = std::min(_searchInfo._maxHits, _maxOffset + _systemMaxHits - _searchInfo._searchOffset);
- }
-}
-
-
-void
-FastS_QueryCacheUtil::AdjustSearchParametersFinal(uint32_t partitions)
-{
- if (IsEstimate()) {
-
- FastS_assert(_searchInfo._searchOffset == 0);
- FastS_assert(_searchInfo._maxHits == 0);
-
- _alignedSearchOffset = 0;
- _alignedMaxHits = 0;
- } else {
- _alignedSearchOffset = (partitions > 1) ? 0 : _searchInfo._searchOffset;
- _alignedMaxHits = _searchInfo._maxHits + _searchInfo._searchOffset - _alignedSearchOffset;
- FastS_assert(_alignedMaxHits <= _maxOffset + _systemMaxHits);
- }
-}
-
-void
-FastS_QueryCacheUtil::DropResult()
-{
- _queryResult._groupResultLen = 0;
- _queryResult._groupResult = NULL;
-
- if (_hitbuf_needfree) {
- FastS_assert(_alignedHitBuf != NULL);
- free(_alignedHitBuf);
- }
- if (_sortdata_needfree) {
- FastS_assert(_sortIndex != NULL);
- free(_sortIndex);
- }
- _sortIndex = NULL;
- _sortData = NULL;
- _sortdata_needfree = false;
- _alignedHitBuf = NULL;
- _queryResult._hitbuf = NULL;
- _hitbuf_needfree = false;
- free(_docsumsResult._fullresult);
- _docsumsResult._fullresult = NULL;
- _queryResult._hitCount = 0;
- _docsumsResult._fullResultCount = 0;
- _queryResult._totalHitCount = 0;
- _queryResult._maxRank = std::numeric_limits<search::HitRank>::is_integer ?
- std::numeric_limits<search::HitRank>::min() :
- - std::numeric_limits<search::HitRank>::max();
-
- _alignedHitCount = 0;
-}
-
-
-bool
-FastS_QueryCacheUtil::IsEstimate() const
-{
- return _query.IsFlagSet(search::fs4transport::QFLAG_ESTIMATE);
-}
-
-void
-FastS_QueryCacheUtil::InitEstimateMode()
-{
- _searchInfo._searchOffset = 0;
- _searchInfo._maxHits = 0;
- _ranking.clear();
- _dateTime = 0;
-}
diff --git a/searchcore/src/vespa/searchcore/fdispatch/search/querycacheutil.h b/searchcore/src/vespa/searchcore/fdispatch/search/querycacheutil.h
deleted file mode 100644
index 442fa3f0710..00000000000
--- a/searchcore/src/vespa/searchcore/fdispatch/search/querycacheutil.h
+++ /dev/null
@@ -1,151 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-
-#pragma once
-
-#include <vespa/searchcore/fdispatch/search/query.h>
-#include <vespa/searchlib/common/transport.h>
-
-#include <vespa/searchcore/fdispatch/common/search.h>
-#include <vespa/searchcore/util/log.h>
-
-class FastS_DataSetCollection;
-
-class FastS_QueryCacheUtil
-{
-private:
- FastS_QueryCacheUtil(const FastS_QueryCacheUtil &);
- FastS_QueryCacheUtil& operator=(const FastS_QueryCacheUtil &);
-
- double _startTime; // For the query
-
- uint32_t _userMaxHits; // Max hits spec.d by user; NB: see _systemMaxHits
- uint32_t _alignedMaxHits; // Max hits (forwarded to engine)
- uint32_t _alignedSearchOffset; // Search offset (forwarded to engine)
- vespalib::string _ranking; // ranking profile to be used
- uint32_t _randomSeed; // seed for random rank values
- uint32_t _dateTime; // datetime used for freshness boost
-
- FastS_query _query; // NB: Here it is!
-
- FastS_QueryResult _queryResult;
- FastS_DocsumsResult _docsumsResult;
- FastS_SearchInfo _searchInfo;
-
- FastS_hitresult *_alignedHitBuf; // Hits from engine
- bool _hitbuf_needfree; // Destructor should free _hitbuf.
- uint32_t _alignedHitCount; // # Hits from engine
-
- uint32_t *_sortIndex;
- char *_sortData; // NB: same malloc as _sortIndex
- bool _sortdata_needfree;
-
-public:
- static uint32_t _systemMaxHits;
- static uint32_t _maxOffset;
-public:
- FastS_QueryCacheUtil();
- ~FastS_QueryCacheUtil();
- bool AgeDropCheck();
- void DropResult();
- bool GotNoResultsYet() const { return _queryResult._hitbuf == NULL; }
- uint32_t GetSearchOffset() const { return _searchInfo._searchOffset; }
- uint32_t GetMaxHits() const { return _searchInfo._maxHits; }
- uint32_t GetAlignedMaxHits() const { return _alignedMaxHits; }
- uint32_t GetAlignedSearchOffset() const { return _alignedSearchOffset; }
- const vespalib::string & GetRanking() const { return _ranking; }
- uint32_t GetRandomSeed() const { return _randomSeed; }
- uint32_t GetDateTime() const { return _dateTime; }
- FastS_query &GetQuery() { return _query; }
- const char *GetSortSpec() const { return _query.GetSortSpec(); }
- const char *GetLocation() const { return _query.GetLocation(); }
- bool ShouldDropSortData() const {
- return _query.IsFlagSet(search::fs4transport::QFLAG_DROP_SORTDATA);
- }
- bool IsQueryFlagSet(uint32_t flag) const { return _query.IsFlagSet(flag); }
- FastS_QueryResult *GetQueryResult() {
- return &_queryResult;
- }
- FastS_DocsumsResult *GetDocsumsResult() { return &_docsumsResult; }
- FastS_SearchInfo *GetSearchInfo() { return &_searchInfo; }
- void SetStartTime(double timeref) { _startTime = timeref; }
- void AdjustSearchParameters(uint32_t partitions);
- void AdjustSearchParametersFinal(uint32_t partitions);
- void SetupQuery(uint32_t maxhits, uint32_t offset);
- bool IsEstimate() const;
- void ForceStrictLimits();
- void InitEstimateMode();
- double ElapsedSecs(double now) const {
- double ret = now - _startTime;
- if (ret < 0.0)
- ret = 0.0;
- return ret;
- }
- void SetCoverage(uint64_t coverageDocs, uint64_t activeDocs, uint64_t soonActiveDocs,
- uint32_t degradeReason, uint16_t nodesQueried, uint16_t nodesReplied)
- {
- _searchInfo._coverageDocs = coverageDocs;
- _searchInfo._activeDocs = activeDocs;
- _searchInfo._soonActiveDocs = soonActiveDocs;
- _searchInfo._degradeReason = degradeReason;
- _searchInfo._nodesQueried = nodesQueried;
- _searchInfo._nodesReplied = nodesReplied;
- }
- void SetAlignedHitCount(uint32_t alignedHitCount) {
- if (alignedHitCount > _alignedMaxHits) {
- alignedHitCount = _alignedMaxHits;
- }
- _alignedHitCount = alignedHitCount;
- }
- void CalcHitCount() {
- if (_alignedHitCount + _alignedSearchOffset > _searchInfo._searchOffset) {
- _queryResult._hitCount = _alignedHitCount + _alignedSearchOffset - _searchInfo._searchOffset;
- } else {
- _queryResult._hitCount = 0;
- }
- if (_queryResult._hitCount > _searchInfo._maxHits) {
- _queryResult._hitCount = _searchInfo._maxHits;
- }
- }
- void AllocAlignedHitBuf() {
- FastS_assert(_alignedHitBuf == NULL);
- if (_alignedHitCount != 0) {
- _alignedHitBuf = (FastS_hitresult*)malloc(sizeof(FastS_hitresult) * _alignedHitCount);
- _hitbuf_needfree = true;
- _queryResult._hitbuf = _alignedHitBuf + _searchInfo._searchOffset - _alignedSearchOffset;
- }
- }
- void AllocSortData(uint32_t sortDataLen)
- {
- FastS_assert(_sortIndex == NULL && _sortData == NULL);
- uint32_t hitcnt = _alignedHitCount;
- if (hitcnt == 0) {
- FastS_assert(sortDataLen == 0);
- return;
- }
- void *pt = malloc((hitcnt + 1) * sizeof(uint32_t) + sortDataLen);
- FastS_assert(pt != NULL);
- _sortIndex = (uint32_t *) pt;
- _sortData = (char *)(void *)(_sortIndex + hitcnt + 1);
- _sortdata_needfree = true;
- if (hitcnt > _searchInfo._searchOffset) {
- _queryResult._sortIndex =
- _sortIndex + _searchInfo._searchOffset - _alignedSearchOffset;
- _queryResult._sortData = _sortData;
- }
- }
- uint32_t *GetSortIndex() const { return _sortIndex; }
- char *GetSortData() const { return _sortData; }
- FastS_hitresult *GetAlignedHitBuf() const { return _alignedHitBuf; }
- FastS_hitresult *GetAlignedHitBufEnd() const {
- return _alignedHitBuf + _alignedHitCount;
- }
- uint32_t GetAlignedHitCount() const { return _alignedHitCount; }
- void SetGroupResult(const char *groupResult) {
- _queryResult._groupResult = groupResult;
- }
- void SetGroupResultLen(uint32_t groupResultLen) {
- _queryResult._groupResultLen = groupResultLen;
- }
- void setSearchRequest(const search::engine::SearchRequest * request);
-};
-
diff --git a/searchcore/src/vespa/searchcore/fdispatch/search/rowstate.cpp b/searchcore/src/vespa/searchcore/fdispatch/search/rowstate.cpp
deleted file mode 100644
index b11d9aeac53..00000000000
--- a/searchcore/src/vespa/searchcore/fdispatch/search/rowstate.cpp
+++ /dev/null
@@ -1,90 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-
-#include <vespa/searchcore/fdispatch/search/rowstate.h>
-
-namespace fdispatch {
-
-constexpr uint64_t MIN_DECAY_RATE = 42;
-constexpr double MIN_QUERY_TIME = 0.001;
-
-RowState::RowState(double initialValue, uint64_t decayRate) :
- _decayRate(std::max(decayRate, MIN_DECAY_RATE)),
- _avgSearchTime(std::max(initialValue, MIN_QUERY_TIME)),
- _sumActiveDocs(0),
- _numQueries(0)
-{ }
-
-void RowState::updateSearchTime(double searchTime)
-{
- searchTime = std::max(searchTime, MIN_QUERY_TIME);
- double decayRate = std::min(_numQueries + MIN_DECAY_RATE, _decayRate);
- _avgSearchTime = (searchTime + (decayRate-1)*_avgSearchTime)/decayRate;
- ++_numQueries;
-}
-
-StateOfRows::StateOfRows(size_t numRows, double initialValue, uint64_t decayRate) :
- _rows(numRows, RowState(initialValue, decayRate)),
- _sumActiveDocs(0), _invalidActiveDocsCounter(0)
-{
- srand48(1);
-}
-
-void
-StateOfRows::updateSearchTime(double searchTime, uint32_t rowId)
-{
- _rows[rowId].updateSearchTime(searchTime);
-}
-
-uint32_t
-StateOfRows::getRandomWeightedRow() const
-{
- return getWeightedNode(drand48());
-}
-
-uint32_t
-StateOfRows::getWeightedNode(double cand) const
-{
- double sum = 0;
- for (const RowState & rs : _rows) {
- sum += rs.getAverageSearchTimeInverse();
- }
- double accum(0.0);
- for (size_t rowId(0); (rowId + 1) < _rows.size(); rowId++) {
- accum += _rows[rowId].getAverageSearchTimeInverse();
- if (cand < accum/sum) {
- return rowId;
- }
- }
- return _rows.size() - 1;
-}
-
-void
-StateOfRows::updateActiveDocs(uint32_t rowId, PossCount newVal, PossCount oldVal)
-{
- uint64_t tmp = _sumActiveDocs + newVal.count - oldVal.count;
- _sumActiveDocs = tmp;
- _rows[rowId].updateActiveDocs(newVal.count, oldVal.count);
- if (newVal.valid != oldVal.valid) {
- if (oldVal.valid) {
- ++_invalidActiveDocsCounter;
- } else {
- --_invalidActiveDocsCounter;
- }
- }
-}
-
-PossCount
-StateOfRows::getActiveDocs() const
-{
- PossCount r;
- if (activeDocsValid()) {
- r.valid = true;
- r.count = 0;
- for (const RowState &row : _rows) {
- r.count = std::max(r.count, row.activeDocs());
- }
- }
- return r;
-}
-
-}
diff --git a/searchcore/src/vespa/searchcore/fdispatch/search/rowstate.h b/searchcore/src/vespa/searchcore/fdispatch/search/rowstate.h
deleted file mode 100644
index 92268a291bb..00000000000
--- a/searchcore/src/vespa/searchcore/fdispatch/search/rowstate.h
+++ /dev/null
@@ -1,60 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-
-#pragma once
-
-#include <vector>
-#include <stdint.h>
-#include <stdlib.h>
-#include "poss_count.h"
-
-namespace fdispatch {
-
-/**
- * RowState keeps track of state per row or rather group.
- * Currently it just keeps the average searchtime as exponential decay.
- **/
-class RowState {
-public:
- RowState(double initialValue, uint64_t decayRate);
- double getAverageSearchTime() const { return _avgSearchTime; }
- double getAverageSearchTimeInverse() const { return 1.0/_avgSearchTime; }
- void updateSearchTime(double searchTime);
- void setAverageSearchTime(double avgSearchTime) { _avgSearchTime = avgSearchTime; }
- uint64_t activeDocs() const { return _sumActiveDocs; }
- void updateActiveDocs(uint64_t newVal, uint64_t oldVal) {
- uint64_t tmp = _sumActiveDocs + newVal - oldVal;
- _sumActiveDocs = tmp;
- }
-private:
- const uint64_t _decayRate;
- double _avgSearchTime;
- uint64_t _sumActiveDocs;
- uint64_t _numQueries;
-};
-
-/**
- * StateOfRows keeps track of the state of all rows/groups.
- * Currently used for tracking latency in groups. This latency
- * can be used for selecting a random node with weighted probability
- * with the intention to favor load on fast groups.
- **/
-class StateOfRows {
-public:
- StateOfRows(size_t numRows, double initial, uint64_t decayRate);
- void updateSearchTime(double searchTime, uint32_t rowId);
- const RowState & getRowState(uint32_t rowId) const { return _rows[rowId]; }
- RowState & getRowState(uint32_t rowId) { return _rows[rowId]; }
- uint32_t getRandomWeightedRow() const;
- uint32_t getWeightedNode(double rnd) const;
- void updateActiveDocs(uint32_t rowId, PossCount newVal, PossCount oldVal);
- uint32_t numRowStates() const { return _rows.size(); }
- uint64_t sumActiveDocs() const { return _sumActiveDocs; }
- PossCount getActiveDocs() const;
- bool activeDocsValid() const { return _invalidActiveDocsCounter == 0; }
-private:
- std::vector<RowState> _rows;
- uint64_t _sumActiveDocs;
- size_t _invalidActiveDocsCounter;
-};
-
-}
diff --git a/searchcore/src/vespa/searchcore/fdispatch/search/search_path.cpp b/searchcore/src/vespa/searchcore/fdispatch/search/search_path.cpp
deleted file mode 100644
index daaf2a7c1db..00000000000
--- a/searchcore/src/vespa/searchcore/fdispatch/search/search_path.cpp
+++ /dev/null
@@ -1,114 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-
-#include "search_path.h"
-#include <vespa/vespalib/stllike/asciistream.h>
-
-#include <vespa/log/log.h>
-LOG_SETUP(".fdispatch.search_path");
-
-namespace fdispatch {
-
-SearchPath::Element::Element()
- : _nodes(),
- _row(std::numeric_limits<size_t>::max())
-{
-}
-
-vespalib::stringref
-SearchPath::parseElement(vespalib::stringref spec, size_t numNodes)
-{
- _elements.push_back(Element());
- vespalib::string::size_type specSepPos(spec.find('/'));
- parsePartList(spec.substr(0, specSepPos), numNodes);
-
- vespalib::stringref remaining = spec.substr(specSepPos + 1);
- vespalib::string::size_type elementSepPos = remaining.find(';');
- parseRow(remaining.substr(0, elementSepPos));
-
- if (elementSepPos != vespalib::string::npos) {
- return remaining.substr(elementSepPos + 1);
- }
- return vespalib::stringref();
-}
-
-void
-SearchPath::parsePartList(vespalib::stringref partSpec, size_t numNodes)
-{
- try {
- if (!partSpec.empty() && (partSpec[0] != '*')) {
- vespalib::asciistream is(partSpec);
- is.eatWhite();
- parsePartList(is, numNodes);
- } else {
- for (size_t i(0); i < numNodes; i++) {
- _elements.back().addPart(i);
- }
- }
- } catch (const std::exception & e) {
- LOG(warning, "Failed parsing part of searchpath='%s' with error '%s'. Result might be mumbo jumbo.",
- vespalib::string(partSpec).c_str(), e.what());
- }
-}
-
-void
-SearchPath::parsePartList(vespalib::asciistream &spec, size_t numNodes)
-{
- spec.eatWhite();
- if ( !spec.empty() ) {
- char c(spec.c_str()[0]);
- if (c == '[') {
- parsePartRange(spec, numNodes);
- } else {
- size_t num(0);
- spec >> num;
- _elements.back().addPart(num);
- }
- if ( ! spec.eof() ) {
- spec >> c;
- if (c == ',') {
- parsePartList(spec, numNodes);
- }
- }
- } else {
- throw std::runtime_error("Expected either '[' or a number, got EOF");
- }
-}
-
-void
-SearchPath::parsePartRange(vespalib::asciistream &spec, size_t numNodes)
-{
- size_t from(0);
- size_t to(numNodes);
- char s(0), c(0), e(0);
- spec >> s >> from >> c >> to >> e;
- if (c != ',') {
- throw std::runtime_error("Expected ','");
- }
- if (e != '>') {
- throw std::runtime_error("Expected '>'");
- }
- to = std::min(numNodes, to);
- for (size_t i(from); i < to; i++) {
- _elements.back().addPart(i);
- }
-}
-
-void
-SearchPath::parseRow(vespalib::stringref rowSpec)
-{
- if (!rowSpec.empty()) {
- // FIXME C++17 range-safe from_chars() instead of strtoul()
- _elements.back().setRow(strtoul(rowSpec.data(), nullptr, 0));
- }
-}
-
-SearchPath::SearchPath(const vespalib::string &spec, size_t numNodes)
- : _elements()
-{
- vespalib::stringref specBuf = spec;
- while (!specBuf.empty()) {
- specBuf = parseElement(specBuf, numNodes);
- }
-}
-
-} // namespace fdispatch
diff --git a/searchcore/src/vespa/searchcore/fdispatch/search/search_path.h b/searchcore/src/vespa/searchcore/fdispatch/search/search_path.h
deleted file mode 100644
index bbf1002742e..00000000000
--- a/searchcore/src/vespa/searchcore/fdispatch/search/search_path.h
+++ /dev/null
@@ -1,58 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-
-#pragma once
-
-#include <vespa/vespalib/stllike/string.h>
-#include <set>
-#include <vector>
-#include <limits>
-
-namespace vespalib {
- class asciistream;
-}
-namespace fdispatch {
-
-class SearchPath
-{
-public:
- typedef std::set<size_t> NodeList;
-
- class Element
- {
- private:
- NodeList _nodes;
- size_t _row;
-
- public:
- Element();
- Element &addPart(size_t part) {
- _nodes.insert(part);
- return *this;
- }
- Element &setRow(size_t row_) {
- _row = row_;
- return *this;
- }
- bool hasRow() const { return _row != std::numeric_limits<size_t>::max(); }
- size_t row() const { return _row; }
- const NodeList &nodes() const { return _nodes; }
- };
-
- typedef std::vector<Element> ElementVector;
-
-private:
- ElementVector _elements;
-
- vespalib::stringref parseElement(vespalib::stringref spec, size_t numNodes);
- void parsePartList(vespalib::stringref partSpec, size_t numNodes);
- void parsePartList(vespalib::asciistream &spec, size_t numNodes);
- void parsePartRange(vespalib::asciistream &spec, size_t numNodes);
- void parseRow(vespalib::stringref rowSpec);
-
-public:
- SearchPath(const vespalib::string &spec, size_t numNodes);
- const ElementVector &elements() const { return _elements; }
-};
-
-} // namespace fdispatch
-