aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorHenning Baldersheim <balder@yahoo-inc.com>2017-10-06 18:24:05 +0200
committerGitHub <noreply@github.com>2017-10-06 18:24:05 +0200
commit8c0427bd8b0de46d8c61f259f80aa3b81bfa128c (patch)
tree5b044748ebb96c39a0a0a9a90d01f5e7e57e9a04
parent4bcfe2688131797449e20b8400228f75a95bc000 (diff)
parenta0c814fafbedba6edc8e730dfca1363623bf3437 (diff)
Merge pull request #3675 from vespa-engine/balder/summary.numthreads-2-backgroundnumthreads-rebased-1
Balder/summary.numthreads 2 backgroundnumthreads rebased 1
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomSearchTuningBuilder.java14
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/search/NodeFlavorTuning.java6
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/search/Tuning.java13
-rw-r--r--config-model/src/main/resources/schema/content.rnc3
-rw-r--r--config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/DomSearchTuningBuilderTest.java10
-rw-r--r--config-model/src/test/java/com/yahoo/vespa/model/search/NodeFlavorTuningTest.java2
-rw-r--r--searchcore/src/vespa/searchcore/config/proton.def7
-rw-r--r--searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.cpp5
-rw-r--r--searchcore/src/vespa/searchcore/proton/server/proton.cpp70
-rw-r--r--searchlib/src/apps/docstore/benchmarkdatastore.cpp2
-rw-r--r--searchlib/src/apps/docstore/documentstoreinspect.cpp2
-rw-r--r--searchlib/src/apps/docstore/verifylogdatastore.cpp2
-rw-r--r--searchlib/src/apps/tests/biglogtest.cpp2
-rw-r--r--searchlib/src/tests/docstore/document_store/document_store_test.cpp2
-rw-r--r--searchlib/src/tests/docstore/document_store_visitor/document_store_visitor_test.cpp4
-rw-r--r--searchlib/src/tests/docstore/logdatastore/logdatastore_test.cpp49
-rw-r--r--searchlib/src/vespa/searchlib/docstore/logdatastore.cpp2
-rw-r--r--searchlib/src/vespa/searchlib/docstore/logdatastore.h3
18 files changed, 114 insertions, 84 deletions
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomSearchTuningBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomSearchTuningBuilder.java
index ba89169fc4c..4ea638ca41a 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomSearchTuningBuilder.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomSearchTuningBuilder.java
@@ -74,6 +74,8 @@ public class DomSearchTuningBuilder extends VespaDomBuilder.DomConfigProducerBui
handleSummary(parent, e, t.searchNode);
} else if (equals("initialize", e)) {
handleInitialize(e, t.searchNode);
+ } else if (equals("background", e)) {
+ handleBackground(e, t.searchNode);
}
}
}
@@ -251,6 +253,9 @@ public class DomSearchTuningBuilder extends VespaDomBuilder.DomConfigProducerBui
} else if (equals("minfilesizefactor", e)) {
s.logStore.minFileSizeFactor = asDouble(e);
} else if (equals("numthreads", e)) {
+ parent.deployLogger().log(Level.WARNING,
+ "Element 'numthreads is deprecated. Use background.threads instead." +
+ " For now it will take max of the two.");
s.logStore.numThreads = asInt(e);
} else if (equals("chunk", e)) {
s.logStore.chunk = new Tuning.SearchNode.Summary.Store.Component(true);
@@ -268,4 +273,13 @@ public class DomSearchTuningBuilder extends VespaDomBuilder.DomConfigProducerBui
}
}
+ private void handleBackground(Element spec, Tuning.SearchNode sn) {
+ sn.background = new Tuning.SearchNode.Background();
+ for (Element e : XML.getChildren(spec)) {
+ if (equals("threads", e)) {
+ sn.background.threads = asInt(e);
+ }
+ }
+ }
+
}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/NodeFlavorTuning.java b/config-model/src/main/java/com/yahoo/vespa/model/search/NodeFlavorTuning.java
index 5750db08178..6cfd52cc232 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/search/NodeFlavorTuning.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/search/NodeFlavorTuning.java
@@ -27,7 +27,7 @@ public class NodeFlavorTuning implements ProtonConfig.Producer {
setHwInfo(builder);
tuneDiskWriteSpeed(builder);
tuneDocumentStoreMaxFileSize(builder.summary.log);
- tuneDocumentStoreNumThreads(builder.summary.log);
+ tuneDocumentStoreNumThreads(builder.background);
tuneFlushStrategyMemoryLimits(builder.flush.memory);
tuneFlushStrategyTlsSize(builder.flush.memory);
}
@@ -56,8 +56,8 @@ public class NodeFlavorTuning implements ProtonConfig.Producer {
builder.maxfilesize(fileSizeBytes);
}
- private void tuneDocumentStoreNumThreads(ProtonConfig.Summary.Log.Builder builder) {
- builder.numthreads(max(8, (int)nodeFlavor.getMinCpuCores()/2));
+ private void tuneDocumentStoreNumThreads(ProtonConfig.Background.Builder builder) {
+ builder.threads(max(8, (int)nodeFlavor.getMinCpuCores()/2));
}
private void tuneFlushStrategyMemoryLimits(ProtonConfig.Flush.Memory.Builder builder) {
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/Tuning.java b/config-model/src/main/java/com/yahoo/vespa/model/search/Tuning.java
index 17af4030cb6..1e3b1783f9b 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/search/Tuning.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/search/Tuning.java
@@ -334,6 +334,17 @@ public class Tuning extends AbstractConfigProducer implements PartitionsConfig.P
}
}
+ public static class Background implements ProtonConfig.Producer {
+ public Integer threads = null;
+
+ @Override
+ public void getConfig(ProtonConfig.Builder builder) {
+ if (threads != null) {
+ builder.background.threads(threads);
+ }
+ }
+ }
+
public RequestThreads threads = null;
public FlushStrategy strategy = null;
public Resizing resizing = null;
@@ -341,6 +352,7 @@ public class Tuning extends AbstractConfigProducer implements PartitionsConfig.P
public Attribute attribute = null;
public Summary summary = null;
public Initialize initialize = null;
+ public Background background = null;
@Override
public void getConfig(ProtonConfig.Builder builder) {
@@ -351,6 +363,7 @@ public class Tuning extends AbstractConfigProducer implements PartitionsConfig.P
if (attribute != null) attribute.getConfig(builder);
if (summary != null) summary.getConfig(builder);
if (initialize != null) initialize.getConfig(builder);
+ if (background != null) background.getConfig(builder);
}
}
diff --git a/config-model/src/main/resources/schema/content.rnc b/config-model/src/main/resources/schema/content.rnc
index 9031e8128df..e75c15ea524 100644
--- a/config-model/src/main/resources/schema/content.rnc
+++ b/config-model/src/main/resources/schema/content.rnc
@@ -353,6 +353,9 @@ Tuning = element tuning {
}? &
element initialize {
element threads { xsd:nonNegativeInteger }?
+ }? &
+ element background {
+ element threads { xsd:nonNegativeInteger }?
}?
}?
}
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/DomSearchTuningBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/DomSearchTuningBuilderTest.java
index 613827d2cf1..210334a1f23 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/DomSearchTuningBuilderTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/DomSearchTuningBuilderTest.java
@@ -233,4 +233,14 @@ public class DomSearchTuningBuilderTest extends DomBuilderTest {
assertThat(cfg, containsString("initialize.threads 7"));
}
+ @Test
+ public void requireThatWeCanParseBackgroundTag() {
+ Tuning t = createTuning(parseXml("<background>",
+ "<threads>7</threads>",
+ "</background>"));
+ assertEquals(7, t.searchNode.background.threads.intValue());
+ String cfg = getProtonCfg(t);
+ assertThat(cfg, containsString("background.threads 7"));
+ }
+
}
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/search/NodeFlavorTuningTest.java b/config-model/src/test/java/com/yahoo/vespa/model/search/NodeFlavorTuningTest.java
index 0e1ad92c70d..75f6abb2542 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/search/NodeFlavorTuningTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/search/NodeFlavorTuningTest.java
@@ -97,7 +97,7 @@ public class NodeFlavorTuningTest {
}
private static void assertDocumentStoreNumThreads(int numThreads, double numCores) {
- assertEquals(numThreads, configFromNumCoresSetting(numCores).summary().log().numthreads());
+ assertEquals(numThreads, configFromNumCoresSetting(numCores).background().threads());
}
private static void assertFlushStrategyTlsSize(long expTlsSizeBytes, int diskGb) {
diff --git a/searchcore/src/vespa/searchcore/config/proton.def b/searchcore/src/vespa/searchcore/config/proton.def
index 594a8717d7f..8c01c45dac8 100644
--- a/searchcore/src/vespa/searchcore/config/proton.def
+++ b/searchcore/src/vespa/searchcore/config/proton.def
@@ -256,6 +256,8 @@ summary.log.maxbucketspread double default=2.5
summary.log.minfilesizefactor double default=0.2
## Number of threads used for compressing incomming documents/compacting.
+## Deprecated. Use background.threads instead.
+## TODO Remove
summary.log.numthreads int default=8 restart
## Control io options during flush of stored documents.
@@ -376,6 +378,11 @@ visit.ignoremaxbytes bool default=true
## When set to 0 (default) we use 1 separate thread per document database.
initialize.threads int default = 0
+## Number of worker threads doing background compaction/compression tasks.
+## They all live i a shared thread pool.
+## When set to 0 (default), it will have enough threads to saturate half of the cores.
+background.threads int default=0
+
## Portion of enumstore address space that can be used before put and update
## portion of feed is blocked.
writefilter.attribute.enumstorelimit double default = 0.9
diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.cpp b/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.cpp
index f45fc658129..6aea5234fbe 100644
--- a/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.cpp
@@ -170,9 +170,8 @@ deriveConfig(const ProtonConfig::Summary & summary, const ProtonConfig::Flush::M
logConfig.setMaxFileSize(log.maxfilesize)
.setMaxDiskBloatFactor(std::min(flush.diskbloatfactor, flush.each.diskbloatfactor))
.setMaxBucketSpread(log.maxbucketspread).setMinFileSizeFactor(log.minfilesizefactor)
- .setNumThreads(log.numthreads).compact2ActiveFile(log.compact2activefile)
- .compactCompression(deriveCompression(log.compact.compression)).setFileConfig(fileConfig)
- .disableCrcOnRead(chunk.skipcrconread);
+ .compact2ActiveFile(log.compact2activefile).compactCompression(deriveCompression(log.compact.compression))
+ .setFileConfig(fileConfig).disableCrcOnRead(chunk.skipcrconread);
return LogDocumentStore::Config(config, logConfig);
}
diff --git a/searchcore/src/vespa/searchcore/proton/server/proton.cpp b/searchcore/src/vespa/searchcore/proton/server/proton.cpp
index 7caed5198bd..5863c03226c 100644
--- a/searchcore/src/vespa/searchcore/proton/server/proton.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/proton.cpp
@@ -85,12 +85,24 @@ diskMemUsageSamplerConfig(const ProtonConfig &proton, const HwInfo &hwInfo)
hwInfo);
}
+size_t
+deriveBackgroundThreads(const ProtonConfig & proton) {
+ size_t threads = std::thread::hardware_concurrency();
+ if (proton.background.threads != 0) {
+ threads = proton.background.threads;
+ }
+ threads = std::max(threads, size_t(proton.summary.log.numthreads));
+
+ // We need at least 1 guaranteed free worker in order to ensure progress so #documentsdbs + 1 should suffice,
+ // but we will not be cheap and give #documentsdbs * 2
+ return std::max(threads, proton.documentdb.size() * 2);;
}
-static const vespalib::string CUSTOM_COMPONENT_API_PATH = "/state/v1/custom/component";
+const vespalib::string CUSTOM_COMPONENT_API_PATH = "/state/v1/custom/component";
-Proton::ProtonFileHeaderContext::ProtonFileHeaderContext(const Proton &proton_,
- const vespalib::string &creator)
+}
+
+Proton::ProtonFileHeaderContext::ProtonFileHeaderContext(const Proton &proton_, const vespalib::string &creator)
: _proton(proton_),
_hostName(),
_creator(creator),
@@ -208,7 +220,7 @@ Proton::init()
{
assert( ! _initStarted && ! _initComplete );
_initStarted = true;
- if (_threadPool.NewThread(&_clock, NULL) == NULL) {
+ if (_threadPool.NewThread(&_clock, nullptr) == nullptr) {
throw IllegalStateException("Failed starting thread for the cheap clock");
}
_protonConfigFetcher.start();
@@ -234,8 +246,7 @@ Proton::init(const BootstrapConfig::SP & configSnapshot)
hwDiskCfg.samplewritesize,
hwDiskCfg.shared,
hwMemoryCfg.size);
- _hwInfoSampler = std::make_unique<HwInfoSampler>(protonConfig.basedir,
- samplerCfg);
+ _hwInfoSampler = std::make_unique<HwInfoSampler>(protonConfig.basedir, samplerCfg);
_hwInfo = _hwInfoSampler->hwInfo();
setFS4Compression(protonConfig);
_diskMemUsageSampler = std::make_unique<DiskMemUsageSampler>
@@ -244,8 +255,7 @@ Proton::init(const BootstrapConfig::SP & configSnapshot)
_metricsEngine.reset(new MetricsEngine());
_metricsEngine->addMetricsHook(_metricsHook);
- _fileHeaderContext.setClusterName(protonConfig.clustername,
- protonConfig.basedir);
+ _fileHeaderContext.setClusterName(protonConfig.clustername, protonConfig.basedir);
_tls.reset(new TLS(_configUri.createWithNewId(protonConfig.tlsconfigid), _fileHeaderContext));
_matchEngine.reset(new MatchEngine(protonConfig.numsearcherthreads,
protonConfig.numthreadspersearch,
@@ -257,7 +267,7 @@ Proton::init(const BootstrapConfig::SP & configSnapshot)
const ProtonConfig::Flush & flush(protonConfig.flush);
switch (flush.strategy) {
case ProtonConfig::Flush::MEMORY: {
- MemoryFlush::SP memoryFlush = std::make_shared<MemoryFlush>(
+ auto memoryFlush = std::make_shared<MemoryFlush>(
MemoryFlushConfigUpdater::convertConfig(flush.memory), fastos::ClockSystem::now());
_memoryFlushConfigUpdater = std::make_unique<MemoryFlushConfigUpdater>(memoryFlush, flush.memory);
_diskMemUsageSampler->notifier().addDiskMemUsageListener(_memoryFlushConfigUpdater.get());
@@ -290,14 +300,12 @@ Proton::init(const BootstrapConfig::SP & configSnapshot)
vespalib::string fileConfigId;
_warmupExecutor.reset(new vespalib::ThreadStackExecutor(4, 128*1024));
- // We need at least 1 guaranteed free worker in order to ensure progress so #documentsdbs + 1 should suffice,
- // but we will not be cheap and give #documentsdbs * 2
- const size_t summaryThreads = std::max(size_t(protonConfig.summary.log.numthreads), protonConfig.documentdb.size() * 2);
+
+ const size_t summaryThreads = deriveBackgroundThreads(protonConfig);
_summaryExecutor.reset(new vespalib::BlockingThreadStackExecutor(summaryThreads, 128*1024, summaryThreads*16));
InitializeThreads initializeThreads;
if (protonConfig.initialize.threads > 0) {
- initializeThreads = std::make_shared<vespalib::ThreadStackExecutor>
- (protonConfig.initialize.threads, 128 * 1024);
+ initializeThreads = std::make_shared<vespalib::ThreadStackExecutor>(protonConfig.initialize.threads, 128 * 1024);
_initDocumentDbsInSequence = (protonConfig.initialize.threads == 1);
}
_protonConfigurer.applyInitialConfig(initializeThreads);
@@ -563,10 +571,8 @@ Proton::addDocumentDB(const document::DocumentType &docType,
try {
ret->start();
} catch (vespalib::Exception &e) {
- LOG(warning,
- "Failed to start database for document type '%s'; %s",
- docTypeName.toString().c_str(),
- e.what());
+ LOG(warning, "Failed to start database for document type '%s'; %s",
+ docTypeName.toString().c_str(), e.what());
return DocumentDB::SP();
}
// Wait for replay done on document dbs added due to reconfigs, since engines are already up and running.
@@ -577,14 +583,12 @@ Proton::addDocumentDB(const document::DocumentType &docType,
_metricsEngine->addDocumentDBMetrics(ret->getMetricsCollection());
_metricsEngine->addMetricsHook(ret->getMetricsUpdateHook());
_documentDBMap[docTypeName] = ret;
- if (_persistenceEngine.get() != NULL) {
+ if (_persistenceEngine) {
// Not allowed to get to service layer to call pause().
std::unique_lock<std::shared_timed_mutex> persistenceWGuard(_persistenceEngine->getWLock());
- PersistenceHandlerProxy::SP
- persistenceHandler(new PersistenceHandlerProxy(ret));
+ auto persistenceHandler = std::make_shared<PersistenceHandlerProxy>(ret);
if (!_isInitializing) {
- _persistenceEngine->
- propagateSavedClusterState(*persistenceHandler);
+ _persistenceEngine->propagateSavedClusterState(*persistenceHandler);
_persistenceEngine->populateInitialBucketDB(bucketSpace, *persistenceHandler);
}
// TODO: Fix race with new cluster state setting.
@@ -649,10 +653,7 @@ Proton::ping(MonitorRequest::UP request, MonitorClient & client)
BootstrapConfig::SP configSnapshot = getActiveConfigSnapshot();
const ProtonConfig &protonConfig = configSnapshot->getProtonConfig();
ret.partid = protonConfig.partition;
- if (_matchEngine->isOnline())
- ret.timestamp = 42; // change to flush caches on tld/qrs
- else
- ret.timestamp = 0;
+ ret.timestamp = (_matchEngine->isOnline()) ? 42 : 0;
ret.activeDocs = getNumActiveDocs();
ret.activeDocsRequested = request->reportActiveDocs;
return reply;
@@ -661,7 +662,7 @@ Proton::ping(MonitorRequest::UP request, MonitorClient & client)
bool
Proton::triggerFlush()
{
- if ((_flushEngine.get() == NULL) || ! _flushEngine->HasThread()) {
+ if (!_flushEngine || ! _flushEngine->HasThread()) {
return false;
}
_flushEngine->triggerFlush();
@@ -682,13 +683,12 @@ createPrepareRestartConfig(const ProtonConfig &protonConfig)
bool
Proton::prepareRestart()
{
- if ((_flushEngine.get() == NULL) || ! _flushEngine->HasThread()) {
+ if (!_flushEngine || ! _flushEngine->HasThread()) {
return false;
}
BootstrapConfig::SP configSnapshot = getActiveConfigSnapshot();
- IFlushStrategy::SP strategy =
- std::make_shared<PrepareRestartFlushStrategy>(
- createPrepareRestartConfig(configSnapshot->getProtonConfig()));
+ auto strategy = std::make_shared<PrepareRestartFlushStrategy>(
+ createPrepareRestartConfig(configSnapshot->getProtonConfig()));
_flushEngine->setStrategy(strategy);
return true;
}
@@ -700,7 +700,7 @@ int countOpenFiles()
static const char * const fd_dir_name = "/proc/self/fd";
int count = 0;
DIR *dp = opendir(fd_dir_name);
- if (dp != NULL) {
+ if (dp != nullptr) {
struct dirent *ptr;
while ((ptr = readdir(dp)) != nullptr) {
if (strcmp(".", ptr->d_name) == 0) continue;
@@ -738,10 +738,6 @@ Proton::updateMetrics(const vespalib::MonitorGuard &)
}
}
-namespace {
-const std::string config_id_tag = "CONFIG ID";
-} // namespace
-
void
Proton::waitForInitDone()
{
diff --git a/searchlib/src/apps/docstore/benchmarkdatastore.cpp b/searchlib/src/apps/docstore/benchmarkdatastore.cpp
index 3dcfbbb31cc..620a139d451 100644
--- a/searchlib/src/apps/docstore/benchmarkdatastore.cpp
+++ b/searchlib/src/apps/docstore/benchmarkdatastore.cpp
@@ -99,7 +99,7 @@ BenchmarkDataStoreApp::benchmark(const vespalib::string & dir, size_t numReads,
tuning._randRead.setWantMemoryMap();
}
search::index::DummyFileHeaderContext fileHeaderContext;
- vespalib::ThreadStackExecutor executor(config.getNumThreads(), 128*1024);
+ vespalib::ThreadStackExecutor executor(1, 128*1024);
transactionlog::NoSyncProxy noTlSyncer;
LogDataStore store(executor, dir, config, growStrategy, tuning,
fileHeaderContext,
diff --git a/searchlib/src/apps/docstore/documentstoreinspect.cpp b/searchlib/src/apps/docstore/documentstoreinspect.cpp
index b7f202f1209..40f603c3da1 100644
--- a/searchlib/src/apps/docstore/documentstoreinspect.cpp
+++ b/searchlib/src/apps/docstore/documentstoreinspect.cpp
@@ -105,7 +105,7 @@ DocumentStoreInspectApp::verify(const vespalib::string & dir)
GrowStrategy growStrategy;
TuneFileSummary tuning;
search::index::DummyFileHeaderContext fileHeaderContext;
- vespalib::ThreadStackExecutor executor(config.getNumThreads(), 128*1024);
+ vespalib::ThreadStackExecutor executor(1, 128*1024);
transactionlog::NoSyncProxy noTlSyncer;
LogDataStore store(executor, dir, config, growStrategy, tuning,
diff --git a/searchlib/src/apps/docstore/verifylogdatastore.cpp b/searchlib/src/apps/docstore/verifylogdatastore.cpp
index 1fb83248b5b..498516882c1 100644
--- a/searchlib/src/apps/docstore/verifylogdatastore.cpp
+++ b/searchlib/src/apps/docstore/verifylogdatastore.cpp
@@ -49,7 +49,7 @@ VerifyLogDataStoreApp::verify(const vespalib::string & dir)
GrowStrategy growStrategy;
TuneFileSummary tuning;
search::index::DummyFileHeaderContext fileHeaderContext;
- vespalib::ThreadStackExecutor executor(config.getNumThreads(), 128*1024);
+ vespalib::ThreadStackExecutor executor(1, 128*1024);
transactionlog::NoSyncProxy noTlSyncer;
try {
diff --git a/searchlib/src/apps/tests/biglogtest.cpp b/searchlib/src/apps/tests/biglogtest.cpp
index b5eb76d40a7..788fb855c7f 100644
--- a/searchlib/src/apps/tests/biglogtest.cpp
+++ b/searchlib/src/apps/tests/biglogtest.cpp
@@ -134,7 +134,7 @@ factory<LogDataStore>::factory(std::string dir)
: DioTune(),
_fileHeaderContext(),
_config(),
- _executor(_config.getNumThreads(), 128*1024),
+ _executor(1, 128*1024),
_noTlSyncer(),
_datastore(_executor, dir, _config, GrowStrategy(), tuning, _fileHeaderContext, _noTlSyncer, NULL)
{}
diff --git a/searchlib/src/tests/docstore/document_store/document_store_test.cpp b/searchlib/src/tests/docstore/document_store/document_store_test.cpp
index b48d754bfb5..2ab24a00557 100644
--- a/searchlib/src/tests/docstore/document_store/document_store_test.cpp
+++ b/searchlib/src/tests/docstore/document_store/document_store_test.cpp
@@ -77,7 +77,7 @@ TEST("require that LogDocumentStore::Config equality operator detects inequality
EXPECT_TRUE(C() == C());
EXPECT_FALSE(C() != C());
EXPECT_FALSE(C(DC(CompressionConfig::NONE, 100000, 100), LC()) == C());
- EXPECT_FALSE(C(DC(), LC().setNumThreads(7)) == C());
+ EXPECT_FALSE(C(DC(), LC().setMaxBucketSpread(7)) == C());
}
TEST_MAIN() { TEST_RUN_ALL(); }
diff --git a/searchlib/src/tests/docstore/document_store_visitor/document_store_visitor_test.cpp b/searchlib/src/tests/docstore/document_store_visitor/document_store_visitor_test.cpp
index 7e4b4fa4363..247ee134854 100644
--- a/searchlib/src/tests/docstore/document_store_visitor/document_store_visitor_test.cpp
+++ b/searchlib/src/tests/docstore/document_store_visitor/document_store_visitor_test.cpp
@@ -234,9 +234,9 @@ Fixture::Fixture()
: _baseDir("visitor"),
_repo(makeDocTypeRepoConfig()),
_storeConfig(DocumentStore::Config(CompressionConfig::NONE, 0, 0),
- LogDataStore::Config().setMaxFileSize(50000).setMaxBucketSpread(3.0).setNumThreads(1)
+ LogDataStore::Config().setMaxFileSize(50000).setMaxBucketSpread(3.0)
.setFileConfig(WriteableFileChunk::Config(CompressionConfig(), 16384))),
- _executor(_storeConfig.getLogConfig().getNumThreads(), 128 * 1024),
+ _executor(1, 128 * 1024),
_fileHeaderContext(),
_tlSyncer(),
_store(),
diff --git a/searchlib/src/tests/docstore/logdatastore/logdatastore_test.cpp b/searchlib/src/tests/docstore/logdatastore/logdatastore_test.cpp
index c7da0c17c80..ed99003c2f7 100644
--- a/searchlib/src/tests/docstore/logdatastore/logdatastore_test.cpp
+++ b/searchlib/src/tests/docstore/logdatastore/logdatastore_test.cpp
@@ -205,9 +205,9 @@ TEST("testGrowing") {
LogDataStore::Config config; //(100000, 0.1, 3.0, 0.2, 8, true, CompressionConfig::LZ4,
// WriteableFileChunk::Config(CompressionConfig(CompressionConfig::LZ4, 9, 60), 1000));
config.setMaxFileSize(100000).setMaxDiskBloatFactor(0.1).setMaxBucketSpread(3.0).setMinFileSizeFactor(0.2)
- .setNumThreads(8).compact2ActiveFile(true).compactCompression({CompressionConfig::LZ4})
+ .compact2ActiveFile(true).compactCompression({CompressionConfig::LZ4})
.setFileConfig({{CompressionConfig::LZ4, 9, 60}, 1000});
- vespalib::ThreadStackExecutor executor(config.getNumThreads(), 128*1024);
+ vespalib::ThreadStackExecutor executor(8, 128*1024);
DummyFileHeaderContext fileHeaderContext;
MyTlSyncer tlSyncer;
{
@@ -276,30 +276,26 @@ void fetchAndTest(IDataStore & datastore, uint32_t lid, const void *a, size_t sz
TEST("testTruncatedIdxFile"){
LogDataStore::Config config;
DummyFileHeaderContext fileHeaderContext;
- vespalib::ThreadStackExecutor executor(config.getNumThreads(), 128*1024);
+ vespalib::ThreadStackExecutor executor(1, 128*1024);
MyTlSyncer tlSyncer;
{
// Files comes from the 'growing test'.
- LogDataStore datastore(executor,
- TEST_PATH("bug-7257706"), config,
- GrowStrategy(), TuneFileSummary(),
- fileHeaderContext, tlSyncer, NULL);
+ LogDataStore datastore(executor, TEST_PATH("bug-7257706"), config, GrowStrategy(),
+ TuneFileSummary(), fileHeaderContext, tlSyncer, NULL);
EXPECT_EQUAL(354ul, datastore.lastSyncToken());
}
const char * magic = "mumbo jumbo";
{
- LogDataStore datastore(executor, "bug-7257706-truncated", config,
- GrowStrategy(), TuneFileSummary(),
- fileHeaderContext, tlSyncer, NULL);
+ LogDataStore datastore(executor, "bug-7257706-truncated", config, GrowStrategy(),
+ TuneFileSummary(), fileHeaderContext, tlSyncer, NULL);
EXPECT_EQUAL(331ul, datastore.lastSyncToken());
datastore.write(332, 7, magic, strlen(magic));
datastore.write(333, 8, magic, strlen(magic));
datastore.flush(datastore.initFlush(334));
}
{
- LogDataStore datastore(executor, "bug-7257706-truncated", config,
- GrowStrategy(), TuneFileSummary(),
- fileHeaderContext, tlSyncer, NULL);
+ LogDataStore datastore(executor, "bug-7257706-truncated", config, GrowStrategy(),
+ TuneFileSummary(), fileHeaderContext, tlSyncer, NULL);
EXPECT_EQUAL(334ul, datastore.lastSyncToken());
}
}
@@ -307,7 +303,7 @@ TEST("testTruncatedIdxFile"){
TEST("testThatEmptyIdxFilesAndDanglingDatFilesAreRemoved") {
LogDataStore::Config config;
DummyFileHeaderContext fileHeaderContext;
- vespalib::ThreadStackExecutor executor(config.getNumThreads(), 128*1024);
+ vespalib::ThreadStackExecutor executor(1, 128*1024);
MyTlSyncer tlSyncer;
LogDataStore datastore(executor, "dangling-test", config,
GrowStrategy(), TuneFileSummary(),
@@ -320,7 +316,7 @@ TEST("testThatEmptyIdxFilesAndDanglingDatFilesAreRemoved") {
TEST("testThatIncompleteCompactedFilesAreRemoved") {
LogDataStore::Config config;
DummyFileHeaderContext fileHeaderContext;
- vespalib::ThreadStackExecutor executor(config.getNumThreads(), 128*1024);
+ vespalib::ThreadStackExecutor executor(1, 128*1024);
MyTlSyncer tlSyncer;
LogDataStore datastore(executor, "incompletecompact-test", config,
GrowStrategy(), TuneFileSummary(),
@@ -340,11 +336,10 @@ public:
_myDir("visitcache"),
_config(),
_fileHeaderContext(),
- _executor(_config.getNumThreads(), 128*1024),
+ _executor(1, 128*1024),
_tlSyncer(),
- _datastore(_executor, _myDir.getDir(), _config,
- GrowStrategy(), TuneFileSummary(),
- _fileHeaderContext, _tlSyncer, NULL)
+ _datastore(_executor, _myDir.getDir(), _config, GrowStrategy(),
+ TuneFileSummary(), _fileHeaderContext, _tlSyncer, NULL)
{ }
~VisitStore();
IDataStore & getStore() { return _datastore; }
@@ -518,10 +513,10 @@ VisitCacheStore::VisitCacheStore() :
_myDir("visitcache"),
_repo(makeDocTypeRepoConfig()),
_config(DocumentStore::Config(CompressionConfig::LZ4, 1000000, 0).allowVisitCaching(true),
- LogDataStore::Config().setMaxFileSize(50000).setMaxBucketSpread(3.0).setNumThreads(1)
+ LogDataStore::Config().setMaxFileSize(50000).setMaxBucketSpread(3.0)
.setFileConfig(WriteableFileChunk::Config(CompressionConfig(), 16384))),
_fileHeaderContext(),
- _executor(_config.getLogConfig().getNumThreads(), 128*1024),
+ _executor(1, 128*1024),
_tlSyncer(),
_datastore(_executor, _myDir.getDir(), _config, GrowStrategy(),
TuneFileSummary(), _fileHeaderContext, _tlSyncer, nullptr),
@@ -596,11 +591,10 @@ TEST("testWriteRead") {
{
EXPECT_TRUE(FastOS_File::MakeDirectory("empty"));
DummyFileHeaderContext fileHeaderContext;
- vespalib::ThreadStackExecutor executor(config.getNumThreads(), 128*1024);
+ vespalib::ThreadStackExecutor executor(1, 128*1024);
MyTlSyncer tlSyncer;
- LogDataStore datastore(executor, "empty", config,
- GrowStrategy(), TuneFileSummary(),
- fileHeaderContext, tlSyncer, NULL);
+ LogDataStore datastore(executor, "empty", config, GrowStrategy(),
+ TuneFileSummary(), fileHeaderContext, tlSyncer, NULL);
ASSERT_TRUE(datastore.lastSyncToken() == 0);
size_t headerFootprint = datastore.getDiskHeaderFootprint();
EXPECT_LESS(0u, headerFootprint);
@@ -633,7 +627,7 @@ TEST("testWriteRead") {
}
{
DummyFileHeaderContext fileHeaderContext;
- vespalib::ThreadStackExecutor executor(config.getNumThreads(), 128*1024);
+ vespalib::ThreadStackExecutor executor(1, 128*1024);
MyTlSyncer tlSyncer;
LogDataStore datastore(executor, "empty", config,
GrowStrategy(), TuneFileSummary(),
@@ -684,7 +678,7 @@ TEST("requireThatFlushTimeIsAvailableAfterFlush") {
fastos::TimeStamp before(fastos::ClockSystem::now());
DummyFileHeaderContext fileHeaderContext;
LogDataStore::Config config;
- vespalib::ThreadStackExecutor executor(config.getNumThreads(), 128*1024);
+ vespalib::ThreadStackExecutor executor(1, 128*1024);
MyTlSyncer tlSyncer;
LogDataStore store(executor, testDir.getDir(), config, GrowStrategy(),
TuneFileSummary(), fileHeaderContext, tlSyncer, nullptr);
@@ -980,7 +974,6 @@ TEST("require that config equality operator detects inequality") {
EXPECT_FALSE(C() == C().setMaxDiskBloatFactor(0.3));
EXPECT_FALSE(C() == C().setMaxBucketSpread(0.3));
EXPECT_FALSE(C() == C().setMinFileSizeFactor(0.3));
- EXPECT_FALSE(C() == C().setNumThreads(3));
EXPECT_FALSE(C() == C().setFileConfig(WriteableFileChunk::Config({}, 70)));
EXPECT_FALSE(C() == C().disableCrcOnRead(true));
EXPECT_FALSE(C() == C().compact2ActiveFile(false));
diff --git a/searchlib/src/vespa/searchlib/docstore/logdatastore.cpp b/searchlib/src/vespa/searchlib/docstore/logdatastore.cpp
index 6c2509c53c5..4e3178fcdb1 100644
--- a/searchlib/src/vespa/searchlib/docstore/logdatastore.cpp
+++ b/searchlib/src/vespa/searchlib/docstore/logdatastore.cpp
@@ -34,7 +34,6 @@ LogDataStore::Config::Config()
_maxDiskBloatFactor(0.2),
_maxBucketSpread(2.5),
_minFileSizeFactor(0.2),
- _numThreads(8),
_skipCrcOnRead(false),
_compact2ActiveFile(true),
_compactCompression(CompressionConfig::LZ4),
@@ -49,7 +48,6 @@ LogDataStore::Config::operator == (const Config & rhs) const {
(_minFileSizeFactor == rhs._minFileSizeFactor) &&
(_compact2ActiveFile == rhs._compact2ActiveFile) &&
(_skipCrcOnRead == rhs._skipCrcOnRead) &&
- (_numThreads == rhs._numThreads) &&
(_compactCompression == rhs._compactCompression) &&
(_fileConfig == rhs._fileConfig);
}
diff --git a/searchlib/src/vespa/searchlib/docstore/logdatastore.h b/searchlib/src/vespa/searchlib/docstore/logdatastore.h
index 5928602e196..52d78cac2bd 100644
--- a/searchlib/src/vespa/searchlib/docstore/logdatastore.h
+++ b/searchlib/src/vespa/searchlib/docstore/logdatastore.h
@@ -43,7 +43,6 @@ public:
Config & setMaxBucketSpread(double v) { _maxBucketSpread = v; return *this; }
Config & setMinFileSizeFactor(double v) { _minFileSizeFactor = v; return *this; }
- Config & setNumThreads(size_t v) { _numThreads = v; return *this; }
Config & compactCompression(CompressionConfig v) { _compactCompression = v; return *this; }
Config & setFileConfig(WriteableFileChunk::Config v) { _fileConfig = v; return *this; }
@@ -52,7 +51,6 @@ public:
double getMaxBucketSpread() const { return _maxBucketSpread; }
double getMinFileSizeFactor() const { return _minFileSizeFactor; }
- size_t getNumThreads() const { return _numThreads; }
bool crcOnReadDisabled() const { return _skipCrcOnRead; }
bool compact2ActiveFile() const { return _compact2ActiveFile; }
const CompressionConfig & compactCompression() const { return _compactCompression; }
@@ -67,7 +65,6 @@ public:
double _maxDiskBloatFactor;
double _maxBucketSpread;
double _minFileSizeFactor;
- size_t _numThreads;
bool _skipCrcOnRead;
bool _compact2ActiveFile;
CompressionConfig _compactCompression;