From e2345fd577213a86481f275bc88199c3390c1af0 Mon Sep 17 00:00:00 2001 From: Harald Musum Date: Fri, 11 Sep 2020 11:45:31 +0200 Subject: Move files in C++ modules back to original location --- .../src/vespa/all-clusters-bucket-spaces.def | 9 + configdefinitions/src/vespa/application-id.def | 8 + .../src/vespa/athenz-provider-service.def | 29 ++ configdefinitions/src/vespa/attributes.def | 45 ++ configdefinitions/src/vespa/bucketspaces.def | 12 + .../src/vespa/cloud.config.application-id.def | 8 - .../src/vespa/cloud.config.cluster-info.def | 12 - .../src/vespa/cloud.config.cluster-list.def | 5 - .../src/vespa/cloud.config.configserver.def | 69 --- ...loud.config.filedistribution.filereferences.def | 3 - .../src/vespa/cloud.config.lb-services.def | 20 - .../src/vespa/cloud.config.logforwarder.def | 9 - configdefinitions/src/vespa/cloud.config.model.def | 13 - .../src/vespa/cloud.config.routing-provider.def | 9 - .../src/vespa/cloud.config.sentinel.def | 50 -- .../src/vespa/cloud.config.slobroks.def | 13 - configdefinitions/src/vespa/cloud.config.ymon.def | 6 - .../src/vespa/cloud.config.zookeeper-server.def | 45 -- .../src/vespa/cloud.config.zookeepers.def | 3 - configdefinitions/src/vespa/cluster-info.def | 12 + configdefinitions/src/vespa/cluster-list.def | 5 + configdefinitions/src/vespa/configserver.def | 69 +++ configdefinitions/src/vespa/dispatch.def | 73 +++ configdefinitions/src/vespa/filereferences.def | 3 + configdefinitions/src/vespa/fleetcontroller.def | 183 ++++++++ configdefinitions/src/vespa/ilscripts.def | 10 + configdefinitions/src/vespa/imported-fields.def | 11 + configdefinitions/src/vespa/indexschema.def | 25 + configdefinitions/src/vespa/lb-services.def | 20 + configdefinitions/src/vespa/load-type.def | 30 ++ configdefinitions/src/vespa/logforwarder.def | 9 + .../src/vespa/messagetyperouteselectorpolicy.def | 11 + configdefinitions/src/vespa/model.def | 13 + configdefinitions/src/vespa/orchestrator.def | 5 + configdefinitions/src/vespa/persistence.def | 60 +++ configdefinitions/src/vespa/rank-profiles.def | 12 + configdefinitions/src/vespa/routing-provider.def | 9 + configdefinitions/src/vespa/sentinel.def | 50 ++ configdefinitions/src/vespa/slobroks.def | 13 + configdefinitions/src/vespa/specialtokens.def | 32 ++ configdefinitions/src/vespa/stateserver.def | 5 + configdefinitions/src/vespa/stor-distribution.def | 70 +++ configdefinitions/src/vespa/stor-filestor.def | 60 +++ configdefinitions/src/vespa/summary.def | 8 + configdefinitions/src/vespa/summarymap.def | 15 + configdefinitions/src/vespa/upgrading.def | 24 + ...a.config.content.all-clusters-bucket-spaces.def | 9 - .../vespa.config.content.core.bucketspaces.def | 12 - .../vespa/vespa.config.content.fleetcontroller.def | 183 -------- .../src/vespa/vespa.config.content.load-type.def | 30 -- ...nfig.content.messagetyperouteselectorpolicy.def | 11 - .../src/vespa/vespa.config.content.persistence.def | 60 --- .../vespa.config.content.stor-distribution.def | 70 --- .../vespa/vespa.config.content.stor-filestor.def | 60 --- .../src/vespa/vespa.config.content.upgrading.def | 24 - .../src/vespa/vespa.config.core.stateserver.def | 5 - .../src/vespa/vespa.config.search.attributes.def | 45 -- .../src/vespa/vespa.config.search.dispatch.def | 73 --- .../vespa/vespa.config.search.imported-fields.def | 11 - .../src/vespa/vespa.config.search.indexschema.def | 25 - .../vespa/vespa.config.search.rank-profiles.def | 12 - .../src/vespa/vespa.config.search.summary.def | 8 - .../src/vespa/vespa.config.search.summarymap.def | 15 - .../src/vespa/vespa.configdefinition.ilscripts.def | 10 - .../vespa/vespa.configdefinition.specialtokens.def | 32 -- ...viderservice.config.athenz-provider-service.def | 29 -- .../vespa.orchestrator.config.orchestrator.def | 5 - configdefinitions/src/vespa/ymon.def | 6 + configdefinitions/src/vespa/zookeeper-server.def | 45 ++ configdefinitions/src/vespa/zookeepers.def | 3 + .../config/document.config.documentmanager.def | 106 ----- .../document/config/document.documenttypes.def | 109 ----- .../src/vespa/document/config/documentmanager.def | 106 +++++ .../src/vespa/document/config/documenttypes.def | 109 +++++ ....config.filedistribution.filedistributorrpc.def | 1 - .../src/vespa/fileacquirer/filedistributorrpc.def | 1 + .../src/vespa/metrics/metrics.metricsmanager.def | 63 --- metrics/src/vespa/metrics/metricsmanager.def | 63 +++ .../src/apps/verify_ranksetup/verify-ranksetup.def | 5 + .../vespa.config.search.core.verify-ranksetup.def | 5 - .../documentdb/fileconfigmanager/config.mycfg.def | 4 - .../proton/documentdb/fileconfigmanager/mycfg.def | 4 + .../src/vespa/searchcore/config/fdispatchrc.def | 5 + searchcore/src/vespa/searchcore/config/hwinfo.def | 8 + .../src/vespa/searchcore/config/onnx-models.def | 5 + searchcore/src/vespa/searchcore/config/proton.def | 502 +++++++++++++++++++++ .../vespa/searchcore/config/ranking-constants.def | 6 + .../vespa.config.search.core.fdispatchrc.def | 5 - .../config/vespa.config.search.core.hwinfo.def | 8 - .../vespa.config.search.core.onnx-models.def | 5 - .../config/vespa.config.search.core.proton.def | 502 --------------------- .../vespa.config.search.core.ranking-constants.def | 6 - .../searchlib/config/searchlib.translogserver.def | 38 -- .../src/vespa/searchlib/config/translogserver.def | 38 ++ .../src/vespa/searchsummary/config/juniperrc.def | 79 ++++ .../vespa.config.search.summary.juniperrc.def | 79 ---- .../vespa/storage/bucketdb/stor-bucket-init.def | 35 ++ .../src/vespa/storage/bucketdb/stor-bucketdb.def | 9 + .../vespa.config.content.core.stor-bucket-init.def | 35 -- .../vespa.config.content.core.stor-bucketdb.def | 9 - storage/src/vespa/storage/config/rpc-provider.def | 4 + storage/src/vespa/storage/config/stor-bouncer.def | 30 ++ .../src/vespa/storage/config/stor-bucketmover.def | 37 ++ .../storage/config/stor-communicationmanager.def | 57 +++ .../storage/config/stor-distributormanager.def | 246 ++++++++++ .../vespa/storage/config/stor-integritychecker.def | 38 ++ .../vespa/storage/config/stor-messageforwarder.def | 4 + .../src/vespa/storage/config/stor-opslogger.def | 4 + .../vespa/storage/config/stor-prioritymapping.def | 20 + storage/src/vespa/storage/config/stor-server.def | 90 ++++ storage/src/vespa/storage/config/stor-status.def | 4 + .../storage/config/stor-visitordispatcher.def | 19 + .../vespa.config.content.core.rpc-provider.def | 4 - .../vespa.config.content.core.stor-bouncer.def | 30 -- .../vespa.config.content.core.stor-bucketmover.def | 37 -- ...nfig.content.core.stor-communicationmanager.def | 57 --- ...config.content.core.stor-distributormanager.def | 246 ---------- ...a.config.content.core.stor-integritychecker.def | 38 -- ...a.config.content.core.stor-messageforwarder.def | 4 - .../vespa.config.content.core.stor-opslogger.def | 4 - ...pa.config.content.core.stor-prioritymapping.def | 20 - .../vespa.config.content.core.stor-server.def | 90 ---- .../vespa.config.content.core.stor-status.def | 4 - ....config.content.core.stor-visitordispatcher.def | 19 - .../src/vespa/storage/visiting/stor-visitor.def | 63 +++ .../vespa.config.content.core.stor-visitor.def | 63 --- .../vsm/config/vespa.config.search.vsm.vsm.def | 13 - .../config/vespa.config.search.vsm.vsmfields.def | 31 -- .../config/vespa.config.search.vsm.vsmsummary.def | 21 - vsm/src/vespa/vsm/config/vsm.def | 13 + vsm/src/vespa/vsm/config/vsmfields.def | 31 ++ vsm/src/vespa/vsm/config/vsmsummary.def | 21 + 132 files changed, 2650 insertions(+), 2650 deletions(-) create mode 100644 configdefinitions/src/vespa/all-clusters-bucket-spaces.def create mode 100644 configdefinitions/src/vespa/application-id.def create mode 100644 configdefinitions/src/vespa/athenz-provider-service.def create mode 100644 configdefinitions/src/vespa/attributes.def create mode 100644 configdefinitions/src/vespa/bucketspaces.def delete mode 100644 configdefinitions/src/vespa/cloud.config.application-id.def delete mode 100644 configdefinitions/src/vespa/cloud.config.cluster-info.def delete mode 100644 configdefinitions/src/vespa/cloud.config.cluster-list.def delete mode 100644 configdefinitions/src/vespa/cloud.config.configserver.def delete mode 100644 configdefinitions/src/vespa/cloud.config.filedistribution.filereferences.def delete mode 100644 configdefinitions/src/vespa/cloud.config.lb-services.def delete mode 100644 configdefinitions/src/vespa/cloud.config.logforwarder.def delete mode 100644 configdefinitions/src/vespa/cloud.config.model.def delete mode 100755 configdefinitions/src/vespa/cloud.config.routing-provider.def delete mode 100644 configdefinitions/src/vespa/cloud.config.sentinel.def delete mode 100644 configdefinitions/src/vespa/cloud.config.slobroks.def delete mode 100644 configdefinitions/src/vespa/cloud.config.ymon.def delete mode 100644 configdefinitions/src/vespa/cloud.config.zookeeper-server.def delete mode 100644 configdefinitions/src/vespa/cloud.config.zookeepers.def create mode 100644 configdefinitions/src/vespa/cluster-info.def create mode 100644 configdefinitions/src/vespa/cluster-list.def create mode 100644 configdefinitions/src/vespa/configserver.def create mode 100644 configdefinitions/src/vespa/dispatch.def create mode 100644 configdefinitions/src/vespa/filereferences.def create mode 100644 configdefinitions/src/vespa/fleetcontroller.def create mode 100644 configdefinitions/src/vespa/ilscripts.def create mode 100644 configdefinitions/src/vespa/imported-fields.def create mode 100644 configdefinitions/src/vespa/indexschema.def create mode 100644 configdefinitions/src/vespa/lb-services.def create mode 100644 configdefinitions/src/vespa/load-type.def create mode 100644 configdefinitions/src/vespa/logforwarder.def create mode 100644 configdefinitions/src/vespa/messagetyperouteselectorpolicy.def create mode 100644 configdefinitions/src/vespa/model.def create mode 100644 configdefinitions/src/vespa/orchestrator.def create mode 100644 configdefinitions/src/vespa/persistence.def create mode 100644 configdefinitions/src/vespa/rank-profiles.def create mode 100755 configdefinitions/src/vespa/routing-provider.def create mode 100644 configdefinitions/src/vespa/sentinel.def create mode 100644 configdefinitions/src/vespa/slobroks.def create mode 100644 configdefinitions/src/vespa/specialtokens.def create mode 100644 configdefinitions/src/vespa/stateserver.def create mode 100644 configdefinitions/src/vespa/stor-distribution.def create mode 100644 configdefinitions/src/vespa/stor-filestor.def create mode 100644 configdefinitions/src/vespa/summary.def create mode 100644 configdefinitions/src/vespa/summarymap.def create mode 100644 configdefinitions/src/vespa/upgrading.def delete mode 100644 configdefinitions/src/vespa/vespa.config.content.all-clusters-bucket-spaces.def delete mode 100644 configdefinitions/src/vespa/vespa.config.content.core.bucketspaces.def delete mode 100644 configdefinitions/src/vespa/vespa.config.content.fleetcontroller.def delete mode 100644 configdefinitions/src/vespa/vespa.config.content.load-type.def delete mode 100644 configdefinitions/src/vespa/vespa.config.content.messagetyperouteselectorpolicy.def delete mode 100644 configdefinitions/src/vespa/vespa.config.content.persistence.def delete mode 100644 configdefinitions/src/vespa/vespa.config.content.stor-distribution.def delete mode 100644 configdefinitions/src/vespa/vespa.config.content.stor-filestor.def delete mode 100644 configdefinitions/src/vespa/vespa.config.content.upgrading.def delete mode 100644 configdefinitions/src/vespa/vespa.config.core.stateserver.def delete mode 100644 configdefinitions/src/vespa/vespa.config.search.attributes.def delete mode 100644 configdefinitions/src/vespa/vespa.config.search.dispatch.def delete mode 100644 configdefinitions/src/vespa/vespa.config.search.imported-fields.def delete mode 100644 configdefinitions/src/vespa/vespa.config.search.indexschema.def delete mode 100644 configdefinitions/src/vespa/vespa.config.search.rank-profiles.def delete mode 100644 configdefinitions/src/vespa/vespa.config.search.summary.def delete mode 100644 configdefinitions/src/vespa/vespa.config.search.summarymap.def delete mode 100644 configdefinitions/src/vespa/vespa.configdefinition.ilscripts.def delete mode 100644 configdefinitions/src/vespa/vespa.configdefinition.specialtokens.def delete mode 100644 configdefinitions/src/vespa/vespa.hosted.athenz.instanceproviderservice.config.athenz-provider-service.def delete mode 100644 configdefinitions/src/vespa/vespa.orchestrator.config.orchestrator.def create mode 100644 configdefinitions/src/vespa/ymon.def create mode 100644 configdefinitions/src/vespa/zookeeper-server.def create mode 100644 configdefinitions/src/vespa/zookeepers.def delete mode 100644 document/src/vespa/document/config/document.config.documentmanager.def delete mode 100644 document/src/vespa/document/config/document.documenttypes.def create mode 100644 document/src/vespa/document/config/documentmanager.def create mode 100644 document/src/vespa/document/config/documenttypes.def delete mode 120000 fileacquirer/src/vespa/fileacquirer/cloud.config.filedistribution.filedistributorrpc.def create mode 120000 fileacquirer/src/vespa/fileacquirer/filedistributorrpc.def delete mode 100644 metrics/src/vespa/metrics/metrics.metricsmanager.def create mode 100644 metrics/src/vespa/metrics/metricsmanager.def create mode 100644 searchcore/src/apps/verify_ranksetup/verify-ranksetup.def delete mode 100644 searchcore/src/apps/verify_ranksetup/vespa.config.search.core.verify-ranksetup.def delete mode 100644 searchcore/src/tests/proton/documentdb/fileconfigmanager/config.mycfg.def create mode 100644 searchcore/src/tests/proton/documentdb/fileconfigmanager/mycfg.def create mode 100644 searchcore/src/vespa/searchcore/config/fdispatchrc.def create mode 100644 searchcore/src/vespa/searchcore/config/hwinfo.def create mode 100644 searchcore/src/vespa/searchcore/config/onnx-models.def create mode 100644 searchcore/src/vespa/searchcore/config/proton.def create mode 100644 searchcore/src/vespa/searchcore/config/ranking-constants.def delete mode 100644 searchcore/src/vespa/searchcore/config/vespa.config.search.core.fdispatchrc.def delete mode 100644 searchcore/src/vespa/searchcore/config/vespa.config.search.core.hwinfo.def delete mode 100644 searchcore/src/vespa/searchcore/config/vespa.config.search.core.onnx-models.def delete mode 100644 searchcore/src/vespa/searchcore/config/vespa.config.search.core.proton.def delete mode 100644 searchcore/src/vespa/searchcore/config/vespa.config.search.core.ranking-constants.def delete mode 100644 searchlib/src/vespa/searchlib/config/searchlib.translogserver.def create mode 100644 searchlib/src/vespa/searchlib/config/translogserver.def create mode 100644 searchsummary/src/vespa/searchsummary/config/juniperrc.def delete mode 100644 searchsummary/src/vespa/searchsummary/config/vespa.config.search.summary.juniperrc.def create mode 100644 storage/src/vespa/storage/bucketdb/stor-bucket-init.def create mode 100644 storage/src/vespa/storage/bucketdb/stor-bucketdb.def delete mode 100644 storage/src/vespa/storage/bucketdb/vespa.config.content.core.stor-bucket-init.def delete mode 100644 storage/src/vespa/storage/bucketdb/vespa.config.content.core.stor-bucketdb.def create mode 100644 storage/src/vespa/storage/config/rpc-provider.def create mode 100644 storage/src/vespa/storage/config/stor-bouncer.def create mode 100644 storage/src/vespa/storage/config/stor-bucketmover.def create mode 100644 storage/src/vespa/storage/config/stor-communicationmanager.def create mode 100644 storage/src/vespa/storage/config/stor-distributormanager.def create mode 100644 storage/src/vespa/storage/config/stor-integritychecker.def create mode 100644 storage/src/vespa/storage/config/stor-messageforwarder.def create mode 100644 storage/src/vespa/storage/config/stor-opslogger.def create mode 100644 storage/src/vespa/storage/config/stor-prioritymapping.def create mode 100644 storage/src/vespa/storage/config/stor-server.def create mode 100644 storage/src/vespa/storage/config/stor-status.def create mode 100644 storage/src/vespa/storage/config/stor-visitordispatcher.def delete mode 100644 storage/src/vespa/storage/config/vespa.config.content.core.rpc-provider.def delete mode 100644 storage/src/vespa/storage/config/vespa.config.content.core.stor-bouncer.def delete mode 100644 storage/src/vespa/storage/config/vespa.config.content.core.stor-bucketmover.def delete mode 100644 storage/src/vespa/storage/config/vespa.config.content.core.stor-communicationmanager.def delete mode 100644 storage/src/vespa/storage/config/vespa.config.content.core.stor-distributormanager.def delete mode 100644 storage/src/vespa/storage/config/vespa.config.content.core.stor-integritychecker.def delete mode 100644 storage/src/vespa/storage/config/vespa.config.content.core.stor-messageforwarder.def delete mode 100644 storage/src/vespa/storage/config/vespa.config.content.core.stor-opslogger.def delete mode 100644 storage/src/vespa/storage/config/vespa.config.content.core.stor-prioritymapping.def delete mode 100644 storage/src/vespa/storage/config/vespa.config.content.core.stor-server.def delete mode 100644 storage/src/vespa/storage/config/vespa.config.content.core.stor-status.def delete mode 100644 storage/src/vespa/storage/config/vespa.config.content.core.stor-visitordispatcher.def create mode 100644 storage/src/vespa/storage/visiting/stor-visitor.def delete mode 100644 storage/src/vespa/storage/visiting/vespa.config.content.core.stor-visitor.def delete mode 100644 vsm/src/vespa/vsm/config/vespa.config.search.vsm.vsm.def delete mode 100644 vsm/src/vespa/vsm/config/vespa.config.search.vsm.vsmfields.def delete mode 100644 vsm/src/vespa/vsm/config/vespa.config.search.vsm.vsmsummary.def create mode 100644 vsm/src/vespa/vsm/config/vsm.def create mode 100644 vsm/src/vespa/vsm/config/vsmfields.def create mode 100644 vsm/src/vespa/vsm/config/vsmsummary.def diff --git a/configdefinitions/src/vespa/all-clusters-bucket-spaces.def b/configdefinitions/src/vespa/all-clusters-bucket-spaces.def new file mode 100644 index 00000000000..7676bc2a03f --- /dev/null +++ b/configdefinitions/src/vespa/all-clusters-bucket-spaces.def @@ -0,0 +1,9 @@ +# Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +namespace=vespa.config.content + +## This config contains the document types handled by all content clusters +## and the bucket spaces they belong to. + +## The bucket space a document type in a particular cluster belongs to. +cluster{}.documentType{}.bucketSpace string + diff --git a/configdefinitions/src/vespa/application-id.def b/configdefinitions/src/vespa/application-id.def new file mode 100644 index 00000000000..72f69f48519 --- /dev/null +++ b/configdefinitions/src/vespa/application-id.def @@ -0,0 +1,8 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +# Application Id config (tenant, application, instance) + +namespace=cloud.config + +tenant string default="(unknowntenant)" +application string default="(unknownapplication)" +instance string default="(unknowninstance)" diff --git a/configdefinitions/src/vespa/athenz-provider-service.def b/configdefinitions/src/vespa/athenz-provider-service.def new file mode 100644 index 00000000000..bd929cb17d4 --- /dev/null +++ b/configdefinitions/src/vespa/athenz-provider-service.def @@ -0,0 +1,29 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +namespace=vespa.hosted.athenz.instanceproviderservice.config + +# Athenz domain +domain string + +# Athenz service name +serviceName string + +# Secret name of private Key +secretName string + +# Secret version +secretVersion int + +# Certificate DNS suffix +certDnsSuffix string + +# Athenz ZTS server url +ztsUrl string + +# Path to Athenz CA JKS trust store +athenzCaTrustStore string + +# Period between certificate updates +updatePeriodDays int default=1 + +# Tenant Service id +tenantService string default=vespa.vespa.tenant \ No newline at end of file diff --git a/configdefinitions/src/vespa/attributes.def b/configdefinitions/src/vespa/attributes.def new file mode 100644 index 00000000000..6c69d71fdf6 --- /dev/null +++ b/configdefinitions/src/vespa/attributes.def @@ -0,0 +1,45 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +namespace=vespa.config.search + +attribute[].name string +attribute[].datatype enum { STRING, BOOL, UINT2, UINT4, INT8, INT16, INT32, INT64, FLOAT16, FLOAT, DOUBLE, PREDICATE, TENSOR, REFERENCE, NONE } default=NONE +attribute[].collectiontype enum { SINGLE, ARRAY, WEIGHTEDSET } default=SINGLE +attribute[].removeifzero bool default=false +attribute[].createifnonexistent bool default=false +attribute[].fastsearch bool default=false +attribute[].huge bool default=false +# An attribute marked mutable can be updated by a query. +attribute[].ismutable bool default=false +attribute[].sortascending bool default=true +attribute[].sortfunction enum { RAW, LOWERCASE, UCA } default=UCA +attribute[].sortstrength enum { PRIMARY, SECONDARY, TERTIARY, QUATERNARY, IDENTICAL } default=PRIMARY +attribute[].sortlocale string default="" +# Allow bitvector postings in addition to btree postings ? +attribute[].enablebitvectors bool default=false +# Allow only bitvector postings, i.e. drop btree postings to save memory.? +attribute[].enableonlybitvector bool default=false +# Allow fast access to this attribute at all times. +# If so, attribute is kept in memory also for non-searchable documents. +attribute[].fastaccess bool default=false +attribute[].arity int default=8 +attribute[].lowerbound long default=-9223372036854775808 +attribute[].upperbound long default=9223372036854775807 +# The threshold is given as a ratio of the corpus size +attribute[].densepostinglistthreshold double default=0.40 +# Specification of tensor type if this attribute is of type TENSOR. +attribute[].tensortype string default="" +# Whether this is an imported attribute (from parent document db) or not. +attribute[].imported bool default=false + +# The distance metric to use for nearest neighbor search. +# Is only used when the attribute is a 1-dimensional indexed tensor. +attribute[].distancemetric enum { EUCLIDEAN, ANGULAR, GEODEGREES, INNERPRODUCT, HAMMING } default=EUCLIDEAN + +# Configuration parameters for a hnsw index used together with a 1-dimensional indexed tensor for approximate nearest neighbor search. +attribute[].index.hnsw.enabled bool default=false +attribute[].index.hnsw.maxlinkspernode int default=16 +attribute[].index.hnsw.neighborstoexploreatinsert int default=200 +# Deprecated: Remove on Vespa 8 or before when possible. +attribute[].index.hnsw.distancemetric enum { EUCLIDEAN, ANGULAR, GEODEGREES, HAMMING } default=EUCLIDEAN +# Whether multi-threaded indexing is enabled for this hnsw index. +attribute[].index.hnsw.multithreadedindexing bool default=true diff --git a/configdefinitions/src/vespa/bucketspaces.def b/configdefinitions/src/vespa/bucketspaces.def new file mode 100644 index 00000000000..c9468850018 --- /dev/null +++ b/configdefinitions/src/vespa/bucketspaces.def @@ -0,0 +1,12 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +namespace=vespa.config.content.core + +## This config contains the document types handled by a given content cluster +## and the bucket space they belong to. + +## The name of a document type. +documenttype[].name string + +## The bucket space this document type belongs to. +documenttype[].bucketspace string + diff --git a/configdefinitions/src/vespa/cloud.config.application-id.def b/configdefinitions/src/vespa/cloud.config.application-id.def deleted file mode 100644 index 72f69f48519..00000000000 --- a/configdefinitions/src/vespa/cloud.config.application-id.def +++ /dev/null @@ -1,8 +0,0 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -# Application Id config (tenant, application, instance) - -namespace=cloud.config - -tenant string default="(unknowntenant)" -application string default="(unknownapplication)" -instance string default="(unknowninstance)" diff --git a/configdefinitions/src/vespa/cloud.config.cluster-info.def b/configdefinitions/src/vespa/cloud.config.cluster-info.def deleted file mode 100644 index f5e94ad61b3..00000000000 --- a/configdefinitions/src/vespa/cloud.config.cluster-info.def +++ /dev/null @@ -1,12 +0,0 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -# Information about the cluster the subscriber is a part of -namespace=cloud.config - -# clusterId is same as clustername in model -clusterId string -nodeCount int - -services[].index int default=-1 -services[].hostname string default="(nohostname)" -services[].ports[].number int default=-1 -services[].ports[].tags string default="(notags)" diff --git a/configdefinitions/src/vespa/cloud.config.cluster-list.def b/configdefinitions/src/vespa/cloud.config.cluster-list.def deleted file mode 100644 index ad6f9715eb9..00000000000 --- a/configdefinitions/src/vespa/cloud.config.cluster-list.def +++ /dev/null @@ -1,5 +0,0 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -namespace=cloud.config - -storage[].name string -storage[].configid string default="" diff --git a/configdefinitions/src/vespa/cloud.config.configserver.def b/configdefinitions/src/vespa/cloud.config.configserver.def deleted file mode 100644 index 7405f5f2d05..00000000000 --- a/configdefinitions/src/vespa/cloud.config.configserver.def +++ /dev/null @@ -1,69 +0,0 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -namespace=cloud.config - -# Ports -rpcport int default=19070 -httpport int default=19071 -# 0 means use the number of CPU cores available -numRpcThreads int default=0 - -# ZooKeeper -zookeeperserver[].hostname string -zookeeperserver[].port int default=2181 -# ZK write timeout in seconds, must be long enough to write application packages to other nodes -zookeeper.barrierTimeout long default=360 -zookeeperLocalhostAffinity bool default=true - -# Directories -configModelPluginDir[] string -configServerDBDir string default="var/db/vespa/config_server/serverdb/" -configDefinitionsDir string default="share/vespa/configdefinitions/" -fileReferencesDir string default="var/db/vespa/filedistribution/" - -# Misc -sessionLifetime long default=3600 # in seconds -masterGeneration long default=0 -multitenant bool default=false -numDelayedResponseThreads int default=1 -serverId string default="localhost" -hostedVespa bool default=false -numParallelTenantLoaders int default=4 - -# Configserver app -applicationDirectory string default="conf/configserver-app" - -# Zone information -cloud string default="default" -environment string default="prod" -region string default="default" -system string default="main" - -# RPC protocol -maxgetconfigclients int default=1000000 -maxoutputbuffersize int default=65536 -useVespaVersionInRequest bool default=false -payloadCompressionType enum { UNCOMPRESSED, LZ4 } default=LZ4 - -# Athenz config -loadBalancerAddress string default="" -athenzDnsSuffix string default="" -ztsUrl string default="" - -# Maintainers -maintainerIntervalMinutes int default=30 -keepUnusedFileReferencesHours int default=2 - -# Bootstrapping -# How long bootstrapping can take before giving up (in seconds) -maxDurationOfBootstrap long default=7200 -# How long to sleep before redeploying again if it fails (in seconds) -sleepTimeWhenRedeployingFails long default=30 - -# Features (to be overridden in configserver-config.xml if needed) -buildMinimalSetOfConfigModels bool default=true - -# Unused, remove in Vespa 8 -throwIfActiveSessionCannotBeLoaded bool default=true - -canReturnEmptySentinelConfig bool default=false -serverNodeType enum {config, controller} default=config diff --git a/configdefinitions/src/vespa/cloud.config.filedistribution.filereferences.def b/configdefinitions/src/vespa/cloud.config.filedistribution.filereferences.def deleted file mode 100644 index 03e071959ce..00000000000 --- a/configdefinitions/src/vespa/cloud.config.filedistribution.filereferences.def +++ /dev/null @@ -1,3 +0,0 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -namespace=cloud.config.filedistribution -filereferences[] string diff --git a/configdefinitions/src/vespa/cloud.config.lb-services.def b/configdefinitions/src/vespa/cloud.config.lb-services.def deleted file mode 100644 index 33c568061fe..00000000000 --- a/configdefinitions/src/vespa/cloud.config.lb-services.def +++ /dev/null @@ -1,20 +0,0 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -# Config for load balancer that needs to know about all services for -# all tenants and applications in a config server cluster - -namespace=cloud.config - -# Active rotation given as flag 'active' for a prod region in deployment.xml -# Default true for now (since code in config-model to set it is not ready yet), should have no default value -tenants{}.applications{}.activeRotation bool default=true - -tenants{}.applications{}.hosts{}.hostname string default="(unknownhostname)" -tenants{}.applications{}.hosts{}.services{}.type string default="(noservicetype)" -tenants{}.applications{}.hosts{}.services{}.clustertype string default="(unknownclustertype)" -tenants{}.applications{}.hosts{}.services{}.clustername string default="(unknownclustername)" -tenants{}.applications{}.hosts{}.services{}.configId string -tenants{}.applications{}.hosts{}.services{}.index int default=0 -tenants{}.applications{}.hosts{}.services{}.ports[].number int default=-1 -tenants{}.applications{}.hosts{}.services{}.ports[].tags string default="(notags)" -tenants{}.applications{}.hosts{}.services{}.servicealiases[] string -tenants{}.applications{}.hosts{}.services{}.endpointaliases[] string diff --git a/configdefinitions/src/vespa/cloud.config.logforwarder.def b/configdefinitions/src/vespa/cloud.config.logforwarder.def deleted file mode 100644 index ab56589298f..00000000000 --- a/configdefinitions/src/vespa/cloud.config.logforwarder.def +++ /dev/null @@ -1,9 +0,0 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -namespace=cloud.config - -# only splunk type config for now - -deploymentServer string default="" -clientName string default="" -splunkHome string default="/opt/splunkforwarder" -phoneHomeInterval int default=60 diff --git a/configdefinitions/src/vespa/cloud.config.model.def b/configdefinitions/src/vespa/cloud.config.model.def deleted file mode 100644 index 59c04cee143..00000000000 --- a/configdefinitions/src/vespa/cloud.config.model.def +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -namespace=cloud.config - -vespaVersion string default="unknown" -hosts[].name string default="(unknownhostname)" -hosts[].services[].name string default="(noservicename)" -hosts[].services[].type string default="(noservicetype)" -hosts[].services[].configid string default="(noconfigid)" -hosts[].services[].clustertype string default="(unknownclustertype)" -hosts[].services[].clustername string default="(unknownclustername)" -hosts[].services[].index int default=0 -hosts[].services[].ports[].number int default=-1 -hosts[].services[].ports[].tags string default="(notags)" diff --git a/configdefinitions/src/vespa/cloud.config.routing-provider.def b/configdefinitions/src/vespa/cloud.config.routing-provider.def deleted file mode 100755 index eedf4f3c86f..00000000000 --- a/configdefinitions/src/vespa/cloud.config.routing-provider.def +++ /dev/null @@ -1,9 +0,0 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -# Configuration for the routing config provider - -# TODO Remove once YCA filter is removed - -namespace=cloud.config - -# disabled by default, automatically enabled for hosted vespa -enabled bool default=false diff --git a/configdefinitions/src/vespa/cloud.config.sentinel.def b/configdefinitions/src/vespa/cloud.config.sentinel.def deleted file mode 100644 index d318a55cccf..00000000000 --- a/configdefinitions/src/vespa/cloud.config.sentinel.def +++ /dev/null @@ -1,50 +0,0 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -namespace=cloud.config - -## The port where sentinel should run its telnet interface -port.telnet int default=19098 - -## The port where sentinel should run its RPC interface -port.rpc int default=19097 - -## Meta info about the application running on this node. All values are -## exported to child processes as $VESPA_APPLICATION_TENANT, -## $VESPA_APPLICATION_NAME, $VESPA_APPLICATION_ENVIRONMENT, $VESPA_APPLICATION_REGION -## and $VESPA_APPLICATION_INSTANCE -application.tenant string default="default" -application.name string default="default" -application.environment string default="default" -application.instance string default="default" -application.region string default="default" - -## The command to run. This will be run by sh -c, and the following -## environment variables are defined: $ROOT, $VESPA_SERVICE_NAME, -## $VESPA_CONFIG_ID -## -## stdin points to /dev/null -## stdout and stderr connected to sentinel via pipe on startup. -service[].command string - -## The command to run before stopping service. The same properties as for -## startup command holds. -service[].preShutdownCommand string default="" - -## The service name for this service. Exported in VESPA_SERVICE_NAME -## environment variable. All services must have unique names. -## The service name is used by to start/stop services, and to list -## them. It is also logged in the service field in every log message. -service[].name string - -## Should the service be auto-started by configd as soon as it starts up, -## or must it be started manually? Almost always true. -service[].autostart bool default=false - -## Should the service automatically be restarted if it exits? -service[].autorestart bool default=true - -## Config reference to the service being started. -## Exported in the VESPA_CONFIG_ID environment variable. -service[].id reference - -## Affinity of this service towards a CPU socket -service[].affinity.cpuSocket int default=-1 diff --git a/configdefinitions/src/vespa/cloud.config.slobroks.def b/configdefinitions/src/vespa/cloud.config.slobroks.def deleted file mode 100644 index b4e5536d2df..00000000000 --- a/configdefinitions/src/vespa/cloud.config.slobroks.def +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -namespace=cloud.config - -### All params must be flagged as 'restart' because this config is manually -### retrieved by ConfiguredApplication.start to init the rpc server - -## The connectionspec for a slobrok is used for connecting to it using -## the FNET Remote Tools framework. It is normally on the form -## tcp/hostname:portnumber where hostname is the same name returned by -## gethostname() on the machine. This string is also used by the -## actual slobrok to figure out which port it should listen to. - -slobrok[].connectionspec string restart diff --git a/configdefinitions/src/vespa/cloud.config.ymon.def b/configdefinitions/src/vespa/cloud.config.ymon.def deleted file mode 100644 index 2296248bf8b..00000000000 --- a/configdefinitions/src/vespa/cloud.config.ymon.def +++ /dev/null @@ -1,6 +0,0 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -namespace=cloud.config - -services[].servicetype string default="(unknownservicetype)" -services[].ymonname string default="(unknownymonname)" -services[].hosts[] string diff --git a/configdefinitions/src/vespa/cloud.config.zookeeper-server.def b/configdefinitions/src/vespa/cloud.config.zookeeper-server.def deleted file mode 100644 index 483e772b818..00000000000 --- a/configdefinitions/src/vespa/cloud.config.zookeeper-server.def +++ /dev/null @@ -1,45 +0,0 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -namespace=cloud.config - -# Vespa home is prepended if the file is relative -zooKeeperConfigFile string default="conf/zookeeper/zookeeper.cfg" - -# For more info about the values below, see ZooKeeper documentation - -# tick time in milliseconds -tickTime int default=2000 -initLimit int default=20 -syncLimit int default=15 -maxClientConnections int default=0 - -# Vespa home is prepended if the file is relative -dataDir string default="var/zookeeper" - -clientPort int default=2181 -secureClientPort int default=2184 - -snapshotCount int default=50000 -# Purge interval in hours -autopurge.purgeInterval int default=1 -autopurge.snapRetainCount int default=15 - -# Vespa home is prepended if the file is relative -myidFile string default="var/zookeeper/myid" -# Change from default of 1 Mb in zookeeper to 50 Mb -juteMaxBuffer int default=52428800 - -myid int -server[].id int -server[].hostname string -server[].quorumPort int default=2182 -server[].electionPort int default=2183 - -# Needed when upgrading from ZooKeeper 3.4 to 3.5, see https://issues.apache.org/jira/browse/ZOOKEEPER-3056, -# and in general where there is a zookeeper ensemble running that has had few transactions. -# TODO: Consider setting this to false by default (and override where appropriate) -trustEmptySnapshot bool default=true - -# TLS options -tlsForQuorumCommunication enum { OFF, PORT_UNIFICATION, TLS_WITH_PORT_UNIFICATION, TLS_ONLY } default=OFF -tlsForClientServerCommunication enum { OFF, PORT_UNIFICATION, TLS_WITH_PORT_UNIFICATION, TLS_ONLY } default=OFF -jksKeyStoreFile string default="conf/zookeeper/zookeeper.jks" diff --git a/configdefinitions/src/vespa/cloud.config.zookeepers.def b/configdefinitions/src/vespa/cloud.config.zookeepers.def deleted file mode 100644 index d12bdd334ca..00000000000 --- a/configdefinitions/src/vespa/cloud.config.zookeepers.def +++ /dev/null @@ -1,3 +0,0 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -namespace=cloud.config -zookeeperserverlist string diff --git a/configdefinitions/src/vespa/cluster-info.def b/configdefinitions/src/vespa/cluster-info.def new file mode 100644 index 00000000000..f5e94ad61b3 --- /dev/null +++ b/configdefinitions/src/vespa/cluster-info.def @@ -0,0 +1,12 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +# Information about the cluster the subscriber is a part of +namespace=cloud.config + +# clusterId is same as clustername in model +clusterId string +nodeCount int + +services[].index int default=-1 +services[].hostname string default="(nohostname)" +services[].ports[].number int default=-1 +services[].ports[].tags string default="(notags)" diff --git a/configdefinitions/src/vespa/cluster-list.def b/configdefinitions/src/vespa/cluster-list.def new file mode 100644 index 00000000000..ad6f9715eb9 --- /dev/null +++ b/configdefinitions/src/vespa/cluster-list.def @@ -0,0 +1,5 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +namespace=cloud.config + +storage[].name string +storage[].configid string default="" diff --git a/configdefinitions/src/vespa/configserver.def b/configdefinitions/src/vespa/configserver.def new file mode 100644 index 00000000000..7405f5f2d05 --- /dev/null +++ b/configdefinitions/src/vespa/configserver.def @@ -0,0 +1,69 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +namespace=cloud.config + +# Ports +rpcport int default=19070 +httpport int default=19071 +# 0 means use the number of CPU cores available +numRpcThreads int default=0 + +# ZooKeeper +zookeeperserver[].hostname string +zookeeperserver[].port int default=2181 +# ZK write timeout in seconds, must be long enough to write application packages to other nodes +zookeeper.barrierTimeout long default=360 +zookeeperLocalhostAffinity bool default=true + +# Directories +configModelPluginDir[] string +configServerDBDir string default="var/db/vespa/config_server/serverdb/" +configDefinitionsDir string default="share/vespa/configdefinitions/" +fileReferencesDir string default="var/db/vespa/filedistribution/" + +# Misc +sessionLifetime long default=3600 # in seconds +masterGeneration long default=0 +multitenant bool default=false +numDelayedResponseThreads int default=1 +serverId string default="localhost" +hostedVespa bool default=false +numParallelTenantLoaders int default=4 + +# Configserver app +applicationDirectory string default="conf/configserver-app" + +# Zone information +cloud string default="default" +environment string default="prod" +region string default="default" +system string default="main" + +# RPC protocol +maxgetconfigclients int default=1000000 +maxoutputbuffersize int default=65536 +useVespaVersionInRequest bool default=false +payloadCompressionType enum { UNCOMPRESSED, LZ4 } default=LZ4 + +# Athenz config +loadBalancerAddress string default="" +athenzDnsSuffix string default="" +ztsUrl string default="" + +# Maintainers +maintainerIntervalMinutes int default=30 +keepUnusedFileReferencesHours int default=2 + +# Bootstrapping +# How long bootstrapping can take before giving up (in seconds) +maxDurationOfBootstrap long default=7200 +# How long to sleep before redeploying again if it fails (in seconds) +sleepTimeWhenRedeployingFails long default=30 + +# Features (to be overridden in configserver-config.xml if needed) +buildMinimalSetOfConfigModels bool default=true + +# Unused, remove in Vespa 8 +throwIfActiveSessionCannotBeLoaded bool default=true + +canReturnEmptySentinelConfig bool default=false +serverNodeType enum {config, controller} default=config diff --git a/configdefinitions/src/vespa/dispatch.def b/configdefinitions/src/vespa/dispatch.def new file mode 100644 index 00000000000..aa40c317d75 --- /dev/null +++ b/configdefinitions/src/vespa/dispatch.def @@ -0,0 +1,73 @@ +# Copyright 2019 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +# Configuration of dispatch from container nodes to search clusters + +namespace=vespa.config.search + +# The active docs a group must have as a % of the average active docs of all other groups, +# for that group to be included in queries +minActivedocsPercentage double default=97.0 + +# Minimum coverage for allowing a group to be considered for serving +minGroupCoverage double default=100 + +# Maximum number of nodes allowed to be down for group to be considered for serving +maxNodesDownPerGroup int default=0 + +# Distribution policy for group selection +distributionPolicy enum { ROUNDROBIN, ADAPTIVE } default=ADAPTIVE + +## Maximum number of hits that will be requested from a single node +## in this dataset. If not set, there is no limit. Using this option +## may help reduce network traffic when searching in datasets with big +## fan-out, but it will also result in incorrect and incomplete results; +## don't use it if you don't (really) mean it. +maxHitsPerNode int default=2147483647 + +## Probability for getting the K best hits (topK). +## A value of 1.0 will ask all N partitions for K hits. +## Any value between <0, 1> will use a Student T with 30 degrees freedom and compute a value Q that +## will give you the globally K best hits according to this formula with the desired probability. +## q = k/n + qT (p',30) x √(k × (1/n) × (1 − 1/n)) +## With a probability of 0.999 and K=200 and N=10 will give a Q of 38, meaning that you only need to fetch 19% compared to +## a setting of 1.0. This is a significant optimisation with with very little loss in presicion. +topKProbability double default=0.9999 + +# Is multi-level dispatch configured for this cluster +# Deprecated, will go away soon, NOOP +useMultilevelDispatch bool default=false + +# Dispatch only to local nodes. DEPRECATED: The container will automatically do this when it is appropriate. +useLocalNode bool default=false + +# Number of document copies +searchableCopies long default=1 + +# Minimum search coverage required before returning the results of a query +minSearchCoverage double default=100 + +# Minimum wait time for full coverage after minimum coverage is achieved, factored based on time left at minimum coverage +minWaitAfterCoverageFactor double default=0 + +# Maximum wait time for full coverage after minimum coverage is achieved, factored based on time left at minimum coverage +maxWaitAfterCoverageFactor double default=1 + +# Number of JRT transport threads +numJrtTransportThreads int default=8 + +# Number of JRT connections per backend node +numJrtConnectionsPerNode int default=8 + +# Number of seconds to spend warming up code to prevent JIT cold start issues. +warmuptime double default=0.1 + +# The unique key of a search node +node[].key int + +# The index of the group this search node belongs to +node[].group int default=0 + +# The host name of this search node +node[].host string + +# The rpc port of this search node +node[].port int diff --git a/configdefinitions/src/vespa/filereferences.def b/configdefinitions/src/vespa/filereferences.def new file mode 100644 index 00000000000..03e071959ce --- /dev/null +++ b/configdefinitions/src/vespa/filereferences.def @@ -0,0 +1,3 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +namespace=cloud.config.filedistribution +filereferences[] string diff --git a/configdefinitions/src/vespa/fleetcontroller.def b/configdefinitions/src/vespa/fleetcontroller.def new file mode 100644 index 00000000000..96b43a15c5e --- /dev/null +++ b/configdefinitions/src/vespa/fleetcontroller.def @@ -0,0 +1,183 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +namespace=vespa.config.content + +## Name of VDS cluster +cluster_name string restart + +## The fleetcontroller index. Each fleetcontroller should have a unique index +## which can be used to identify them. +index int restart + +## Number of fleet controllers. If more than one, the fleetcontroller needs to +## do master election in order to know which one is active. It then needs to +## know how many exist. +fleet_controller_count int default=1 + +## The number of seconds we can attempt to talk to zookeeper before our session +## time out and we lose our connection. +zookeeper_session_timeout double default=30.0 + +## When a master candidate see the master disappearing from ZooKeeper, it wants +## to take over as master. But, before taking over, the master should be aware +## that it has lost its zookeeper connection, so it will stop reacting as +## master. Suggests setting this to 2-3 times the zookeeper session timeout. +## (Set in number of seconds) +master_zookeeper_cooldown_period double default=60.0 + +## Sets how many fleetcontrollers will gather state. A fleetcontroller +## gathering state can take over quicker should ## the master fail. +## If set to 1, only master will gather state. If set higher, others will +## also do so, prioritizing those fleetcontrollers likely to be the ones to +## take over if the master fails. +state_gather_count int default=2 + +## Location of ZooKeeper servers +zookeeper_server string restart + +## RPC Port used by fleetcontroller +rpc_port int default=6500 restart + +## Port where fleetcontroller listens for HTTP status request +http_port int default=0 restart + +## Maximum number of milliseconds a storage will be automatically reported in +## maintenance due to node recently being availble. (If 0, no time at all) +storage_transition_time int default=30000 + +## Maximum number of milliseconds a distributor will be automatically reported +## in maintenance due to node recently being availble. (If 0, no time at all) +## +## Currently default is 0.. Should probably be more then we know it is working +## correctly +distributor_transition_time int default=0 + +## Maximum number of milliseconds allowed between progress increase during +## initialization. If no progress have been made during this time period, the +## node will be considered down. +## +## Currently disabled as 5.0 nodes gets load while initializing which may be +## higher pri than initializing, so nodes can stay in init for a long time. +init_progress_time int default=0 + +## Minimum time (in ms) between system state updates. To limit updates in a +## system where a lot is happening at the same time, this value will make sure +## we dont change the state too often. +min_time_between_new_systemstates int default=10000 + +## Sets how many milliseconds to wait between each state poll for old nodes +## requiring state polling. (4.1 or older) +state_polling_frequency int default=5000 + +## The maximum amount of premature crashes a node is allowed to have in a row +## before the fleetcontroller disables that node. +max_premature_crashes int default=100000 + +## If a node has been down or up this many milliseconds, clear the premature +## crash count of a node and consider the node as stable +stable_state_time_period int default=7200000 + +## The maximum number of events to keep in the event log +event_log_max_size int default=1024 + +## The maximum number of node events to keep in the node event log per node +event_node_log_max_size int default=1024 + +## The total number of distributor nodes that can exist. If 0, we dont know and +## will use the highest distributor index number we have ever seen + 1. +total_distributor_count int default=0 + +## The total number of storage nodes that can exist. If 0, we dont know and +## will use the highest storage index number we have ever seen + 1. +total_storage_count int default=0 + +## The minimum number of distributor nodes that should be up for the cluster +## state to be up. (Retired nodes counts as up in this case) +min_distributors_up_count int default=1 + +## The minimum number of storage nodes that should be up for the cluster state +## to be up (Retired nodes counts as up in this case) +min_storage_up_count int default=1 + +## The minimum ratio of known distributor nodes that should be up (or retired) +## for the cluster state to stay up. +min_distributor_up_ratio double default=0.01 + +## The minimum ratio of known storage nodes that should be up (or retired) for +## the cluster state to stay up. +min_storage_up_ratio double default=0.01 + +## Seconds to sleep after doing a work cycle where we did no work. Some +## events do not interrupt the sleeping, such as slobrok changes, so shouldn't +## set this too high +cycle_wait_time double default=0.1 + +## Minimum time to pass in seconds before broadcasting our first systemstate as +## a new fleetcontroller. (Will broadcast earlier than this if we have gathered +## state from all before this). To prevent disturbance when taking over as +## fleetcontroller, give nodes a bit of time to answer so we dont temporarily +## report nodes as down. +min_time_before_first_system_state_broadcast double default=5.0 + +## Request timeout of node state requests. Keeping a high timeout allows us to +## always have a pending operation with very low cost. Keeping a low timeout is +## good to detect issues like packet loss. The default tries to balance the two +## by not resending too often, but detecting packet loss within a minute at +## least. If we can guarantee RPC layer to fail on packet loss within +## reasonable time we should increase this default. +get_node_state_request_timeout double default=120.0 + +## If a node is out of slobrok longer than this time period, assume the node +## is down, even if we have a pending node state request to it. Slobrok does +## a bit more keep alive checking than fleetcontroller, so it is possible that +## the node disappears from slobrok while it still looks ok in fleetcontroller. +max_slobrok_disconnect_grace_period double default=60.0 + +## Whether to show system states that have never been sent to storage nodes in +## the event log. +show_local_systemstates_in_event_log bool default=true + +## The ideal number of distribution bits this system should have +ideal_distribution_bits int default=16 + +## Minimum ratio of nodes that have to be available (i.e. not Down) in any +## hierarchic content cluster group. If a higher ratio than this is Down at +## any point, the remaning nodes in the group will be automatically marked +## as down. Group nodes will automatically be taken back up as soon as node +## availability has been restored above the given threshold. +## Default is 0, i.e. functionality is for all intents and purposes disabled. +min_node_ratio_per_group double default=0.0 + +## If a cluster controller task has a dependency on a given cluster state +## version being published and ACKed by the cluster, it will be put on a wait +## queue while holding up the container thread associated with the task. +## This config specifies the maximum time a task can be held in this queue +## before being automatically failed out, if a version has not been ACKed +## within this duration. +max_deferred_task_version_wait_time_sec double default=30.0 + +## Whether or not the content cluster the controller has responsibility for +## contains any document types that are tagged as global. If this is true, +## global document-specific behavior is enabled that marks nodes down in the +## default space if they have merges pending in the global bucket space. +cluster_has_global_document_types bool default=false + +## The minimum merge completion ratio of buckets in a bucket space before it is considered complete. +## +## Bucket merges are considered complete when: +## ((buckets_total - buckets_pending) / buckets_total)) >= min_merge_completion_ratio +min_merge_completion_ratio double default=1.0 + +## If enabled, cluster state transitions are performed as two distinct phases: +## +## 1) state bundle propagation and bucket info gathering phase +## 2) state activation phase, which is not performed until all nodes have completed phase 1 +## +## This is to enable read-only operations to pass through the system during phase 1 +## while nodes await phase 2. If this feature is disabled, nodes will implicitly do +## phase 2 as part of phase 1 at their own leisure, which means that actual state +## activation may happen at wildly different times throughout the cluster. The 2 phase +## transition logic aims to minimize the window of time where active states diverge. +enable_two_phase_cluster_state_transitions bool default=false + +## Deprecated - not used +determine_buckets_from_bucket_space_metric bool default=true diff --git a/configdefinitions/src/vespa/ilscripts.def b/configdefinitions/src/vespa/ilscripts.def new file mode 100644 index 00000000000..901e87dbd04 --- /dev/null +++ b/configdefinitions/src/vespa/ilscripts.def @@ -0,0 +1,10 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +namespace=vespa.configdefinition + +## The maximum number of occurrences of a given term to index per field +maxtermoccurrences int default=100 +fieldmatchmaxlength int default=1000000 + +ilscript[].doctype string +ilscript[].docfield[] string +ilscript[].content[] string diff --git a/configdefinitions/src/vespa/imported-fields.def b/configdefinitions/src/vespa/imported-fields.def new file mode 100644 index 00000000000..0947ff63e2a --- /dev/null +++ b/configdefinitions/src/vespa/imported-fields.def @@ -0,0 +1,11 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +namespace=vespa.config.search + +# The name of an imported attribute field in context of this document type. +attribute[].name string + +# The name of the field (in this document type) that refers to a parent document type. +attribute[].referencefield string + +# The name of the target attribute field in the parent document type that is imported into this document type. +attribute[].targetfield string diff --git a/configdefinitions/src/vespa/indexschema.def b/configdefinitions/src/vespa/indexschema.def new file mode 100644 index 00000000000..245570ef7b5 --- /dev/null +++ b/configdefinitions/src/vespa/indexschema.def @@ -0,0 +1,25 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +## Config specifying the index fields and field collections that are part of an index schema. +namespace=vespa.config.search + +## The name of the index field. +indexfield[].name string +## The data type of the index field. +indexfield[].datatype enum { STRING, INT64 } default=STRING +## The collection type of the index field. +indexfield[].collectiontype enum { SINGLE, ARRAY, WEIGHTEDSET } default=SINGLE +## Whether the index should support prefix searches. +indexfield[].prefix bool default=false +## Whether the index should support phrase searches. +indexfield[].phrases bool default=false +## Whether the index should have posting lists with word positions. +indexfield[].positions bool default=true +## Average element length +indexfield[].averageelementlen int default=512 +## Whether the index field should use posting lists with interleaved features or not. +indexfield[].interleavedfeatures bool default=false + +## The name of the field collection (aka logical view). +fieldset[].name string +## The name of the index field that are part of this collection. +fieldset[].field[].name string diff --git a/configdefinitions/src/vespa/lb-services.def b/configdefinitions/src/vespa/lb-services.def new file mode 100644 index 00000000000..33c568061fe --- /dev/null +++ b/configdefinitions/src/vespa/lb-services.def @@ -0,0 +1,20 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +# Config for load balancer that needs to know about all services for +# all tenants and applications in a config server cluster + +namespace=cloud.config + +# Active rotation given as flag 'active' for a prod region in deployment.xml +# Default true for now (since code in config-model to set it is not ready yet), should have no default value +tenants{}.applications{}.activeRotation bool default=true + +tenants{}.applications{}.hosts{}.hostname string default="(unknownhostname)" +tenants{}.applications{}.hosts{}.services{}.type string default="(noservicetype)" +tenants{}.applications{}.hosts{}.services{}.clustertype string default="(unknownclustertype)" +tenants{}.applications{}.hosts{}.services{}.clustername string default="(unknownclustername)" +tenants{}.applications{}.hosts{}.services{}.configId string +tenants{}.applications{}.hosts{}.services{}.index int default=0 +tenants{}.applications{}.hosts{}.services{}.ports[].number int default=-1 +tenants{}.applications{}.hosts{}.services{}.ports[].tags string default="(notags)" +tenants{}.applications{}.hosts{}.services{}.servicealiases[] string +tenants{}.applications{}.hosts{}.services{}.endpointaliases[] string diff --git a/configdefinitions/src/vespa/load-type.def b/configdefinitions/src/vespa/load-type.def new file mode 100644 index 00000000000..2e3f65cfbba --- /dev/null +++ b/configdefinitions/src/vespa/load-type.def @@ -0,0 +1,30 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +## This config is used to define load types in VESPA. Load types are merely +## a way to distinguish types of load, as to present such types differently. +## For instance, some metrics will be presented per load type, such that you +## can see how that specific type is being handled. Load types are especially +## useful for separating loads of different priority. + +namespace=vespa.config.content + +## The identifying index of this type. 0 cannot be used as it is +## reserved for the default type. Other types should be +## non-overlapping from 1 and upwards. No gaps are allowed in the +## index range. (For performance reasons we want to use arrays +## internally where the id is the index) +## +## The identifier is used for efficient transfer of type information. +## The types will be identified by names instead in more textual or +## verbose interfaces. +## restart flag was added automatically and needs to be verified. +type[].id int restart + +## The name of a given load type. Each load type must have a unique +## name. The name "default" is reserved for the default type of id 0. +## restart flag was added automatically and needs to be verified. +type[].name string restart + +## The default priority for the load type. +## restart flag was added automatically and needs to be verified. +type[].priority string restart + diff --git a/configdefinitions/src/vespa/logforwarder.def b/configdefinitions/src/vespa/logforwarder.def new file mode 100644 index 00000000000..ab56589298f --- /dev/null +++ b/configdefinitions/src/vespa/logforwarder.def @@ -0,0 +1,9 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +namespace=cloud.config + +# only splunk type config for now + +deploymentServer string default="" +clientName string default="" +splunkHome string default="/opt/splunkforwarder" +phoneHomeInterval int default=60 diff --git a/configdefinitions/src/vespa/messagetyperouteselectorpolicy.def b/configdefinitions/src/vespa/messagetyperouteselectorpolicy.def new file mode 100644 index 00000000000..7fbd6e45bd7 --- /dev/null +++ b/configdefinitions/src/vespa/messagetyperouteselectorpolicy.def @@ -0,0 +1,11 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +namespace=vespa.config.content + +# Default route if no override is set for a type. +defaultroute string + +# The name of the route. +route[].name string + +# The document protocol messagetype triggering this route. +route[].messagetype int diff --git a/configdefinitions/src/vespa/model.def b/configdefinitions/src/vespa/model.def new file mode 100644 index 00000000000..59c04cee143 --- /dev/null +++ b/configdefinitions/src/vespa/model.def @@ -0,0 +1,13 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +namespace=cloud.config + +vespaVersion string default="unknown" +hosts[].name string default="(unknownhostname)" +hosts[].services[].name string default="(noservicename)" +hosts[].services[].type string default="(noservicetype)" +hosts[].services[].configid string default="(noconfigid)" +hosts[].services[].clustertype string default="(unknownclustertype)" +hosts[].services[].clustername string default="(unknownclustername)" +hosts[].services[].index int default=0 +hosts[].services[].ports[].number int default=-1 +hosts[].services[].ports[].tags string default="(notags)" diff --git a/configdefinitions/src/vespa/orchestrator.def b/configdefinitions/src/vespa/orchestrator.def new file mode 100644 index 00000000000..ab52c777398 --- /dev/null +++ b/configdefinitions/src/vespa/orchestrator.def @@ -0,0 +1,5 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +namespace=vespa.orchestrator.config + +# TODO: Change the default to actual latency in real setup. +serviceMonitorConvergenceLatencySeconds int default=0 diff --git a/configdefinitions/src/vespa/persistence.def b/configdefinitions/src/vespa/persistence.def new file mode 100644 index 00000000000..b2709dedc57 --- /dev/null +++ b/configdefinitions/src/vespa/persistence.def @@ -0,0 +1,60 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +namespace=vespa.config.content + +## If set, causes the service layer to trigger an abort of any queued +## operations for buckets whose ownership changes during a cluster state +## change. This helps ensure consistency between the distributors and +## the storage nodes by discarding operations that were not sent by the +## new owning distributor. The service layer will also block any new +## operations from happening until all operations currently executing for +## buckets with changed ownership have completed. +## Enabling this option has a potential of causing a minor performance +## impact when a node goes up or down due to its blocking semantics, but +## this should be on the order of tens to a few hundred milliseconds. +abort_operations_with_changed_bucket_ownership bool default=true + +## If set, aborts any mutating ideal state operations that arrive to a node +## that were scheduled and sent by distributors that no longer own the buckets +## the operations are bound for. This option complements the above config, +## as this is checked for every arriving message and not when the state changes. +abort_outdated_mutating_ideal_state_ops bool default=true + +## If set, aborts any external (client) operations that arrive to a node from +## a distributor that is no longer the owner of the bucket those operations +## belong to. As with the abort_outdated_mutating_ideal_state_ops option, +## this is a check that happens whenever a message arrives, not when the state +## changes. +abort_outdated_mutating_external_load_ops bool default=true + +## Whether or not to disable partitions when they report I/O errors +fail_partition_on_error bool default=true + +## Time period to keep all updates (given in seconds). One can revert any +## operation done within this time. +revert_time_period int default=300 + +## Minimum number of seconds that document remove (tombstone) entries are +## retained in a bucket before being automatically compacted away. In the case +## where a node has been down for a longer period of time than this value +## and then comes back up, there exists a possibility for documents to be +## resurrected. This happens when documents existing on both the downed node +## and the online nodes are marked as removed on the online nodes and then +## subsequently compacted away along with their remove entries. When the old +## node comes back online, it will appear to the system as if the node has +## documents that do not exist on other nodes (as no remove-entries can be +## found for them) and they will subsequently re-appear. +keep_remove_time_period int default=604800 + +## For providers supporting multiple document versions, this option specifies +## a soft limit on the maximum number of versions that can be stored for a +## single document at any given time. This is considered a soft limit as it +## is possible for more versions to exist concurrently than what this config +## value specifies in a case where the pressure on the containing bucket is not +## yet high enough to warrant a compaction. +## After a compaction has taken place, only the n _newest_ document versions +## will be present in the compacted representation of the bucket. +## This limitation takes precedence over what versions may be stored according +## to revert_time_period. +## Configured value must be at least 1. +maximum_versions_of_single_document_stored int default=5 + diff --git a/configdefinitions/src/vespa/rank-profiles.def b/configdefinitions/src/vespa/rank-profiles.def new file mode 100644 index 00000000000..525930c990b --- /dev/null +++ b/configdefinitions/src/vespa/rank-profiles.def @@ -0,0 +1,12 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +namespace=vespa.config.search + +## name of this rank profile. maps to table index for internal use. +rankprofile[].name string + +## the name of a generic property available to the feature execution framework and feature plugins +rankprofile[].fef.property[].name string + +## the value of a generic property available to feature plugins +rankprofile[].fef.property[].value string + diff --git a/configdefinitions/src/vespa/routing-provider.def b/configdefinitions/src/vespa/routing-provider.def new file mode 100755 index 00000000000..eedf4f3c86f --- /dev/null +++ b/configdefinitions/src/vespa/routing-provider.def @@ -0,0 +1,9 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +# Configuration for the routing config provider + +# TODO Remove once YCA filter is removed + +namespace=cloud.config + +# disabled by default, automatically enabled for hosted vespa +enabled bool default=false diff --git a/configdefinitions/src/vespa/sentinel.def b/configdefinitions/src/vespa/sentinel.def new file mode 100644 index 00000000000..d318a55cccf --- /dev/null +++ b/configdefinitions/src/vespa/sentinel.def @@ -0,0 +1,50 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +namespace=cloud.config + +## The port where sentinel should run its telnet interface +port.telnet int default=19098 + +## The port where sentinel should run its RPC interface +port.rpc int default=19097 + +## Meta info about the application running on this node. All values are +## exported to child processes as $VESPA_APPLICATION_TENANT, +## $VESPA_APPLICATION_NAME, $VESPA_APPLICATION_ENVIRONMENT, $VESPA_APPLICATION_REGION +## and $VESPA_APPLICATION_INSTANCE +application.tenant string default="default" +application.name string default="default" +application.environment string default="default" +application.instance string default="default" +application.region string default="default" + +## The command to run. This will be run by sh -c, and the following +## environment variables are defined: $ROOT, $VESPA_SERVICE_NAME, +## $VESPA_CONFIG_ID +## +## stdin points to /dev/null +## stdout and stderr connected to sentinel via pipe on startup. +service[].command string + +## The command to run before stopping service. The same properties as for +## startup command holds. +service[].preShutdownCommand string default="" + +## The service name for this service. Exported in VESPA_SERVICE_NAME +## environment variable. All services must have unique names. +## The service name is used by to start/stop services, and to list +## them. It is also logged in the service field in every log message. +service[].name string + +## Should the service be auto-started by configd as soon as it starts up, +## or must it be started manually? Almost always true. +service[].autostart bool default=false + +## Should the service automatically be restarted if it exits? +service[].autorestart bool default=true + +## Config reference to the service being started. +## Exported in the VESPA_CONFIG_ID environment variable. +service[].id reference + +## Affinity of this service towards a CPU socket +service[].affinity.cpuSocket int default=-1 diff --git a/configdefinitions/src/vespa/slobroks.def b/configdefinitions/src/vespa/slobroks.def new file mode 100644 index 00000000000..b4e5536d2df --- /dev/null +++ b/configdefinitions/src/vespa/slobroks.def @@ -0,0 +1,13 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +namespace=cloud.config + +### All params must be flagged as 'restart' because this config is manually +### retrieved by ConfiguredApplication.start to init the rpc server + +## The connectionspec for a slobrok is used for connecting to it using +## the FNET Remote Tools framework. It is normally on the form +## tcp/hostname:portnumber where hostname is the same name returned by +## gethostname() on the machine. This string is also used by the +## actual slobrok to figure out which port it should listen to. + +slobrok[].connectionspec string restart diff --git a/configdefinitions/src/vespa/specialtokens.def b/configdefinitions/src/vespa/specialtokens.def new file mode 100644 index 00000000000..1e6addcf358 --- /dev/null +++ b/configdefinitions/src/vespa/specialtokens.def @@ -0,0 +1,32 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +## Named lists of special tokens - string which should be +## treated as words no matter what characters they consist of. +## A special token can also optionally be replaced by another +## token. +## +## Special tokens are case sensitive, since queries and tokens +## are lowercased they should normally be written in lower case. +## +## The list named default is the only list which is used currently +## +## Order matters in token lists. If one special token is a +## prefix of another token, Vespa will prefer to return the first +## matching token in the list when encountering the longest +## special token. +namespace=vespa.configdefinition + +## Path to makefsa binary, needed to create specialtokens-dictionary +## If the path is relative, vespa home is prepended +makefsaPath string default="bin/vespa-makefsa" + +## Use 'default' as the name of the list to use in regular +## indexing and queries +tokenlist[].name string + +## Special tokens, any string is permitted +tokenlist[].tokens[].token string + +## The token which should replace the special token +## If no replace is set for a token, the word token +## will be the special token string itself +tokenlist[].tokens[].replace string default="" diff --git a/configdefinitions/src/vespa/stateserver.def b/configdefinitions/src/vespa/stateserver.def new file mode 100644 index 00000000000..a25844fa148 --- /dev/null +++ b/configdefinitions/src/vespa/stateserver.def @@ -0,0 +1,5 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +namespace=vespa.config.core + +# Port used for serving the state/vN api +httpport int default = 0 diff --git a/configdefinitions/src/vespa/stor-distribution.def b/configdefinitions/src/vespa/stor-distribution.def new file mode 100644 index 00000000000..518626228c1 --- /dev/null +++ b/configdefinitions/src/vespa/stor-distribution.def @@ -0,0 +1,70 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +namespace=vespa.config.content + +## Redundancy decides how many copies of documents will be stored. Documents +## will store copies on enough nodes, such that the sum of the node reliability +## factors is equal or above the redundancy. +redundancy int default=3 + +## Initial redundancy allows put-operations to return as completed after +## a subset of all copies have been persisted (known as initial persistence +## in BCP terms). +## Set to 0 to disable and use normal redundancy behavior instead. +initial_redundancy int default=0 + +## If set, requires that the primary nodes copy should always be persisted +## before an n-of-m initial persistence operation can complete. +ensure_primary_persisted bool default=true + +## The number of copies that should be "ready" to be active. +## Maximum is redundancy. +ready_copies int default=0 + +## If this option is set true, the distributor will try to enforce one active copy +## of buckets per leaf hierarchical group. This is a simple implementation for +## search to be able to setup top level dispatcher to only send search to all +## nodes in one group as they have a static cost per node used. It used, +## hierarchical grouping can not be used for other purposes. Using this option +## implies that: +## - ready_copies == redundancy +## - Only one level of hierarchical grouping defined. +## - That level distributes copies to all defined groups. +active_per_leaf_group bool default=false + +## Search have some unfortunate properties with some queries, adding a static +## query cost, independent of the number of documents searched on a node. For +## these cases they use many small groups to be able to scale. In such cases, +## where groups are small, having distributors able to take over when none are +## available in a group is useful. +distributor_auto_ownership_transfer_on_whole_group_down bool default=true + +## Hierarchical grouping divides the nodes into a tree of groups. Due to config +## liking flat structures. The tree of groups is represented by a single array +## of groups defined here, where index is a string that can have a form like +## "1.2.1" in order to tell where in the tree structure the group is. +group[].index string +## Each group needs to have a name. Makes it easier to operations that they can +## name their groups according to what they want to call them. +group[].name string +## Capacity of the given group +group[].capacity double default=1 +## Partitions define how copies are divided among child groups/nodes. +group[].partitions string default="" +## Leaf groups will have a set of nodes within them. Branch groups will have +## none. +group[].nodes[].index int +# Whether this node is retired. +# The system will migrate all data away from retired nodes such that they can +# eventually be removed without partial data loss +group[].nodes[].retired bool default=false + +## Which disk distribution to use. From the current choices, we very much +## recommend using MODULO_BID. The only reason to use any of the earlier one is +## during an upgrade where you dont want to mess up the distribution in case you +## want to revert the upgrade. +## +## MODULO old one (4.0) +## MODULO_INDEX with node index in seed +## MODULO_KNUTH with random(node index) in seed +## MODULO_BID using all used bits, except count bits, and random(node index) +disk_distribution enum { MODULO, MODULO_INDEX, MODULO_KNUTH, MODULO_BID } default=MODULO_BID diff --git a/configdefinitions/src/vespa/stor-filestor.def b/configdefinitions/src/vespa/stor-filestor.def new file mode 100644 index 00000000000..8ac372c0575 --- /dev/null +++ b/configdefinitions/src/vespa/stor-filestor.def @@ -0,0 +1,60 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +namespace=vespa.config.content + +## DETECT FAILURE PARAMETERS + +## After seeing given number of errors on a disk, storage will disable the +## disk and restart. If set to 0, storage will never disable disks. Note +## that if you get disk errors, which arent automatically fixed, this will +## make storage keep failing all operations towards this disk, and all +## retries will use the same disk and same file, so you might end up with a +## lot of operations that can never succeed. Thus, it is not recommended to +## turn this off. +fail_disk_after_error_count int default=1 restart + +## If a disk operation takes more time in seconds than the disk operation +## timeout, storage will consider that a disk failure, and mark the disk +## bad, restart and not use the disk anymore. Note that currently, this +## includes CPU time too, so if used, this parameter should be set fairly +## high as we still sometimes have some operations that might take a few +## seconds. Disabled by default as we currently have these long operations +## during normal load. +disk_operation_timeout int default=0 restart + +## PERFORMANCE PARAMETERS + +## Number of threads to use for each mountpoint. +num_threads int default=8 restart + +## Number of threads for response processing and delivery +## 0 will give legacy sync behavior. +## Negative number will choose a good number based on # cores. +num_response_threads int default=2 restart + +## Type of sequenced thread executor use for persistence replies. +response_sequencer_type enum {LATENCY, THROUGHPUT, ADAPTIVE} default=ADAPTIVE restart + +## When merging, if we find more than this number of documents that exist on all +## of the same copies, send a separate apply bucket diff with these entries +## to an optimized merge chain that guarantuees minimum data transfer. +common_merge_chain_optimalization_minimum_size int default=64 restart + +## Chunksize to use while merging buckets between nodes. +## +## Default is set to 4 MB - 4k. This is to allow for malloc to waste some bytes +## with tracking info or to align to 512b without passing allocation limit, +## while still reading 4k blocks from disk. +bucket_merge_chunk_size int default=4190208 restart + +## When merging, it is possible to send more metadata than needed in order to +## let local nodes in merge decide which entries fits best to add this time +## based on disk location. Toggle this option on to use it. Note that memory +## consumption might increase in a 4.1 to 4.2 upgrade due to this, as 4.1 +## dont support to only fill in part of the metadata provided and will always +## fill all. +enable_merge_local_node_choose_docs_optimalization bool default=true restart + +## Whether or not to enable the multibit split optimalization. This is useful +## if splitting is expensive, but listing document identifiers is fairly cheap. +## This is true for memfile persistence layer, but not for vespa search. +enable_multibit_split_optimalization bool default=true restart diff --git a/configdefinitions/src/vespa/summary.def b/configdefinitions/src/vespa/summary.def new file mode 100644 index 00000000000..20ca6b10450 --- /dev/null +++ b/configdefinitions/src/vespa/summary.def @@ -0,0 +1,8 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +namespace=vespa.config.search + +defaultsummaryid int default=-1 +classes[].id int +classes[].name string +classes[].fields[].name string +classes[].fields[].type string diff --git a/configdefinitions/src/vespa/summarymap.def b/configdefinitions/src/vespa/summarymap.def new file mode 100644 index 00000000000..a994309e207 --- /dev/null +++ b/configdefinitions/src/vespa/summarymap.def @@ -0,0 +1,15 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +namespace=vespa.config.search + +## The default output summary class id, -1 to use the stored summary class id +defaultoutputclass int default=-1 + +## The name of the summary field which is overridden with a dynamic value +override[].field string + +## The overriding command, one of: +## staticrank, dynamicteaser, dynamicteasermetric, label, ranklog, empty, copy +override[].command string + +## Space-seaparated arguments, dedendent on the given command +override[].arguments string default="" diff --git a/configdefinitions/src/vespa/upgrading.def b/configdefinitions/src/vespa/upgrading.def new file mode 100644 index 00000000000..3f610c9352d --- /dev/null +++ b/configdefinitions/src/vespa/upgrading.def @@ -0,0 +1,24 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +namespace=vespa.config.content + +## Config specifying cluster global parameters for a storage cluster. + +## Set to true when there exist nodes of previous major version in cluster. +## This will enable major version backward compability mode in the cluster. +upgrading_major_to bool default=false + +## Set to true when there exist nodes of the next major version in the +## cluster. This will usually not be needed, unless specified in the upgrade +## docs. Sometimes though, it may be easier to let old version be forward +## compatable than having new version backwards compatable. +upgrading_major_from bool default=false + +## Set to true when there exist nodes of previous minor version in cluster. +## This should not be needed, but settings exist in case we end up having +## no other choice than to have a compatability mode between minor versions. +upgrading_minor_to bool default=false + +## Set to true when there exist nodes of next minor version in cluster. +## This should not be needed, but settings exist in case we end up having +## no other choice than to have a compatability mode between minor versions. +upgrading_minor_from bool default=false diff --git a/configdefinitions/src/vespa/vespa.config.content.all-clusters-bucket-spaces.def b/configdefinitions/src/vespa/vespa.config.content.all-clusters-bucket-spaces.def deleted file mode 100644 index 7676bc2a03f..00000000000 --- a/configdefinitions/src/vespa/vespa.config.content.all-clusters-bucket-spaces.def +++ /dev/null @@ -1,9 +0,0 @@ -# Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -namespace=vespa.config.content - -## This config contains the document types handled by all content clusters -## and the bucket spaces they belong to. - -## The bucket space a document type in a particular cluster belongs to. -cluster{}.documentType{}.bucketSpace string - diff --git a/configdefinitions/src/vespa/vespa.config.content.core.bucketspaces.def b/configdefinitions/src/vespa/vespa.config.content.core.bucketspaces.def deleted file mode 100644 index c9468850018..00000000000 --- a/configdefinitions/src/vespa/vespa.config.content.core.bucketspaces.def +++ /dev/null @@ -1,12 +0,0 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -namespace=vespa.config.content.core - -## This config contains the document types handled by a given content cluster -## and the bucket space they belong to. - -## The name of a document type. -documenttype[].name string - -## The bucket space this document type belongs to. -documenttype[].bucketspace string - diff --git a/configdefinitions/src/vespa/vespa.config.content.fleetcontroller.def b/configdefinitions/src/vespa/vespa.config.content.fleetcontroller.def deleted file mode 100644 index 96b43a15c5e..00000000000 --- a/configdefinitions/src/vespa/vespa.config.content.fleetcontroller.def +++ /dev/null @@ -1,183 +0,0 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -namespace=vespa.config.content - -## Name of VDS cluster -cluster_name string restart - -## The fleetcontroller index. Each fleetcontroller should have a unique index -## which can be used to identify them. -index int restart - -## Number of fleet controllers. If more than one, the fleetcontroller needs to -## do master election in order to know which one is active. It then needs to -## know how many exist. -fleet_controller_count int default=1 - -## The number of seconds we can attempt to talk to zookeeper before our session -## time out and we lose our connection. -zookeeper_session_timeout double default=30.0 - -## When a master candidate see the master disappearing from ZooKeeper, it wants -## to take over as master. But, before taking over, the master should be aware -## that it has lost its zookeeper connection, so it will stop reacting as -## master. Suggests setting this to 2-3 times the zookeeper session timeout. -## (Set in number of seconds) -master_zookeeper_cooldown_period double default=60.0 - -## Sets how many fleetcontrollers will gather state. A fleetcontroller -## gathering state can take over quicker should ## the master fail. -## If set to 1, only master will gather state. If set higher, others will -## also do so, prioritizing those fleetcontrollers likely to be the ones to -## take over if the master fails. -state_gather_count int default=2 - -## Location of ZooKeeper servers -zookeeper_server string restart - -## RPC Port used by fleetcontroller -rpc_port int default=6500 restart - -## Port where fleetcontroller listens for HTTP status request -http_port int default=0 restart - -## Maximum number of milliseconds a storage will be automatically reported in -## maintenance due to node recently being availble. (If 0, no time at all) -storage_transition_time int default=30000 - -## Maximum number of milliseconds a distributor will be automatically reported -## in maintenance due to node recently being availble. (If 0, no time at all) -## -## Currently default is 0.. Should probably be more then we know it is working -## correctly -distributor_transition_time int default=0 - -## Maximum number of milliseconds allowed between progress increase during -## initialization. If no progress have been made during this time period, the -## node will be considered down. -## -## Currently disabled as 5.0 nodes gets load while initializing which may be -## higher pri than initializing, so nodes can stay in init for a long time. -init_progress_time int default=0 - -## Minimum time (in ms) between system state updates. To limit updates in a -## system where a lot is happening at the same time, this value will make sure -## we dont change the state too often. -min_time_between_new_systemstates int default=10000 - -## Sets how many milliseconds to wait between each state poll for old nodes -## requiring state polling. (4.1 or older) -state_polling_frequency int default=5000 - -## The maximum amount of premature crashes a node is allowed to have in a row -## before the fleetcontroller disables that node. -max_premature_crashes int default=100000 - -## If a node has been down or up this many milliseconds, clear the premature -## crash count of a node and consider the node as stable -stable_state_time_period int default=7200000 - -## The maximum number of events to keep in the event log -event_log_max_size int default=1024 - -## The maximum number of node events to keep in the node event log per node -event_node_log_max_size int default=1024 - -## The total number of distributor nodes that can exist. If 0, we dont know and -## will use the highest distributor index number we have ever seen + 1. -total_distributor_count int default=0 - -## The total number of storage nodes that can exist. If 0, we dont know and -## will use the highest storage index number we have ever seen + 1. -total_storage_count int default=0 - -## The minimum number of distributor nodes that should be up for the cluster -## state to be up. (Retired nodes counts as up in this case) -min_distributors_up_count int default=1 - -## The minimum number of storage nodes that should be up for the cluster state -## to be up (Retired nodes counts as up in this case) -min_storage_up_count int default=1 - -## The minimum ratio of known distributor nodes that should be up (or retired) -## for the cluster state to stay up. -min_distributor_up_ratio double default=0.01 - -## The minimum ratio of known storage nodes that should be up (or retired) for -## the cluster state to stay up. -min_storage_up_ratio double default=0.01 - -## Seconds to sleep after doing a work cycle where we did no work. Some -## events do not interrupt the sleeping, such as slobrok changes, so shouldn't -## set this too high -cycle_wait_time double default=0.1 - -## Minimum time to pass in seconds before broadcasting our first systemstate as -## a new fleetcontroller. (Will broadcast earlier than this if we have gathered -## state from all before this). To prevent disturbance when taking over as -## fleetcontroller, give nodes a bit of time to answer so we dont temporarily -## report nodes as down. -min_time_before_first_system_state_broadcast double default=5.0 - -## Request timeout of node state requests. Keeping a high timeout allows us to -## always have a pending operation with very low cost. Keeping a low timeout is -## good to detect issues like packet loss. The default tries to balance the two -## by not resending too often, but detecting packet loss within a minute at -## least. If we can guarantee RPC layer to fail on packet loss within -## reasonable time we should increase this default. -get_node_state_request_timeout double default=120.0 - -## If a node is out of slobrok longer than this time period, assume the node -## is down, even if we have a pending node state request to it. Slobrok does -## a bit more keep alive checking than fleetcontroller, so it is possible that -## the node disappears from slobrok while it still looks ok in fleetcontroller. -max_slobrok_disconnect_grace_period double default=60.0 - -## Whether to show system states that have never been sent to storage nodes in -## the event log. -show_local_systemstates_in_event_log bool default=true - -## The ideal number of distribution bits this system should have -ideal_distribution_bits int default=16 - -## Minimum ratio of nodes that have to be available (i.e. not Down) in any -## hierarchic content cluster group. If a higher ratio than this is Down at -## any point, the remaning nodes in the group will be automatically marked -## as down. Group nodes will automatically be taken back up as soon as node -## availability has been restored above the given threshold. -## Default is 0, i.e. functionality is for all intents and purposes disabled. -min_node_ratio_per_group double default=0.0 - -## If a cluster controller task has a dependency on a given cluster state -## version being published and ACKed by the cluster, it will be put on a wait -## queue while holding up the container thread associated with the task. -## This config specifies the maximum time a task can be held in this queue -## before being automatically failed out, if a version has not been ACKed -## within this duration. -max_deferred_task_version_wait_time_sec double default=30.0 - -## Whether or not the content cluster the controller has responsibility for -## contains any document types that are tagged as global. If this is true, -## global document-specific behavior is enabled that marks nodes down in the -## default space if they have merges pending in the global bucket space. -cluster_has_global_document_types bool default=false - -## The minimum merge completion ratio of buckets in a bucket space before it is considered complete. -## -## Bucket merges are considered complete when: -## ((buckets_total - buckets_pending) / buckets_total)) >= min_merge_completion_ratio -min_merge_completion_ratio double default=1.0 - -## If enabled, cluster state transitions are performed as two distinct phases: -## -## 1) state bundle propagation and bucket info gathering phase -## 2) state activation phase, which is not performed until all nodes have completed phase 1 -## -## This is to enable read-only operations to pass through the system during phase 1 -## while nodes await phase 2. If this feature is disabled, nodes will implicitly do -## phase 2 as part of phase 1 at their own leisure, which means that actual state -## activation may happen at wildly different times throughout the cluster. The 2 phase -## transition logic aims to minimize the window of time where active states diverge. -enable_two_phase_cluster_state_transitions bool default=false - -## Deprecated - not used -determine_buckets_from_bucket_space_metric bool default=true diff --git a/configdefinitions/src/vespa/vespa.config.content.load-type.def b/configdefinitions/src/vespa/vespa.config.content.load-type.def deleted file mode 100644 index 2e3f65cfbba..00000000000 --- a/configdefinitions/src/vespa/vespa.config.content.load-type.def +++ /dev/null @@ -1,30 +0,0 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -## This config is used to define load types in VESPA. Load types are merely -## a way to distinguish types of load, as to present such types differently. -## For instance, some metrics will be presented per load type, such that you -## can see how that specific type is being handled. Load types are especially -## useful for separating loads of different priority. - -namespace=vespa.config.content - -## The identifying index of this type. 0 cannot be used as it is -## reserved for the default type. Other types should be -## non-overlapping from 1 and upwards. No gaps are allowed in the -## index range. (For performance reasons we want to use arrays -## internally where the id is the index) -## -## The identifier is used for efficient transfer of type information. -## The types will be identified by names instead in more textual or -## verbose interfaces. -## restart flag was added automatically and needs to be verified. -type[].id int restart - -## The name of a given load type. Each load type must have a unique -## name. The name "default" is reserved for the default type of id 0. -## restart flag was added automatically and needs to be verified. -type[].name string restart - -## The default priority for the load type. -## restart flag was added automatically and needs to be verified. -type[].priority string restart - diff --git a/configdefinitions/src/vespa/vespa.config.content.messagetyperouteselectorpolicy.def b/configdefinitions/src/vespa/vespa.config.content.messagetyperouteselectorpolicy.def deleted file mode 100644 index 7fbd6e45bd7..00000000000 --- a/configdefinitions/src/vespa/vespa.config.content.messagetyperouteselectorpolicy.def +++ /dev/null @@ -1,11 +0,0 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -namespace=vespa.config.content - -# Default route if no override is set for a type. -defaultroute string - -# The name of the route. -route[].name string - -# The document protocol messagetype triggering this route. -route[].messagetype int diff --git a/configdefinitions/src/vespa/vespa.config.content.persistence.def b/configdefinitions/src/vespa/vespa.config.content.persistence.def deleted file mode 100644 index b2709dedc57..00000000000 --- a/configdefinitions/src/vespa/vespa.config.content.persistence.def +++ /dev/null @@ -1,60 +0,0 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -namespace=vespa.config.content - -## If set, causes the service layer to trigger an abort of any queued -## operations for buckets whose ownership changes during a cluster state -## change. This helps ensure consistency between the distributors and -## the storage nodes by discarding operations that were not sent by the -## new owning distributor. The service layer will also block any new -## operations from happening until all operations currently executing for -## buckets with changed ownership have completed. -## Enabling this option has a potential of causing a minor performance -## impact when a node goes up or down due to its blocking semantics, but -## this should be on the order of tens to a few hundred milliseconds. -abort_operations_with_changed_bucket_ownership bool default=true - -## If set, aborts any mutating ideal state operations that arrive to a node -## that were scheduled and sent by distributors that no longer own the buckets -## the operations are bound for. This option complements the above config, -## as this is checked for every arriving message and not when the state changes. -abort_outdated_mutating_ideal_state_ops bool default=true - -## If set, aborts any external (client) operations that arrive to a node from -## a distributor that is no longer the owner of the bucket those operations -## belong to. As with the abort_outdated_mutating_ideal_state_ops option, -## this is a check that happens whenever a message arrives, not when the state -## changes. -abort_outdated_mutating_external_load_ops bool default=true - -## Whether or not to disable partitions when they report I/O errors -fail_partition_on_error bool default=true - -## Time period to keep all updates (given in seconds). One can revert any -## operation done within this time. -revert_time_period int default=300 - -## Minimum number of seconds that document remove (tombstone) entries are -## retained in a bucket before being automatically compacted away. In the case -## where a node has been down for a longer period of time than this value -## and then comes back up, there exists a possibility for documents to be -## resurrected. This happens when documents existing on both the downed node -## and the online nodes are marked as removed on the online nodes and then -## subsequently compacted away along with their remove entries. When the old -## node comes back online, it will appear to the system as if the node has -## documents that do not exist on other nodes (as no remove-entries can be -## found for them) and they will subsequently re-appear. -keep_remove_time_period int default=604800 - -## For providers supporting multiple document versions, this option specifies -## a soft limit on the maximum number of versions that can be stored for a -## single document at any given time. This is considered a soft limit as it -## is possible for more versions to exist concurrently than what this config -## value specifies in a case where the pressure on the containing bucket is not -## yet high enough to warrant a compaction. -## After a compaction has taken place, only the n _newest_ document versions -## will be present in the compacted representation of the bucket. -## This limitation takes precedence over what versions may be stored according -## to revert_time_period. -## Configured value must be at least 1. -maximum_versions_of_single_document_stored int default=5 - diff --git a/configdefinitions/src/vespa/vespa.config.content.stor-distribution.def b/configdefinitions/src/vespa/vespa.config.content.stor-distribution.def deleted file mode 100644 index 518626228c1..00000000000 --- a/configdefinitions/src/vespa/vespa.config.content.stor-distribution.def +++ /dev/null @@ -1,70 +0,0 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -namespace=vespa.config.content - -## Redundancy decides how many copies of documents will be stored. Documents -## will store copies on enough nodes, such that the sum of the node reliability -## factors is equal or above the redundancy. -redundancy int default=3 - -## Initial redundancy allows put-operations to return as completed after -## a subset of all copies have been persisted (known as initial persistence -## in BCP terms). -## Set to 0 to disable and use normal redundancy behavior instead. -initial_redundancy int default=0 - -## If set, requires that the primary nodes copy should always be persisted -## before an n-of-m initial persistence operation can complete. -ensure_primary_persisted bool default=true - -## The number of copies that should be "ready" to be active. -## Maximum is redundancy. -ready_copies int default=0 - -## If this option is set true, the distributor will try to enforce one active copy -## of buckets per leaf hierarchical group. This is a simple implementation for -## search to be able to setup top level dispatcher to only send search to all -## nodes in one group as they have a static cost per node used. It used, -## hierarchical grouping can not be used for other purposes. Using this option -## implies that: -## - ready_copies == redundancy -## - Only one level of hierarchical grouping defined. -## - That level distributes copies to all defined groups. -active_per_leaf_group bool default=false - -## Search have some unfortunate properties with some queries, adding a static -## query cost, independent of the number of documents searched on a node. For -## these cases they use many small groups to be able to scale. In such cases, -## where groups are small, having distributors able to take over when none are -## available in a group is useful. -distributor_auto_ownership_transfer_on_whole_group_down bool default=true - -## Hierarchical grouping divides the nodes into a tree of groups. Due to config -## liking flat structures. The tree of groups is represented by a single array -## of groups defined here, where index is a string that can have a form like -## "1.2.1" in order to tell where in the tree structure the group is. -group[].index string -## Each group needs to have a name. Makes it easier to operations that they can -## name their groups according to what they want to call them. -group[].name string -## Capacity of the given group -group[].capacity double default=1 -## Partitions define how copies are divided among child groups/nodes. -group[].partitions string default="" -## Leaf groups will have a set of nodes within them. Branch groups will have -## none. -group[].nodes[].index int -# Whether this node is retired. -# The system will migrate all data away from retired nodes such that they can -# eventually be removed without partial data loss -group[].nodes[].retired bool default=false - -## Which disk distribution to use. From the current choices, we very much -## recommend using MODULO_BID. The only reason to use any of the earlier one is -## during an upgrade where you dont want to mess up the distribution in case you -## want to revert the upgrade. -## -## MODULO old one (4.0) -## MODULO_INDEX with node index in seed -## MODULO_KNUTH with random(node index) in seed -## MODULO_BID using all used bits, except count bits, and random(node index) -disk_distribution enum { MODULO, MODULO_INDEX, MODULO_KNUTH, MODULO_BID } default=MODULO_BID diff --git a/configdefinitions/src/vespa/vespa.config.content.stor-filestor.def b/configdefinitions/src/vespa/vespa.config.content.stor-filestor.def deleted file mode 100644 index 8ac372c0575..00000000000 --- a/configdefinitions/src/vespa/vespa.config.content.stor-filestor.def +++ /dev/null @@ -1,60 +0,0 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -namespace=vespa.config.content - -## DETECT FAILURE PARAMETERS - -## After seeing given number of errors on a disk, storage will disable the -## disk and restart. If set to 0, storage will never disable disks. Note -## that if you get disk errors, which arent automatically fixed, this will -## make storage keep failing all operations towards this disk, and all -## retries will use the same disk and same file, so you might end up with a -## lot of operations that can never succeed. Thus, it is not recommended to -## turn this off. -fail_disk_after_error_count int default=1 restart - -## If a disk operation takes more time in seconds than the disk operation -## timeout, storage will consider that a disk failure, and mark the disk -## bad, restart and not use the disk anymore. Note that currently, this -## includes CPU time too, so if used, this parameter should be set fairly -## high as we still sometimes have some operations that might take a few -## seconds. Disabled by default as we currently have these long operations -## during normal load. -disk_operation_timeout int default=0 restart - -## PERFORMANCE PARAMETERS - -## Number of threads to use for each mountpoint. -num_threads int default=8 restart - -## Number of threads for response processing and delivery -## 0 will give legacy sync behavior. -## Negative number will choose a good number based on # cores. -num_response_threads int default=2 restart - -## Type of sequenced thread executor use for persistence replies. -response_sequencer_type enum {LATENCY, THROUGHPUT, ADAPTIVE} default=ADAPTIVE restart - -## When merging, if we find more than this number of documents that exist on all -## of the same copies, send a separate apply bucket diff with these entries -## to an optimized merge chain that guarantuees minimum data transfer. -common_merge_chain_optimalization_minimum_size int default=64 restart - -## Chunksize to use while merging buckets between nodes. -## -## Default is set to 4 MB - 4k. This is to allow for malloc to waste some bytes -## with tracking info or to align to 512b without passing allocation limit, -## while still reading 4k blocks from disk. -bucket_merge_chunk_size int default=4190208 restart - -## When merging, it is possible to send more metadata than needed in order to -## let local nodes in merge decide which entries fits best to add this time -## based on disk location. Toggle this option on to use it. Note that memory -## consumption might increase in a 4.1 to 4.2 upgrade due to this, as 4.1 -## dont support to only fill in part of the metadata provided and will always -## fill all. -enable_merge_local_node_choose_docs_optimalization bool default=true restart - -## Whether or not to enable the multibit split optimalization. This is useful -## if splitting is expensive, but listing document identifiers is fairly cheap. -## This is true for memfile persistence layer, but not for vespa search. -enable_multibit_split_optimalization bool default=true restart diff --git a/configdefinitions/src/vespa/vespa.config.content.upgrading.def b/configdefinitions/src/vespa/vespa.config.content.upgrading.def deleted file mode 100644 index 3f610c9352d..00000000000 --- a/configdefinitions/src/vespa/vespa.config.content.upgrading.def +++ /dev/null @@ -1,24 +0,0 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -namespace=vespa.config.content - -## Config specifying cluster global parameters for a storage cluster. - -## Set to true when there exist nodes of previous major version in cluster. -## This will enable major version backward compability mode in the cluster. -upgrading_major_to bool default=false - -## Set to true when there exist nodes of the next major version in the -## cluster. This will usually not be needed, unless specified in the upgrade -## docs. Sometimes though, it may be easier to let old version be forward -## compatable than having new version backwards compatable. -upgrading_major_from bool default=false - -## Set to true when there exist nodes of previous minor version in cluster. -## This should not be needed, but settings exist in case we end up having -## no other choice than to have a compatability mode between minor versions. -upgrading_minor_to bool default=false - -## Set to true when there exist nodes of next minor version in cluster. -## This should not be needed, but settings exist in case we end up having -## no other choice than to have a compatability mode between minor versions. -upgrading_minor_from bool default=false diff --git a/configdefinitions/src/vespa/vespa.config.core.stateserver.def b/configdefinitions/src/vespa/vespa.config.core.stateserver.def deleted file mode 100644 index a25844fa148..00000000000 --- a/configdefinitions/src/vespa/vespa.config.core.stateserver.def +++ /dev/null @@ -1,5 +0,0 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -namespace=vespa.config.core - -# Port used for serving the state/vN api -httpport int default = 0 diff --git a/configdefinitions/src/vespa/vespa.config.search.attributes.def b/configdefinitions/src/vespa/vespa.config.search.attributes.def deleted file mode 100644 index 6c69d71fdf6..00000000000 --- a/configdefinitions/src/vespa/vespa.config.search.attributes.def +++ /dev/null @@ -1,45 +0,0 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -namespace=vespa.config.search - -attribute[].name string -attribute[].datatype enum { STRING, BOOL, UINT2, UINT4, INT8, INT16, INT32, INT64, FLOAT16, FLOAT, DOUBLE, PREDICATE, TENSOR, REFERENCE, NONE } default=NONE -attribute[].collectiontype enum { SINGLE, ARRAY, WEIGHTEDSET } default=SINGLE -attribute[].removeifzero bool default=false -attribute[].createifnonexistent bool default=false -attribute[].fastsearch bool default=false -attribute[].huge bool default=false -# An attribute marked mutable can be updated by a query. -attribute[].ismutable bool default=false -attribute[].sortascending bool default=true -attribute[].sortfunction enum { RAW, LOWERCASE, UCA } default=UCA -attribute[].sortstrength enum { PRIMARY, SECONDARY, TERTIARY, QUATERNARY, IDENTICAL } default=PRIMARY -attribute[].sortlocale string default="" -# Allow bitvector postings in addition to btree postings ? -attribute[].enablebitvectors bool default=false -# Allow only bitvector postings, i.e. drop btree postings to save memory.? -attribute[].enableonlybitvector bool default=false -# Allow fast access to this attribute at all times. -# If so, attribute is kept in memory also for non-searchable documents. -attribute[].fastaccess bool default=false -attribute[].arity int default=8 -attribute[].lowerbound long default=-9223372036854775808 -attribute[].upperbound long default=9223372036854775807 -# The threshold is given as a ratio of the corpus size -attribute[].densepostinglistthreshold double default=0.40 -# Specification of tensor type if this attribute is of type TENSOR. -attribute[].tensortype string default="" -# Whether this is an imported attribute (from parent document db) or not. -attribute[].imported bool default=false - -# The distance metric to use for nearest neighbor search. -# Is only used when the attribute is a 1-dimensional indexed tensor. -attribute[].distancemetric enum { EUCLIDEAN, ANGULAR, GEODEGREES, INNERPRODUCT, HAMMING } default=EUCLIDEAN - -# Configuration parameters for a hnsw index used together with a 1-dimensional indexed tensor for approximate nearest neighbor search. -attribute[].index.hnsw.enabled bool default=false -attribute[].index.hnsw.maxlinkspernode int default=16 -attribute[].index.hnsw.neighborstoexploreatinsert int default=200 -# Deprecated: Remove on Vespa 8 or before when possible. -attribute[].index.hnsw.distancemetric enum { EUCLIDEAN, ANGULAR, GEODEGREES, HAMMING } default=EUCLIDEAN -# Whether multi-threaded indexing is enabled for this hnsw index. -attribute[].index.hnsw.multithreadedindexing bool default=true diff --git a/configdefinitions/src/vespa/vespa.config.search.dispatch.def b/configdefinitions/src/vespa/vespa.config.search.dispatch.def deleted file mode 100644 index aa40c317d75..00000000000 --- a/configdefinitions/src/vespa/vespa.config.search.dispatch.def +++ /dev/null @@ -1,73 +0,0 @@ -# Copyright 2019 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -# Configuration of dispatch from container nodes to search clusters - -namespace=vespa.config.search - -# The active docs a group must have as a % of the average active docs of all other groups, -# for that group to be included in queries -minActivedocsPercentage double default=97.0 - -# Minimum coverage for allowing a group to be considered for serving -minGroupCoverage double default=100 - -# Maximum number of nodes allowed to be down for group to be considered for serving -maxNodesDownPerGroup int default=0 - -# Distribution policy for group selection -distributionPolicy enum { ROUNDROBIN, ADAPTIVE } default=ADAPTIVE - -## Maximum number of hits that will be requested from a single node -## in this dataset. If not set, there is no limit. Using this option -## may help reduce network traffic when searching in datasets with big -## fan-out, but it will also result in incorrect and incomplete results; -## don't use it if you don't (really) mean it. -maxHitsPerNode int default=2147483647 - -## Probability for getting the K best hits (topK). -## A value of 1.0 will ask all N partitions for K hits. -## Any value between <0, 1> will use a Student T with 30 degrees freedom and compute a value Q that -## will give you the globally K best hits according to this formula with the desired probability. -## q = k/n + qT (p',30) x √(k × (1/n) × (1 − 1/n)) -## With a probability of 0.999 and K=200 and N=10 will give a Q of 38, meaning that you only need to fetch 19% compared to -## a setting of 1.0. This is a significant optimisation with with very little loss in presicion. -topKProbability double default=0.9999 - -# Is multi-level dispatch configured for this cluster -# Deprecated, will go away soon, NOOP -useMultilevelDispatch bool default=false - -# Dispatch only to local nodes. DEPRECATED: The container will automatically do this when it is appropriate. -useLocalNode bool default=false - -# Number of document copies -searchableCopies long default=1 - -# Minimum search coverage required before returning the results of a query -minSearchCoverage double default=100 - -# Minimum wait time for full coverage after minimum coverage is achieved, factored based on time left at minimum coverage -minWaitAfterCoverageFactor double default=0 - -# Maximum wait time for full coverage after minimum coverage is achieved, factored based on time left at minimum coverage -maxWaitAfterCoverageFactor double default=1 - -# Number of JRT transport threads -numJrtTransportThreads int default=8 - -# Number of JRT connections per backend node -numJrtConnectionsPerNode int default=8 - -# Number of seconds to spend warming up code to prevent JIT cold start issues. -warmuptime double default=0.1 - -# The unique key of a search node -node[].key int - -# The index of the group this search node belongs to -node[].group int default=0 - -# The host name of this search node -node[].host string - -# The rpc port of this search node -node[].port int diff --git a/configdefinitions/src/vespa/vespa.config.search.imported-fields.def b/configdefinitions/src/vespa/vespa.config.search.imported-fields.def deleted file mode 100644 index 0947ff63e2a..00000000000 --- a/configdefinitions/src/vespa/vespa.config.search.imported-fields.def +++ /dev/null @@ -1,11 +0,0 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -namespace=vespa.config.search - -# The name of an imported attribute field in context of this document type. -attribute[].name string - -# The name of the field (in this document type) that refers to a parent document type. -attribute[].referencefield string - -# The name of the target attribute field in the parent document type that is imported into this document type. -attribute[].targetfield string diff --git a/configdefinitions/src/vespa/vespa.config.search.indexschema.def b/configdefinitions/src/vespa/vespa.config.search.indexschema.def deleted file mode 100644 index 245570ef7b5..00000000000 --- a/configdefinitions/src/vespa/vespa.config.search.indexschema.def +++ /dev/null @@ -1,25 +0,0 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -## Config specifying the index fields and field collections that are part of an index schema. -namespace=vespa.config.search - -## The name of the index field. -indexfield[].name string -## The data type of the index field. -indexfield[].datatype enum { STRING, INT64 } default=STRING -## The collection type of the index field. -indexfield[].collectiontype enum { SINGLE, ARRAY, WEIGHTEDSET } default=SINGLE -## Whether the index should support prefix searches. -indexfield[].prefix bool default=false -## Whether the index should support phrase searches. -indexfield[].phrases bool default=false -## Whether the index should have posting lists with word positions. -indexfield[].positions bool default=true -## Average element length -indexfield[].averageelementlen int default=512 -## Whether the index field should use posting lists with interleaved features or not. -indexfield[].interleavedfeatures bool default=false - -## The name of the field collection (aka logical view). -fieldset[].name string -## The name of the index field that are part of this collection. -fieldset[].field[].name string diff --git a/configdefinitions/src/vespa/vespa.config.search.rank-profiles.def b/configdefinitions/src/vespa/vespa.config.search.rank-profiles.def deleted file mode 100644 index 525930c990b..00000000000 --- a/configdefinitions/src/vespa/vespa.config.search.rank-profiles.def +++ /dev/null @@ -1,12 +0,0 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -namespace=vespa.config.search - -## name of this rank profile. maps to table index for internal use. -rankprofile[].name string - -## the name of a generic property available to the feature execution framework and feature plugins -rankprofile[].fef.property[].name string - -## the value of a generic property available to feature plugins -rankprofile[].fef.property[].value string - diff --git a/configdefinitions/src/vespa/vespa.config.search.summary.def b/configdefinitions/src/vespa/vespa.config.search.summary.def deleted file mode 100644 index 20ca6b10450..00000000000 --- a/configdefinitions/src/vespa/vespa.config.search.summary.def +++ /dev/null @@ -1,8 +0,0 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -namespace=vespa.config.search - -defaultsummaryid int default=-1 -classes[].id int -classes[].name string -classes[].fields[].name string -classes[].fields[].type string diff --git a/configdefinitions/src/vespa/vespa.config.search.summarymap.def b/configdefinitions/src/vespa/vespa.config.search.summarymap.def deleted file mode 100644 index a994309e207..00000000000 --- a/configdefinitions/src/vespa/vespa.config.search.summarymap.def +++ /dev/null @@ -1,15 +0,0 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -namespace=vespa.config.search - -## The default output summary class id, -1 to use the stored summary class id -defaultoutputclass int default=-1 - -## The name of the summary field which is overridden with a dynamic value -override[].field string - -## The overriding command, one of: -## staticrank, dynamicteaser, dynamicteasermetric, label, ranklog, empty, copy -override[].command string - -## Space-seaparated arguments, dedendent on the given command -override[].arguments string default="" diff --git a/configdefinitions/src/vespa/vespa.configdefinition.ilscripts.def b/configdefinitions/src/vespa/vespa.configdefinition.ilscripts.def deleted file mode 100644 index 901e87dbd04..00000000000 --- a/configdefinitions/src/vespa/vespa.configdefinition.ilscripts.def +++ /dev/null @@ -1,10 +0,0 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -namespace=vespa.configdefinition - -## The maximum number of occurrences of a given term to index per field -maxtermoccurrences int default=100 -fieldmatchmaxlength int default=1000000 - -ilscript[].doctype string -ilscript[].docfield[] string -ilscript[].content[] string diff --git a/configdefinitions/src/vespa/vespa.configdefinition.specialtokens.def b/configdefinitions/src/vespa/vespa.configdefinition.specialtokens.def deleted file mode 100644 index 1e6addcf358..00000000000 --- a/configdefinitions/src/vespa/vespa.configdefinition.specialtokens.def +++ /dev/null @@ -1,32 +0,0 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -## Named lists of special tokens - string which should be -## treated as words no matter what characters they consist of. -## A special token can also optionally be replaced by another -## token. -## -## Special tokens are case sensitive, since queries and tokens -## are lowercased they should normally be written in lower case. -## -## The list named default is the only list which is used currently -## -## Order matters in token lists. If one special token is a -## prefix of another token, Vespa will prefer to return the first -## matching token in the list when encountering the longest -## special token. -namespace=vespa.configdefinition - -## Path to makefsa binary, needed to create specialtokens-dictionary -## If the path is relative, vespa home is prepended -makefsaPath string default="bin/vespa-makefsa" - -## Use 'default' as the name of the list to use in regular -## indexing and queries -tokenlist[].name string - -## Special tokens, any string is permitted -tokenlist[].tokens[].token string - -## The token which should replace the special token -## If no replace is set for a token, the word token -## will be the special token string itself -tokenlist[].tokens[].replace string default="" diff --git a/configdefinitions/src/vespa/vespa.hosted.athenz.instanceproviderservice.config.athenz-provider-service.def b/configdefinitions/src/vespa/vespa.hosted.athenz.instanceproviderservice.config.athenz-provider-service.def deleted file mode 100644 index bd929cb17d4..00000000000 --- a/configdefinitions/src/vespa/vespa.hosted.athenz.instanceproviderservice.config.athenz-provider-service.def +++ /dev/null @@ -1,29 +0,0 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -namespace=vespa.hosted.athenz.instanceproviderservice.config - -# Athenz domain -domain string - -# Athenz service name -serviceName string - -# Secret name of private Key -secretName string - -# Secret version -secretVersion int - -# Certificate DNS suffix -certDnsSuffix string - -# Athenz ZTS server url -ztsUrl string - -# Path to Athenz CA JKS trust store -athenzCaTrustStore string - -# Period between certificate updates -updatePeriodDays int default=1 - -# Tenant Service id -tenantService string default=vespa.vespa.tenant \ No newline at end of file diff --git a/configdefinitions/src/vespa/vespa.orchestrator.config.orchestrator.def b/configdefinitions/src/vespa/vespa.orchestrator.config.orchestrator.def deleted file mode 100644 index ab52c777398..00000000000 --- a/configdefinitions/src/vespa/vespa.orchestrator.config.orchestrator.def +++ /dev/null @@ -1,5 +0,0 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -namespace=vespa.orchestrator.config - -# TODO: Change the default to actual latency in real setup. -serviceMonitorConvergenceLatencySeconds int default=0 diff --git a/configdefinitions/src/vespa/ymon.def b/configdefinitions/src/vespa/ymon.def new file mode 100644 index 00000000000..2296248bf8b --- /dev/null +++ b/configdefinitions/src/vespa/ymon.def @@ -0,0 +1,6 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +namespace=cloud.config + +services[].servicetype string default="(unknownservicetype)" +services[].ymonname string default="(unknownymonname)" +services[].hosts[] string diff --git a/configdefinitions/src/vespa/zookeeper-server.def b/configdefinitions/src/vespa/zookeeper-server.def new file mode 100644 index 00000000000..483e772b818 --- /dev/null +++ b/configdefinitions/src/vespa/zookeeper-server.def @@ -0,0 +1,45 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +namespace=cloud.config + +# Vespa home is prepended if the file is relative +zooKeeperConfigFile string default="conf/zookeeper/zookeeper.cfg" + +# For more info about the values below, see ZooKeeper documentation + +# tick time in milliseconds +tickTime int default=2000 +initLimit int default=20 +syncLimit int default=15 +maxClientConnections int default=0 + +# Vespa home is prepended if the file is relative +dataDir string default="var/zookeeper" + +clientPort int default=2181 +secureClientPort int default=2184 + +snapshotCount int default=50000 +# Purge interval in hours +autopurge.purgeInterval int default=1 +autopurge.snapRetainCount int default=15 + +# Vespa home is prepended if the file is relative +myidFile string default="var/zookeeper/myid" +# Change from default of 1 Mb in zookeeper to 50 Mb +juteMaxBuffer int default=52428800 + +myid int +server[].id int +server[].hostname string +server[].quorumPort int default=2182 +server[].electionPort int default=2183 + +# Needed when upgrading from ZooKeeper 3.4 to 3.5, see https://issues.apache.org/jira/browse/ZOOKEEPER-3056, +# and in general where there is a zookeeper ensemble running that has had few transactions. +# TODO: Consider setting this to false by default (and override where appropriate) +trustEmptySnapshot bool default=true + +# TLS options +tlsForQuorumCommunication enum { OFF, PORT_UNIFICATION, TLS_WITH_PORT_UNIFICATION, TLS_ONLY } default=OFF +tlsForClientServerCommunication enum { OFF, PORT_UNIFICATION, TLS_WITH_PORT_UNIFICATION, TLS_ONLY } default=OFF +jksKeyStoreFile string default="conf/zookeeper/zookeeper.jks" diff --git a/configdefinitions/src/vespa/zookeepers.def b/configdefinitions/src/vespa/zookeepers.def new file mode 100644 index 00000000000..d12bdd334ca --- /dev/null +++ b/configdefinitions/src/vespa/zookeepers.def @@ -0,0 +1,3 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +namespace=cloud.config +zookeeperserverlist string diff --git a/document/src/vespa/document/config/document.config.documentmanager.def b/document/src/vespa/document/config/document.config.documentmanager.def deleted file mode 100644 index d53fec43e5d..00000000000 --- a/document/src/vespa/document/config/document.config.documentmanager.def +++ /dev/null @@ -1,106 +0,0 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -namespace=document.config - -## Whether to enable compression in this process. -enablecompression bool default=false - -## The Id of the datatype. Must be unique, including not -## overlapping with the internal datatypes (defined in datatype.h) -datatype[].id int - -## Use if this datatype is an array type, for instance an int -## or double array. Specifies the datatype we have an array of. -## This can be a built-in datatype or a new one created here, meaning -## that for instance arrays of arrays are allowed. -datatype[].arraytype[].datatype int - -## Map type. Keys and values can be built in types or types created here. -datatype[].maptype[].keytype int -datatype[].maptype[].valtype int - -## Use if this datatype is a weighted set type, for instance an int -## or double weighted set. Specifies the datatype we have a weighted set of. -## This can be a built-in datatype or a new one craeted here, meaning -## that for instance weighted sets of weighted sets are allowed. -datatype[].weightedsettype[].datatype int - -## Should an update to a nonexistant element cause it to be created -datatype[].weightedsettype[].createifnonexistant bool default=false - -## Should an element in a weighted set be removed if an update changes the weight to 0 -datatype[].weightedsettype[].removeifzero bool default=false - -## Specify the name of the document type. Must be unique. -datatype[].structtype[].name string - -## Verison is not in use -datatype[].structtype[].version int default=0 - -## Specify which compression to use if compression is enabled above -datatype[].structtype[].compresstype enum { NONE, UNCOMPRESSABLE, LZ4 } default=NONE - -## Specify the compression level to use if compression is enabled -datatype[].structtype[].compresslevel int default=0 - -## Specify the minimum reduction required from compression in order to keep the compressed version (maximum percentage of original size) -datatype[].structtype[].compressthreshold int default=95 - -## Specify the minimum size of the struct data before we even try to compress it -datatype[].structtype[].compressminsize int default=800 - -## Specify a document field name. Must be unique within the document type. -datatype[].structtype[].field[].name string - -## Specify a document field id. If not specified, this is generated by a hash function -datatype[].structtype[].field[].id[].id int - -## Specify the datatype of the field. Can be a built-in datatype or -## one specified in config. -datatype[].structtype[].field[].datatype int - -## Additional, optional type information which can be changed without -## (necessarily) causing field incompatibility -datatype[].structtype[].field[].detailedtype string default="" - -## Specify a document type to inherit -datatype[].structtype[].inherits[].name string - -## Version is not in use -datatype[].structtype[].inherits[].version int default=0 - -## Specity an annotation reference type and the name of the annotation type it is referencing -datatype[].annotationreftype[].annotation string - -## Specify the name of the document type. Must be unique. -datatype[].documenttype[].name string - -## Version is not in use -datatype[].documenttype[].version int default=0 - -## Specify a document type to inherit -datatype[].documenttype[].inherits[].name string - -## Version is not in use -datatype[].documenttype[].inherits[].version int default=0 - -## Name of header struct defining document header. -datatype[].documenttype[].headerstruct int - -## Specify a document field id. Must be unique within the document type. -datatype[].documenttype[].bodystruct int default=0 - -## Field sets -datatype[].documenttype[].fieldsets{}.fields[] string - -## Imported fields (specified outside the document block in the schema) -datatype[].documenttype[].importedfield[].name string - -## Cross-document reference with ID of target document type -datatype[].referencetype[].target_type_id int - -## The Id of the annotation type. Must be unique. -annotationtype[].id int -annotationtype[].name string -annotationtype[].datatype int default=-1 -annotationtype[].inherits[].id int diff --git a/document/src/vespa/document/config/document.documenttypes.def b/document/src/vespa/document/config/document.documenttypes.def deleted file mode 100644 index d02e9fe49f2..00000000000 --- a/document/src/vespa/document/config/document.documenttypes.def +++ /dev/null @@ -1,109 +0,0 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -namespace=document - -## Whether to enable compression in this process. -enablecompression bool default=false - -## The Id of the documenttype. Must be unique among all document types. -documenttype[].id int - -## Specify the name of the document type. Must be unique. -documenttype[].name string - -## Version is not used -documenttype[].version int default=0 - -## Name of header struct defining document header. -documenttype[].headerstruct int - -## Specify a document field id. Must be unique within the document type. -documenttype[].bodystruct int default=0 - -## Specify a document type to inherit (id) -documenttype[].inherits[].id int - -## This is the id of a datatype defined in the document. -documenttype[].datatype[].id int - -## This is the type of the datatype. -documenttype[].datatype[].type enum {STRUCT, ARRAY, WSET, MAP, ANNOTATIONREF, PRIMITIVE, TENSOR} - -## This is the id of the datatype of the element in the array. -documenttype[].datatype[].array.element.id int default=0 - -## This is the id of the datatype of the key in the map. -documenttype[].datatype[].map.key.id int default=0 - -## This is the id of the datatype of the value in the map. -documenttype[].datatype[].map.value.id int default=0 - -## This is the id of the datatype of the key in the wset. -documenttype[].datatype[].wset.key.id int default=0 - -## Should an update to a nonexistent element cause it to be created -documenttype[].datatype[].wset.createifnonexistent bool default=false - -## Should an element in a weighted set be removed if an update changes the weight to 0 -documenttype[].datatype[].wset.removeifzero bool default=false - -## This is the id of the referenced annotation. -documenttype[].datatype[].annotationref.annotation.id int default=0 - -## Specify the name of the struct type. Must be unique within documenttype. -documenttype[].datatype[].sstruct.name string default="" - -## Version is not used -documenttype[].datatype[].sstruct.version int default=0 - -## Specify which compression to use if compression is enabled above (0 = NONE, 6 = LZ4) -documenttype[].datatype[].sstruct.compression.type enum {NONE, LZ4} default=NONE - -## Specify the compression level to use if compression is enabled -documenttype[].datatype[].sstruct.compression.level int default=0 - -## Specify the minimum reduction required from compression in order to keep the compressed version (maximum percentage of original size) -documenttype[].datatype[].sstruct.compression.threshold int default=95 - -## Specify the minimum size before trying compression. -documenttype[].datatype[].sstruct.compression.minsize int default=200 - -## Specify a struct field name. Must be unique within the struct type. -documenttype[].datatype[].sstruct.field[].name string - -## Specify a document field id. If not specified, this is generated by a hash function -documenttype[].datatype[].sstruct.field[].id int - -## Specify the datatype of the field. Can only be a type defined in document -## or one of its inherited document types. -documenttype[].datatype[].sstruct.field[].datatype int - -## Additional, optional type information which can be changed without -## (necessarily) causing field incompatibility -documenttype[].datatype[].sstruct.field[].detailedtype string default="" - -## The id of the annotation type. -documenttype[].annotationtype[].id int - -## The name of the annotation type. -documenttype[].annotationtype[].name string - -## The contained datatype of the annotation type. -documenttype[].annotationtype[].datatype int default=-1 - -## The annotation type that this type inherits. (0 or 1 type) -documenttype[].annotationtype[].inherits[].id int - -## Field sets -documenttype[].fieldsets{}.fields[] string - -## ID of reference type. This is a regular data type, but is kept in its own -## array to avoid polluting the existing datatype array with a new default -## field value. -documenttype[].referencetype[].id int - -## Numeric ID of the document type instances of the reference point to. -documenttype[].referencetype[].target_type_id int - -## Imported fields (specified outside the document block in the schema) -documenttype[].importedfield[].name string diff --git a/document/src/vespa/document/config/documentmanager.def b/document/src/vespa/document/config/documentmanager.def new file mode 100644 index 00000000000..d53fec43e5d --- /dev/null +++ b/document/src/vespa/document/config/documentmanager.def @@ -0,0 +1,106 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +namespace=document.config + +## Whether to enable compression in this process. +enablecompression bool default=false + +## The Id of the datatype. Must be unique, including not +## overlapping with the internal datatypes (defined in datatype.h) +datatype[].id int + +## Use if this datatype is an array type, for instance an int +## or double array. Specifies the datatype we have an array of. +## This can be a built-in datatype or a new one created here, meaning +## that for instance arrays of arrays are allowed. +datatype[].arraytype[].datatype int + +## Map type. Keys and values can be built in types or types created here. +datatype[].maptype[].keytype int +datatype[].maptype[].valtype int + +## Use if this datatype is a weighted set type, for instance an int +## or double weighted set. Specifies the datatype we have a weighted set of. +## This can be a built-in datatype or a new one craeted here, meaning +## that for instance weighted sets of weighted sets are allowed. +datatype[].weightedsettype[].datatype int + +## Should an update to a nonexistant element cause it to be created +datatype[].weightedsettype[].createifnonexistant bool default=false + +## Should an element in a weighted set be removed if an update changes the weight to 0 +datatype[].weightedsettype[].removeifzero bool default=false + +## Specify the name of the document type. Must be unique. +datatype[].structtype[].name string + +## Verison is not in use +datatype[].structtype[].version int default=0 + +## Specify which compression to use if compression is enabled above +datatype[].structtype[].compresstype enum { NONE, UNCOMPRESSABLE, LZ4 } default=NONE + +## Specify the compression level to use if compression is enabled +datatype[].structtype[].compresslevel int default=0 + +## Specify the minimum reduction required from compression in order to keep the compressed version (maximum percentage of original size) +datatype[].structtype[].compressthreshold int default=95 + +## Specify the minimum size of the struct data before we even try to compress it +datatype[].structtype[].compressminsize int default=800 + +## Specify a document field name. Must be unique within the document type. +datatype[].structtype[].field[].name string + +## Specify a document field id. If not specified, this is generated by a hash function +datatype[].structtype[].field[].id[].id int + +## Specify the datatype of the field. Can be a built-in datatype or +## one specified in config. +datatype[].structtype[].field[].datatype int + +## Additional, optional type information which can be changed without +## (necessarily) causing field incompatibility +datatype[].structtype[].field[].detailedtype string default="" + +## Specify a document type to inherit +datatype[].structtype[].inherits[].name string + +## Version is not in use +datatype[].structtype[].inherits[].version int default=0 + +## Specity an annotation reference type and the name of the annotation type it is referencing +datatype[].annotationreftype[].annotation string + +## Specify the name of the document type. Must be unique. +datatype[].documenttype[].name string + +## Version is not in use +datatype[].documenttype[].version int default=0 + +## Specify a document type to inherit +datatype[].documenttype[].inherits[].name string + +## Version is not in use +datatype[].documenttype[].inherits[].version int default=0 + +## Name of header struct defining document header. +datatype[].documenttype[].headerstruct int + +## Specify a document field id. Must be unique within the document type. +datatype[].documenttype[].bodystruct int default=0 + +## Field sets +datatype[].documenttype[].fieldsets{}.fields[] string + +## Imported fields (specified outside the document block in the schema) +datatype[].documenttype[].importedfield[].name string + +## Cross-document reference with ID of target document type +datatype[].referencetype[].target_type_id int + +## The Id of the annotation type. Must be unique. +annotationtype[].id int +annotationtype[].name string +annotationtype[].datatype int default=-1 +annotationtype[].inherits[].id int diff --git a/document/src/vespa/document/config/documenttypes.def b/document/src/vespa/document/config/documenttypes.def new file mode 100644 index 00000000000..d02e9fe49f2 --- /dev/null +++ b/document/src/vespa/document/config/documenttypes.def @@ -0,0 +1,109 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +namespace=document + +## Whether to enable compression in this process. +enablecompression bool default=false + +## The Id of the documenttype. Must be unique among all document types. +documenttype[].id int + +## Specify the name of the document type. Must be unique. +documenttype[].name string + +## Version is not used +documenttype[].version int default=0 + +## Name of header struct defining document header. +documenttype[].headerstruct int + +## Specify a document field id. Must be unique within the document type. +documenttype[].bodystruct int default=0 + +## Specify a document type to inherit (id) +documenttype[].inherits[].id int + +## This is the id of a datatype defined in the document. +documenttype[].datatype[].id int + +## This is the type of the datatype. +documenttype[].datatype[].type enum {STRUCT, ARRAY, WSET, MAP, ANNOTATIONREF, PRIMITIVE, TENSOR} + +## This is the id of the datatype of the element in the array. +documenttype[].datatype[].array.element.id int default=0 + +## This is the id of the datatype of the key in the map. +documenttype[].datatype[].map.key.id int default=0 + +## This is the id of the datatype of the value in the map. +documenttype[].datatype[].map.value.id int default=0 + +## This is the id of the datatype of the key in the wset. +documenttype[].datatype[].wset.key.id int default=0 + +## Should an update to a nonexistent element cause it to be created +documenttype[].datatype[].wset.createifnonexistent bool default=false + +## Should an element in a weighted set be removed if an update changes the weight to 0 +documenttype[].datatype[].wset.removeifzero bool default=false + +## This is the id of the referenced annotation. +documenttype[].datatype[].annotationref.annotation.id int default=0 + +## Specify the name of the struct type. Must be unique within documenttype. +documenttype[].datatype[].sstruct.name string default="" + +## Version is not used +documenttype[].datatype[].sstruct.version int default=0 + +## Specify which compression to use if compression is enabled above (0 = NONE, 6 = LZ4) +documenttype[].datatype[].sstruct.compression.type enum {NONE, LZ4} default=NONE + +## Specify the compression level to use if compression is enabled +documenttype[].datatype[].sstruct.compression.level int default=0 + +## Specify the minimum reduction required from compression in order to keep the compressed version (maximum percentage of original size) +documenttype[].datatype[].sstruct.compression.threshold int default=95 + +## Specify the minimum size before trying compression. +documenttype[].datatype[].sstruct.compression.minsize int default=200 + +## Specify a struct field name. Must be unique within the struct type. +documenttype[].datatype[].sstruct.field[].name string + +## Specify a document field id. If not specified, this is generated by a hash function +documenttype[].datatype[].sstruct.field[].id int + +## Specify the datatype of the field. Can only be a type defined in document +## or one of its inherited document types. +documenttype[].datatype[].sstruct.field[].datatype int + +## Additional, optional type information which can be changed without +## (necessarily) causing field incompatibility +documenttype[].datatype[].sstruct.field[].detailedtype string default="" + +## The id of the annotation type. +documenttype[].annotationtype[].id int + +## The name of the annotation type. +documenttype[].annotationtype[].name string + +## The contained datatype of the annotation type. +documenttype[].annotationtype[].datatype int default=-1 + +## The annotation type that this type inherits. (0 or 1 type) +documenttype[].annotationtype[].inherits[].id int + +## Field sets +documenttype[].fieldsets{}.fields[] string + +## ID of reference type. This is a regular data type, but is kept in its own +## array to avoid polluting the existing datatype array with a new default +## field value. +documenttype[].referencetype[].id int + +## Numeric ID of the document type instances of the reference point to. +documenttype[].referencetype[].target_type_id int + +## Imported fields (specified outside the document block in the schema) +documenttype[].importedfield[].name string diff --git a/fileacquirer/src/vespa/fileacquirer/cloud.config.filedistribution.filedistributorrpc.def b/fileacquirer/src/vespa/fileacquirer/cloud.config.filedistribution.filedistributorrpc.def deleted file mode 120000 index c12c0b8299e..00000000000 --- a/fileacquirer/src/vespa/fileacquirer/cloud.config.filedistribution.filedistributorrpc.def +++ /dev/null @@ -1 +0,0 @@ -../../main/resources/configdefinitions/filedistributorrpc.def \ No newline at end of file diff --git a/fileacquirer/src/vespa/fileacquirer/filedistributorrpc.def b/fileacquirer/src/vespa/fileacquirer/filedistributorrpc.def new file mode 120000 index 00000000000..c12c0b8299e --- /dev/null +++ b/fileacquirer/src/vespa/fileacquirer/filedistributorrpc.def @@ -0,0 +1 @@ +../../main/resources/configdefinitions/filedistributorrpc.def \ No newline at end of file diff --git a/metrics/src/vespa/metrics/metrics.metricsmanager.def b/metrics/src/vespa/metrics/metrics.metricsmanager.def deleted file mode 100644 index 80f03a8e9e8..00000000000 --- a/metrics/src/vespa/metrics/metrics.metricsmanager.def +++ /dev/null @@ -1,63 +0,0 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -namespace=metrics - -# If any snapshot periods is set, these override all the default ones. -# The array sets the snapshot periods in number of seconds. Note that when -# sorted in rising time length order, it is required that the snapshot length of -# the next snapshot is a multiplum of the previous snapshot. -## restart flag was added automatically and needs to be verified. -snapshot.periods[] int restart - -# The name of the consumer that should include the given metrics specified. -## restart flag was added automatically and needs to be verified. -consumer[].name string restart - -# Include metrics that match the given tags. -# -# A tag specification either adds a single tag by given the full tag name, or -# any tag, by specifying an asterix as wildcard ('*'). An asterix will not match -# metrics without any tags set, but you can match these by specifying an empty -# string. -# -# A metric set matched by tags, will also include every metric contained in it -# regardless of tags. -## restart flag was added automatically and needs to be verified. -consumer[].tags[] string restart - -# Do not include metrics that match the specific tags. This has higher -# presedence than the adding of tags, but lower presedence than name patterns. -# -# Removed tags are specified as added tags, with a small exception. Removing -# wildcard or untagged metrics does not make sense, as metrics are not added by -# default. If you don not want to include these metrics, just not specify them -# as tags to be added. Thus, these are not allowed in remove tags spec. -# -# A metric set matched by remove tag will remove all metrics below that tag. -## restart flag was added automatically and needs to be verified. -consumer[].removedtags[] string restart - -# Include all metrics that matches the names given here. -# Metrics added here have presedence above tag specifications. -# -# Names are specified as a complete dot separated path to metric. Asterix can -# be used to match any part. An asterix will only match a complete name without -# a dot. So if you have an average metric called 'queuesize' in your top level -# metric set called 'myapp', you can add this metric with the spec -# 'myapp.queuesize'. You can also add non-default average metric entries by -# specifying which, such as 'myapp.queuesize.last' or 'myapp.queuesize.max'. -# Patterns can be used, to for instance specify '*.queuesize', 'myapp.*', '*.*', -# 'myapp.*.max' or similar. -# -# A metric set added with specific name will add all metrics within it. -## restart flag was added automatically and needs to be verified. -consumer[].addedmetrics[] string restart - -# Do not include metrics with the below names. -# This has highest presedence. Metrics removed like this will override any -# specification otherwise that would include them. -# -# The remove metrics are specified exactly the same as the added metrics. -# -# A metric set removed with specific name will remove all metrics within it. -## restart flag was added automatically and needs to be verified. -consumer[].removedmetrics[] string restart diff --git a/metrics/src/vespa/metrics/metricsmanager.def b/metrics/src/vespa/metrics/metricsmanager.def new file mode 100644 index 00000000000..80f03a8e9e8 --- /dev/null +++ b/metrics/src/vespa/metrics/metricsmanager.def @@ -0,0 +1,63 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +namespace=metrics + +# If any snapshot periods is set, these override all the default ones. +# The array sets the snapshot periods in number of seconds. Note that when +# sorted in rising time length order, it is required that the snapshot length of +# the next snapshot is a multiplum of the previous snapshot. +## restart flag was added automatically and needs to be verified. +snapshot.periods[] int restart + +# The name of the consumer that should include the given metrics specified. +## restart flag was added automatically and needs to be verified. +consumer[].name string restart + +# Include metrics that match the given tags. +# +# A tag specification either adds a single tag by given the full tag name, or +# any tag, by specifying an asterix as wildcard ('*'). An asterix will not match +# metrics without any tags set, but you can match these by specifying an empty +# string. +# +# A metric set matched by tags, will also include every metric contained in it +# regardless of tags. +## restart flag was added automatically and needs to be verified. +consumer[].tags[] string restart + +# Do not include metrics that match the specific tags. This has higher +# presedence than the adding of tags, but lower presedence than name patterns. +# +# Removed tags are specified as added tags, with a small exception. Removing +# wildcard or untagged metrics does not make sense, as metrics are not added by +# default. If you don not want to include these metrics, just not specify them +# as tags to be added. Thus, these are not allowed in remove tags spec. +# +# A metric set matched by remove tag will remove all metrics below that tag. +## restart flag was added automatically and needs to be verified. +consumer[].removedtags[] string restart + +# Include all metrics that matches the names given here. +# Metrics added here have presedence above tag specifications. +# +# Names are specified as a complete dot separated path to metric. Asterix can +# be used to match any part. An asterix will only match a complete name without +# a dot. So if you have an average metric called 'queuesize' in your top level +# metric set called 'myapp', you can add this metric with the spec +# 'myapp.queuesize'. You can also add non-default average metric entries by +# specifying which, such as 'myapp.queuesize.last' or 'myapp.queuesize.max'. +# Patterns can be used, to for instance specify '*.queuesize', 'myapp.*', '*.*', +# 'myapp.*.max' or similar. +# +# A metric set added with specific name will add all metrics within it. +## restart flag was added automatically and needs to be verified. +consumer[].addedmetrics[] string restart + +# Do not include metrics with the below names. +# This has highest presedence. Metrics removed like this will override any +# specification otherwise that would include them. +# +# The remove metrics are specified exactly the same as the added metrics. +# +# A metric set removed with specific name will remove all metrics within it. +## restart flag was added automatically and needs to be verified. +consumer[].removedmetrics[] string restart diff --git a/searchcore/src/apps/verify_ranksetup/verify-ranksetup.def b/searchcore/src/apps/verify_ranksetup/verify-ranksetup.def new file mode 100644 index 00000000000..f2199c4e726 --- /dev/null +++ b/searchcore/src/apps/verify_ranksetup/verify-ranksetup.def @@ -0,0 +1,5 @@ +# Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +namespace=vespa.config.search.core + +file[].ref string +file[].path string diff --git a/searchcore/src/apps/verify_ranksetup/vespa.config.search.core.verify-ranksetup.def b/searchcore/src/apps/verify_ranksetup/vespa.config.search.core.verify-ranksetup.def deleted file mode 100644 index f2199c4e726..00000000000 --- a/searchcore/src/apps/verify_ranksetup/vespa.config.search.core.verify-ranksetup.def +++ /dev/null @@ -1,5 +0,0 @@ -# Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -namespace=vespa.config.search.core - -file[].ref string -file[].path string diff --git a/searchcore/src/tests/proton/documentdb/fileconfigmanager/config.mycfg.def b/searchcore/src/tests/proton/documentdb/fileconfigmanager/config.mycfg.def deleted file mode 100644 index 52b374e1bbf..00000000000 --- a/searchcore/src/tests/proton/documentdb/fileconfigmanager/config.mycfg.def +++ /dev/null @@ -1,4 +0,0 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -namespace=config - -myField string default="" diff --git a/searchcore/src/tests/proton/documentdb/fileconfigmanager/mycfg.def b/searchcore/src/tests/proton/documentdb/fileconfigmanager/mycfg.def new file mode 100644 index 00000000000..52b374e1bbf --- /dev/null +++ b/searchcore/src/tests/proton/documentdb/fileconfigmanager/mycfg.def @@ -0,0 +1,4 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +namespace=config + +myField string default="" diff --git a/searchcore/src/vespa/searchcore/config/fdispatchrc.def b/searchcore/src/vespa/searchcore/config/fdispatchrc.def new file mode 100644 index 00000000000..f2e33e463b8 --- /dev/null +++ b/searchcore/src/vespa/searchcore/config/fdispatchrc.def @@ -0,0 +1,5 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +namespace=vespa.config.search.core + +## The number of transport threads used when talking to search nodes. +transportthreads int default=1 restart diff --git a/searchcore/src/vespa/searchcore/config/hwinfo.def b/searchcore/src/vespa/searchcore/config/hwinfo.def new file mode 100644 index 00000000000..981c3fe655e --- /dev/null +++ b/searchcore/src/vespa/searchcore/config/hwinfo.def @@ -0,0 +1,8 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +namespace=vespa.config.search.core + +## Write speed, MiB/s, typically measured by writing 1 GiB data to disk. +disk.writespeed double default = 0.0 + +## Sample time, in seconds since epoch +disk.sampletime long default = 0 diff --git a/searchcore/src/vespa/searchcore/config/onnx-models.def b/searchcore/src/vespa/searchcore/config/onnx-models.def new file mode 100644 index 00000000000..ffe71ff70c2 --- /dev/null +++ b/searchcore/src/vespa/searchcore/config/onnx-models.def @@ -0,0 +1,5 @@ +# Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +namespace=vespa.config.search.core + +model[].name string +model[].fileref file diff --git a/searchcore/src/vespa/searchcore/config/proton.def b/searchcore/src/vespa/searchcore/config/proton.def new file mode 100644 index 00000000000..fd60781d868 --- /dev/null +++ b/searchcore/src/vespa/searchcore/config/proton.def @@ -0,0 +1,502 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +namespace=vespa.config.search.core + +## Base directory. The default is ignored as it is assigned by the model +basedir string default="." restart + +## Port to use for the rpcserver. +rpcport int default=8004 restart + +## Port to use for the web server +httpport int default=0 restart + +## Cluster name +clustername string default="" restart + +## Partition number +partition int default=0 restart + +## Distribution key +distributionkey int default=-1 + +## Num searcher threads +numsearcherthreads int default=64 restart + +## Number of threads used per search +numthreadspersearch int default=1 restart + +## Num summary threads +numsummarythreads int default=16 restart + +## Stop on io errors ? +stoponioerrors bool default=false restart + +## Perform extra validation of stored data on startup +## It requires a restart to be turned, but no restart to turned off. +## Hence it must always be followed by a manual restart. +validate_and_sanitize_docstore enum {NO, YES} default = NO + +## Maximum number of concurrent flushes outstanding. +flush.maxconcurrent int default=2 restart + +## Number of seconds between checking for stuff to flush when the system is idling. +flush.idleinterval double default=10.0 restart + +## Which flushstrategy to use. +flush.strategy enum {SIMPLE, MEMORY} default=MEMORY restart + +## The total maximum memory (in bytes) used by FLUSH components before running flush. +## A FLUSH component will free memory when flushed (e.g. memory index). +flush.memory.maxmemory long default=4294967296 + +## Maximum total disk bloat factor before forcing flush. +flush.memory.diskbloatfactor double default=0.2 + +## Max disk usage (in bytes) for all transaction logs before running flush. +## In this case the oldest component is flushed such that transaction log can be pruned and disk freed. +flush.memory.maxtlssize long default=21474836480 + +## The maximum memory (in bytes) used by a single FLUSH component before running flush. +## A FLUSH component will free memory when flushed (e.g. memory index). +flush.memory.each.maxmemory long default=1073741824 + +## Maximum disk bloat factor per component before forcing flush. +flush.memory.each.diskbloatfactor double default=0.2 + +## Age of unflushed content before forcing age prioritization. +## Unit is seconds with 1 day being the default. +flush.memory.maxage.time double default=86400.0 + +## When resource limit for memory is reached we choose a conservative mode for the flush strategy. +## In this case this factor is multiplied with 'maxmemory' and 'each.maxmemory' to calculate conservative values to use instead. +flush.memory.conservative.memorylimitfactor double default=0.5 + +## When resource limit for disk is reached we choose a conservative mode for the flush strategy. +## In this case this factor is multiplied with 'maxtlssize' to calculate a conservative value to use instead. +flush.memory.conservative.disklimitfactor double default=0.5 + +## The factor used to multiply with the resource limits for disk / memory to find the low +## watermark indicating when to go back from conservative to normal mode for the flush strategy. +flush.memory.conservative.lowwatermarkfactor double default=0.9 + +## The cost of replaying a byte when replaying the transaction log. +## +## The estimate of the total cost of replaying the transaction log: +## (number of bytes to replay) * replaycost + (number of operations to replay) * replayoperationcost +## +## The prepare for restart flush strategy will choose a set of components to flush +## such that the cost of flushing these + the cost of replaying the transaction log +## is as low as possible. +flush.preparerestart.replaycost double default=8.0 + +## The cost of replaying an operation when replaying the transaction log. +## +## The estimate of the total cost of replaying the transaction log: +## (number of bytes to replay) * replaycost + (number of operations to replay) * replayoperationcost +## +## The default value is chosen based on the following example: +## Assume we can replay 9 MB/s and this corresponds to 24000 ops/s. +## replayoperationcost = (bytes to replay) * replaycost / (operations to replay) = 9 MB * 8.0 / 24000 = 3000 +## +## The prepare for restart flush strategy will choose a set of components to flush +## such that the cost of flushing these + the cost of replaying the transaction log +## is as low as possible. +flush.preparerestart.replayoperationcost double default=3000.0 + +## The cost of writing a byte when flushing components to disk. +## +## The number of bytes to write (for a set of flushed components) * writecost +## gives an estimate of the total cost of flushing this set of components. +## +## The prepare for restart flush strategy will choose a set of components to flush +## such that the cost of flushing these + the cost of replaying the transaction log +## is as low as possible. +flush.preparerestart.writecost double default=1.0 + +## Control io options during write both under dump and fusion. +indexing.write.io enum {NORMAL, OSYNC, DIRECTIO} default=DIRECTIO restart + +## Control io options during read both under dump and fusion. +indexing.read.io enum {NORMAL, DIRECTIO} default=DIRECTIO restart + +## Control number of threads used for indexing +indexing.threads int default=1 restart + +## Option to specify what is most important during indexing. +## This is experimental and will most likely be temporary. +indexing.optimize enum {LATENCY, THROUGHPUT, ADAPTIVE} default=LATENCY restart + +## Maximum number of pending operations for each of the internal +## indexing threads. Only used when visibility delay is zero. +indexing.tasklimit int default=1000 restart + +## Parameter used to calculate maximum number of pending operations +## for each of the internal indexing threads when visibility delay is +## nonzero. The number is divided by the number of indexing threads, +## i.e. when indexing.threads is 4 and indexing.semiunboundtasklimit +## is 40000 then effective task limit is 10000. +indexing.semiunboundtasklimit int default = 40000 restart + +## Kind of watermark for when to activate extra manpower +## Utilized if optimize is set to either THROUGHPUT or ADAPTIVE +indexing.kind_of_watermark int default = 0 restart + +## Controls minimum reaction time in seconds if using THROUGHPUT +indexing.reactiontime double default = 0.005 restart + + +## How long a freshly loaded index shall be warmed up +## before being used for serving +index.warmup.time double default=0.0 restart + +# Indicate if we also want warm up with full unpack, instead of only cheaper seek. +index.warmup.unpack bool default=false restart + +## How many flushed indexes there can be before fusion is forced while node is +## not in retired state. +## Setting to 1 will force an immediate fusion. +index.maxflushed int default=2 + +## How many flushed indexes there can be before fusion is forced while node is +## in retired state. +## Setting to 1 will force an immediate fusion. +index.maxflushedretired int default=20 + +## How much memory is set aside for caching. +## Now only used for caching of dictionary lookups. +index.cache.size long default=0 restart + +## Control io options during flushing of attributes. +attribute.write.io enum {NORMAL, OSYNC, DIRECTIO} default=DIRECTIO restart + +## Multiple optional options for use with mmap +search.mmap.options[] enum {MLOCK, POPULATE, HUGETLB} restart + +## Advise to give to os when mapping memory. +search.mmap.advise enum {NORMAL, RANDOM, SEQUENTIAL} default=NORMAL restart + +## Max number of threads allowed to handle large queries concurrently +## Positive number means there is a limit, 0 or negative means no limit. +search.memory.limiter.maxthreads int default=0 + +## Minimum coverage of corpus to postprocess before applying above concurrency limit. +search.memory.limiter.mincoverage double default=1.0 + +## Minimum number of hits to postprocess before applying above concurrency limit. +## Both must be covered before applying limiter. +search.memory.limiter.minhits int default=1000000 + +## Control of grouping session manager entries +grouping.sessionmanager.maxentries int default=500 restart + +## Control of pruning interval to remove sessions that have timed out +grouping.sessionmanager.pruning.interval double default=1.0 + +## Redundancy of documents. +distribution.redundancy long default=1 + +## Searchable copies of the documents. +distribution.searchablecopies long default=1 + +## Control cache size in bytes. +## Postive numbers are absolute in bytes. +## Negative numbers are a percentage of memory. +summary.cache.maxbytes long default=-5 + +## Include visits in the cache, if the visitoperation allows it. +## This will enable another separate cache of summary.cache.maxbytes size. +summary.cache.allowvisitcaching bool default=true + +## Control number of cache entries preallocated. +## Default is no preallocation. +## Can be set to a higher number to avoid resizing. +summary.cache.initialentries long default=0 restart + +## Control compression type of the summary while in the cache. +summary.cache.compression.type enum {NONE, LZ4, ZSTD} default=LZ4 + +## Control compression level of the summary while in cache. +## LZ4 has normal range 1..9 while ZSTD has range 1..19 +## 6 is a default for lz4 to prioritize speed. +summary.cache.compression.level int default=6 + +## Control if cache entry is updated or ivalidated when changed. +summary.cache.update_strategy enum {INVALIDATE, UPDATE} default=INVALIDATE + +## Control compression type of the summary while in memory during compaction +## NB So far only stragey=LOG honours it. +summary.log.compact.compression.type enum {NONE, LZ4, ZSTD} default=ZSTD + +## Control compression level of the summary while in memory during compaction +## LZ4 has normal range 1..9 while ZSTD has range 1..19 +## 9 is a reasonable default for both +summary.log.compact.compression.level int default=9 + +## Control compression type of the summary +summary.log.chunk.compression.type enum {NONE, LZ4, ZSTD} default=ZSTD + +## Control compression level of the summary +## LZ4 has normal range 1..9 while ZSTD has range 1..19 +## 9 is a reasonable default for both. Going above for ZSTD can give an improvement, +## but is better done in conjunction with increasing chunk size. +summary.log.chunk.compression.level int default=9 + +## Max size in bytes per chunk. +summary.log.chunk.maxbytes int default=65536 + +## Skip crc32 check on read. +summary.log.chunk.skipcrconread bool default=false + +## Max size per summary file. +summary.log.maxfilesize long default=1000000000 + +## Max number of lid entries per file +summary.log.maxnumlids int default=40000000 + +## Max disk bloat factor. This will trigger compacting. +summary.log.maxdiskbloatfactor double default=0.1 + +## Max bucket spread within a single summary file. This will trigger bucket order compacting. +summary.log.maxbucketspread double default=2.5 + +## If a file goes below this ratio compared to allowed max size it will be joined to the front. +## Value in the range [0.0, 1.0] +summary.log.minfilesizefactor double default=0.2 + +## Control io options during flush of stored documents. +summary.write.io enum {NORMAL, OSYNC, DIRECTIO} default=DIRECTIO + +## Control io options during read of stored documents. +## All summary.read options will take effect immediately on new files written. +## On old files it will take effect either upon compact or on restart. +summary.read.io enum {NORMAL, DIRECTIO, MMAP } default=MMAP restart + +## Multiple optional options for use with mmap +summary.read.mmap.options[] enum {MLOCK, POPULATE, HUGETLB} restart + +## Advise to give to os when mapping memory. +summary.read.mmap.advise enum {NORMAL, RANDOM, SEQUENTIAL} default=NORMAL restart + +## The name of the input document type +documentdb[].inputdoctypename string +## The type of the documentdb +documentdb[].mode enum {INDEX, STREAMING, STORE_ONLY} default=INDEX +## The configid used to subscribe to config for this database. +documentdb[].configid string +## How many seconds is allowed from document is received to it is visible in the index. +documentdb[].visibilitydelay double default=0.0 +## Whether this document type is globally distributed or not. +documentdb[].global bool default=false +## A number between 0.0 and 1.0 that specifies the concurrency when handling feed operations. +## When set to 1.0 all cores on the cpu is utilized. +## +## 3 thread pools used for various aspect of feeding are configured based on this setting: +## 1) Writing changes to attribute fields +## 2) Inverting index fields +## 3) Writing changes to index fields +## +## The number of threads in each of pools is calculated as: +## max(ceil((hwinfo.cpu.cores * feeding.concurrency)/3), indexing.threads) +documentdb[].feeding.concurrency double default=0.2 + +## Minimum initial size for any per document tables. +documentdb[].allocation.initialnumdocs long default=1024 +## Grow factor for any per document tables. +documentdb[].allocation.growfactor double default=0.2 +## Constant added when growing any per document tables. +documentdb[].allocation.growbias int default=1 + +## The number of documents to amortize memory spike cost over +documentdb[].allocation.amortizecount int default=10000 + +## The grow factor used when allocating buffers in the array store +## used in multi-value attribute vectors to store underlying values. +documentdb[].allocation.multivaluegrowfactor double default=0.2 + +## The interval of when periodic tasks should be run +periodic.interval double default=3600.0 + +## Connect spec for transactionlog server. +tlsspec string default="tcp/localhost:13700" restart + +## ConfigId for transactionlogserver +tlsconfigid string default="" restart + +## Slobrok configid +slobrokconfigid string default="" restart + +## Routing configid +routingconfigid string default="" restart + +## filedistributor rpc configuration +filedistributor.configid reference default="" restart + +## Interval between pruning of old removed documents. +## +## If set to 0 (default) the value is calculated as (pruneremoveddocumentsage / 100) (default 3.36 hours). +pruneremoveddocumentsinterval double default=0.0 + +## Age of removed document before it can be pruned. +## +## Default value is 2 weeks (1209600 seconds). +pruneremoveddocumentsage double default=1209600.0 + +## Minimum size of packets to compress (0 means no compression) +## +packetcompresslimit int default=1024 + +## Compression level for packets +## +## Default value is 3 +packetcompresslevel int default=3 + +## Compression type for packets +## +## Default is LZ4 +packetcompresstype enum {NONE, LZ4} default=LZ4 + +## Interval between considering if lid space compaction should be done (in seconds). +## +## Default value is 10 seconds. +lidspacecompaction.interval double default=10.0 + +## The allowed lid bloat (in docs) before considering lid space compaction. +## +## When considering compaction the lid bloat is calculated as (docIdLimit - numDocs). +## The lid bloat must be >= allowedlidbloat before considering compaction. +lidspacecompaction.allowedlidbloat int default=1000 + +## The allowed lid bloat factor (relative) before considering lid space compaction. +## +## When considering compaction the lid bloat factor is calculated as (docIdLimit - numDocs)/docIdLimit. +## The lid bloat factor must be >= allowedlidbloatfactor before considering compaction. +lidspacecompaction.allowedlidbloatfactor double default=0.01 + +## DEPRECATED (no longer used): Remove on Vespa 8 +## The delay (in seconds) for when the last remove batch operation would be considered to block lid space compaction. +## +## When considering compaction, if the document meta store has received a remove batch operation in the last delay seconds, +## the lid space compaction job is blocked. It is considered again at the next regular interval (see above). +## +## Remove batch operations are used when deleting buckets on a content node. +## This functionality ensures that during massive deleting of buckets (e.g. as part of redistribution of data to a new node), +## lid space compaction do not interfere, but instead is applied after deleting of buckets is complete. +lidspacecompaction.removebatchblockdelay double default=2.0 + +## The rate (ops / second) of remove batch operations for when to block lid space compaction. +## +## When considering compaction, if the current observed rate of remove batch operations +## is higher than the given block rate, the lid space compaction job is blocked. +## It is considered again at the next regular interval (see above). +## +## Remove batch operations are used when deleting buckets on a content node. +## This functionality ensures that during massive deleting of buckets (e.g. as part of redistribution of data to a new node), +## lid space compaction do not interfere, but instead is applied after deleting of buckets is complete. +lidspacecompaction.removebatchblockrate double default=0.5 + +## The rate (ops / second) of remove operations for when to block lid space compaction. +## +## When considering compaction, if the current observed rate of remove operations +## is higher than the given block rate, the lid space compaction job is blocked. +## It is considered again at the next regular interval (see above). +lidspacecompaction.removeblockrate double default=100.0 + +## This is the maximum value visibilitydelay you can have. +## A to higher value here will cost more memory while not improving too much. +maxvisibilitydelay double default=1.0 + +## You can set this to a number above zero for visit to shortcut expensive serialize size computation. +## This value will be provided instead. +## negative number will compute it accurately. +## This is only used for weakly consistent visiting, like streaming search. +visit.defaultserializedsize long default=1 + +## This will ignore the maxbytes limit advised from above. +## This is only used for weakly consistent visiting, like streaming search. +visit.ignoremaxbytes bool default=true + +## Number of initializer threads used for loading structures from disk at proton startup. +## The threads are shared between document databases when value is larger than 0. +## When set to 0 (default) we use 1 separate thread per document database. +initialize.threads int default = 0 + +## Portion of enumstore address space that can be used before put and update +## portion of feed is blocked. +writefilter.attribute.enumstorelimit double default = 0.9 + +## Portion of attribute multivalue mapping address space that can be used +## before put and update portion of feed is blocked. +writefilter.attribute.multivaluelimit double default = 0.9 + +## Portion of physical memory that can be resident memory in anonymous mapping +## by the proton process before put and update portion of feed is blocked. +writefilter.memorylimit double default = 0.8 + +## Portion of space on disk partition that can be used or reserved before +## put and update portion of feed is blocked. +writefilter.disklimit double default = 0.8 + +## Interval between sampling of disk and memory usage. Default is 10 seconds. +writefilter.sampleinterval double default = 10.0 + +## The size of the disk partition (in bytes) on which proton basedir is located. +## If set to 0, the disk size is sampled by looking at the filesystem space info. +## The disk size is used when determining if feed should be blocked in writefilter. +hwinfo.disk.size long default = 0 restart + +## Whether the disk partition is shared among several instances of proton (e.g. when using docker). +## If shared, disk usage is sampled by doing a recursive directory scan in proton basedir. +## If not, disk usage is sampled by looking at the filesystem space info. +hwinfo.disk.shared bool default = false restart + +## Override for disk write speed, measured in MiB/s. When zero, the +## actual disk write speed is sampled by writing data to a temporary file. +hwinfo.disk.writespeed double default = 200.0 restart + +## Amount of data to write to temporary file when sampling disk write speed. +## Default is 1 GiB. +hwinfo.disk.samplewritesize long default = 1073741824 restart + +## Minimum write speed needed to avoid disk being considered slow. +## Unit is MiB/s, default is 100.0 MiB/s. +hwinfo.disk.slowwritespeedlimit double default = 100.0 restart + +## The size of physical memory (in bytes) available to proton. +## If set to 0, the memory size is sampled as _SC_PHYS_PAGES * _SC_PAGESIZE by using sysconf to do the lookup. +## The memory size is used when determining if feed should be blocked in writefilter. +hwinfo.memory.size long default = 0 restart + +## The number of cores on the cpu. +## If set to 0, this is sampled by using std::thread::hardware_concurrency(). +hwinfo.cpu.cores int default = 0 restart + +## A number between 0.0 and 1.0 that specifies the concurrency when handling feed operations. +## When set to 1.0 all cores on the cpu is utilized. +## +## 4 thread pools used for various aspect of feeding are configured based on this setting: +## 1) Compressing and compacting documents +## 2) Writing changes to attribute fields +## 3) Inverting index fields +## 4) Writing changes to index fields +## +## The number of threads in pool 1 is calculated as: +## max(ceil(hwinfo.cpu.cores * feeding.concurrency), summary.log.numthreads) +## The number of threads in each of pools 2-4 is calculated as: +## max(ceil((hwinfo.cpu.cores * feeding.concurrency)/3), indexing.threads) +## Deprecated -> Use documentdb.feeding.concurrency +feeding.concurrency double default = 0.2 restart + +## Adjustment to resource limit when determining if maintenance jobs can run. +## +## Currently used by 'lid_space_compaction' and 'move_buckets' jobs. +maintenancejobs.resourcelimitfactor double default = 1.05 + +## The max outstanding move operations a maintenance job can have before being blocked. +## +## The job is unblocked (and executed again) when this goes under the limit again. +## Currently used by 'lid_space_compaction' job. +maintenancejobs.maxoutstandingmoveops int default=10 + +## Controls the type of bucket checksum used. Do not change unless +## in depth understanding is present. +bucketdb.checksumtype enum {LEGACY, XXHASH64} default = LEGACY restart diff --git a/searchcore/src/vespa/searchcore/config/ranking-constants.def b/searchcore/src/vespa/searchcore/config/ranking-constants.def new file mode 100644 index 00000000000..3b55eda3308 --- /dev/null +++ b/searchcore/src/vespa/searchcore/config/ranking-constants.def @@ -0,0 +1,6 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +namespace=vespa.config.search.core + +constant[].name string +constant[].fileref file +constant[].type string diff --git a/searchcore/src/vespa/searchcore/config/vespa.config.search.core.fdispatchrc.def b/searchcore/src/vespa/searchcore/config/vespa.config.search.core.fdispatchrc.def deleted file mode 100644 index f2e33e463b8..00000000000 --- a/searchcore/src/vespa/searchcore/config/vespa.config.search.core.fdispatchrc.def +++ /dev/null @@ -1,5 +0,0 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -namespace=vespa.config.search.core - -## The number of transport threads used when talking to search nodes. -transportthreads int default=1 restart diff --git a/searchcore/src/vespa/searchcore/config/vespa.config.search.core.hwinfo.def b/searchcore/src/vespa/searchcore/config/vespa.config.search.core.hwinfo.def deleted file mode 100644 index 981c3fe655e..00000000000 --- a/searchcore/src/vespa/searchcore/config/vespa.config.search.core.hwinfo.def +++ /dev/null @@ -1,8 +0,0 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -namespace=vespa.config.search.core - -## Write speed, MiB/s, typically measured by writing 1 GiB data to disk. -disk.writespeed double default = 0.0 - -## Sample time, in seconds since epoch -disk.sampletime long default = 0 diff --git a/searchcore/src/vespa/searchcore/config/vespa.config.search.core.onnx-models.def b/searchcore/src/vespa/searchcore/config/vespa.config.search.core.onnx-models.def deleted file mode 100644 index ffe71ff70c2..00000000000 --- a/searchcore/src/vespa/searchcore/config/vespa.config.search.core.onnx-models.def +++ /dev/null @@ -1,5 +0,0 @@ -# Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -namespace=vespa.config.search.core - -model[].name string -model[].fileref file diff --git a/searchcore/src/vespa/searchcore/config/vespa.config.search.core.proton.def b/searchcore/src/vespa/searchcore/config/vespa.config.search.core.proton.def deleted file mode 100644 index fd60781d868..00000000000 --- a/searchcore/src/vespa/searchcore/config/vespa.config.search.core.proton.def +++ /dev/null @@ -1,502 +0,0 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -namespace=vespa.config.search.core - -## Base directory. The default is ignored as it is assigned by the model -basedir string default="." restart - -## Port to use for the rpcserver. -rpcport int default=8004 restart - -## Port to use for the web server -httpport int default=0 restart - -## Cluster name -clustername string default="" restart - -## Partition number -partition int default=0 restart - -## Distribution key -distributionkey int default=-1 - -## Num searcher threads -numsearcherthreads int default=64 restart - -## Number of threads used per search -numthreadspersearch int default=1 restart - -## Num summary threads -numsummarythreads int default=16 restart - -## Stop on io errors ? -stoponioerrors bool default=false restart - -## Perform extra validation of stored data on startup -## It requires a restart to be turned, but no restart to turned off. -## Hence it must always be followed by a manual restart. -validate_and_sanitize_docstore enum {NO, YES} default = NO - -## Maximum number of concurrent flushes outstanding. -flush.maxconcurrent int default=2 restart - -## Number of seconds between checking for stuff to flush when the system is idling. -flush.idleinterval double default=10.0 restart - -## Which flushstrategy to use. -flush.strategy enum {SIMPLE, MEMORY} default=MEMORY restart - -## The total maximum memory (in bytes) used by FLUSH components before running flush. -## A FLUSH component will free memory when flushed (e.g. memory index). -flush.memory.maxmemory long default=4294967296 - -## Maximum total disk bloat factor before forcing flush. -flush.memory.diskbloatfactor double default=0.2 - -## Max disk usage (in bytes) for all transaction logs before running flush. -## In this case the oldest component is flushed such that transaction log can be pruned and disk freed. -flush.memory.maxtlssize long default=21474836480 - -## The maximum memory (in bytes) used by a single FLUSH component before running flush. -## A FLUSH component will free memory when flushed (e.g. memory index). -flush.memory.each.maxmemory long default=1073741824 - -## Maximum disk bloat factor per component before forcing flush. -flush.memory.each.diskbloatfactor double default=0.2 - -## Age of unflushed content before forcing age prioritization. -## Unit is seconds with 1 day being the default. -flush.memory.maxage.time double default=86400.0 - -## When resource limit for memory is reached we choose a conservative mode for the flush strategy. -## In this case this factor is multiplied with 'maxmemory' and 'each.maxmemory' to calculate conservative values to use instead. -flush.memory.conservative.memorylimitfactor double default=0.5 - -## When resource limit for disk is reached we choose a conservative mode for the flush strategy. -## In this case this factor is multiplied with 'maxtlssize' to calculate a conservative value to use instead. -flush.memory.conservative.disklimitfactor double default=0.5 - -## The factor used to multiply with the resource limits for disk / memory to find the low -## watermark indicating when to go back from conservative to normal mode for the flush strategy. -flush.memory.conservative.lowwatermarkfactor double default=0.9 - -## The cost of replaying a byte when replaying the transaction log. -## -## The estimate of the total cost of replaying the transaction log: -## (number of bytes to replay) * replaycost + (number of operations to replay) * replayoperationcost -## -## The prepare for restart flush strategy will choose a set of components to flush -## such that the cost of flushing these + the cost of replaying the transaction log -## is as low as possible. -flush.preparerestart.replaycost double default=8.0 - -## The cost of replaying an operation when replaying the transaction log. -## -## The estimate of the total cost of replaying the transaction log: -## (number of bytes to replay) * replaycost + (number of operations to replay) * replayoperationcost -## -## The default value is chosen based on the following example: -## Assume we can replay 9 MB/s and this corresponds to 24000 ops/s. -## replayoperationcost = (bytes to replay) * replaycost / (operations to replay) = 9 MB * 8.0 / 24000 = 3000 -## -## The prepare for restart flush strategy will choose a set of components to flush -## such that the cost of flushing these + the cost of replaying the transaction log -## is as low as possible. -flush.preparerestart.replayoperationcost double default=3000.0 - -## The cost of writing a byte when flushing components to disk. -## -## The number of bytes to write (for a set of flushed components) * writecost -## gives an estimate of the total cost of flushing this set of components. -## -## The prepare for restart flush strategy will choose a set of components to flush -## such that the cost of flushing these + the cost of replaying the transaction log -## is as low as possible. -flush.preparerestart.writecost double default=1.0 - -## Control io options during write both under dump and fusion. -indexing.write.io enum {NORMAL, OSYNC, DIRECTIO} default=DIRECTIO restart - -## Control io options during read both under dump and fusion. -indexing.read.io enum {NORMAL, DIRECTIO} default=DIRECTIO restart - -## Control number of threads used for indexing -indexing.threads int default=1 restart - -## Option to specify what is most important during indexing. -## This is experimental and will most likely be temporary. -indexing.optimize enum {LATENCY, THROUGHPUT, ADAPTIVE} default=LATENCY restart - -## Maximum number of pending operations for each of the internal -## indexing threads. Only used when visibility delay is zero. -indexing.tasklimit int default=1000 restart - -## Parameter used to calculate maximum number of pending operations -## for each of the internal indexing threads when visibility delay is -## nonzero. The number is divided by the number of indexing threads, -## i.e. when indexing.threads is 4 and indexing.semiunboundtasklimit -## is 40000 then effective task limit is 10000. -indexing.semiunboundtasklimit int default = 40000 restart - -## Kind of watermark for when to activate extra manpower -## Utilized if optimize is set to either THROUGHPUT or ADAPTIVE -indexing.kind_of_watermark int default = 0 restart - -## Controls minimum reaction time in seconds if using THROUGHPUT -indexing.reactiontime double default = 0.005 restart - - -## How long a freshly loaded index shall be warmed up -## before being used for serving -index.warmup.time double default=0.0 restart - -# Indicate if we also want warm up with full unpack, instead of only cheaper seek. -index.warmup.unpack bool default=false restart - -## How many flushed indexes there can be before fusion is forced while node is -## not in retired state. -## Setting to 1 will force an immediate fusion. -index.maxflushed int default=2 - -## How many flushed indexes there can be before fusion is forced while node is -## in retired state. -## Setting to 1 will force an immediate fusion. -index.maxflushedretired int default=20 - -## How much memory is set aside for caching. -## Now only used for caching of dictionary lookups. -index.cache.size long default=0 restart - -## Control io options during flushing of attributes. -attribute.write.io enum {NORMAL, OSYNC, DIRECTIO} default=DIRECTIO restart - -## Multiple optional options for use with mmap -search.mmap.options[] enum {MLOCK, POPULATE, HUGETLB} restart - -## Advise to give to os when mapping memory. -search.mmap.advise enum {NORMAL, RANDOM, SEQUENTIAL} default=NORMAL restart - -## Max number of threads allowed to handle large queries concurrently -## Positive number means there is a limit, 0 or negative means no limit. -search.memory.limiter.maxthreads int default=0 - -## Minimum coverage of corpus to postprocess before applying above concurrency limit. -search.memory.limiter.mincoverage double default=1.0 - -## Minimum number of hits to postprocess before applying above concurrency limit. -## Both must be covered before applying limiter. -search.memory.limiter.minhits int default=1000000 - -## Control of grouping session manager entries -grouping.sessionmanager.maxentries int default=500 restart - -## Control of pruning interval to remove sessions that have timed out -grouping.sessionmanager.pruning.interval double default=1.0 - -## Redundancy of documents. -distribution.redundancy long default=1 - -## Searchable copies of the documents. -distribution.searchablecopies long default=1 - -## Control cache size in bytes. -## Postive numbers are absolute in bytes. -## Negative numbers are a percentage of memory. -summary.cache.maxbytes long default=-5 - -## Include visits in the cache, if the visitoperation allows it. -## This will enable another separate cache of summary.cache.maxbytes size. -summary.cache.allowvisitcaching bool default=true - -## Control number of cache entries preallocated. -## Default is no preallocation. -## Can be set to a higher number to avoid resizing. -summary.cache.initialentries long default=0 restart - -## Control compression type of the summary while in the cache. -summary.cache.compression.type enum {NONE, LZ4, ZSTD} default=LZ4 - -## Control compression level of the summary while in cache. -## LZ4 has normal range 1..9 while ZSTD has range 1..19 -## 6 is a default for lz4 to prioritize speed. -summary.cache.compression.level int default=6 - -## Control if cache entry is updated or ivalidated when changed. -summary.cache.update_strategy enum {INVALIDATE, UPDATE} default=INVALIDATE - -## Control compression type of the summary while in memory during compaction -## NB So far only stragey=LOG honours it. -summary.log.compact.compression.type enum {NONE, LZ4, ZSTD} default=ZSTD - -## Control compression level of the summary while in memory during compaction -## LZ4 has normal range 1..9 while ZSTD has range 1..19 -## 9 is a reasonable default for both -summary.log.compact.compression.level int default=9 - -## Control compression type of the summary -summary.log.chunk.compression.type enum {NONE, LZ4, ZSTD} default=ZSTD - -## Control compression level of the summary -## LZ4 has normal range 1..9 while ZSTD has range 1..19 -## 9 is a reasonable default for both. Going above for ZSTD can give an improvement, -## but is better done in conjunction with increasing chunk size. -summary.log.chunk.compression.level int default=9 - -## Max size in bytes per chunk. -summary.log.chunk.maxbytes int default=65536 - -## Skip crc32 check on read. -summary.log.chunk.skipcrconread bool default=false - -## Max size per summary file. -summary.log.maxfilesize long default=1000000000 - -## Max number of lid entries per file -summary.log.maxnumlids int default=40000000 - -## Max disk bloat factor. This will trigger compacting. -summary.log.maxdiskbloatfactor double default=0.1 - -## Max bucket spread within a single summary file. This will trigger bucket order compacting. -summary.log.maxbucketspread double default=2.5 - -## If a file goes below this ratio compared to allowed max size it will be joined to the front. -## Value in the range [0.0, 1.0] -summary.log.minfilesizefactor double default=0.2 - -## Control io options during flush of stored documents. -summary.write.io enum {NORMAL, OSYNC, DIRECTIO} default=DIRECTIO - -## Control io options during read of stored documents. -## All summary.read options will take effect immediately on new files written. -## On old files it will take effect either upon compact or on restart. -summary.read.io enum {NORMAL, DIRECTIO, MMAP } default=MMAP restart - -## Multiple optional options for use with mmap -summary.read.mmap.options[] enum {MLOCK, POPULATE, HUGETLB} restart - -## Advise to give to os when mapping memory. -summary.read.mmap.advise enum {NORMAL, RANDOM, SEQUENTIAL} default=NORMAL restart - -## The name of the input document type -documentdb[].inputdoctypename string -## The type of the documentdb -documentdb[].mode enum {INDEX, STREAMING, STORE_ONLY} default=INDEX -## The configid used to subscribe to config for this database. -documentdb[].configid string -## How many seconds is allowed from document is received to it is visible in the index. -documentdb[].visibilitydelay double default=0.0 -## Whether this document type is globally distributed or not. -documentdb[].global bool default=false -## A number between 0.0 and 1.0 that specifies the concurrency when handling feed operations. -## When set to 1.0 all cores on the cpu is utilized. -## -## 3 thread pools used for various aspect of feeding are configured based on this setting: -## 1) Writing changes to attribute fields -## 2) Inverting index fields -## 3) Writing changes to index fields -## -## The number of threads in each of pools is calculated as: -## max(ceil((hwinfo.cpu.cores * feeding.concurrency)/3), indexing.threads) -documentdb[].feeding.concurrency double default=0.2 - -## Minimum initial size for any per document tables. -documentdb[].allocation.initialnumdocs long default=1024 -## Grow factor for any per document tables. -documentdb[].allocation.growfactor double default=0.2 -## Constant added when growing any per document tables. -documentdb[].allocation.growbias int default=1 - -## The number of documents to amortize memory spike cost over -documentdb[].allocation.amortizecount int default=10000 - -## The grow factor used when allocating buffers in the array store -## used in multi-value attribute vectors to store underlying values. -documentdb[].allocation.multivaluegrowfactor double default=0.2 - -## The interval of when periodic tasks should be run -periodic.interval double default=3600.0 - -## Connect spec for transactionlog server. -tlsspec string default="tcp/localhost:13700" restart - -## ConfigId for transactionlogserver -tlsconfigid string default="" restart - -## Slobrok configid -slobrokconfigid string default="" restart - -## Routing configid -routingconfigid string default="" restart - -## filedistributor rpc configuration -filedistributor.configid reference default="" restart - -## Interval between pruning of old removed documents. -## -## If set to 0 (default) the value is calculated as (pruneremoveddocumentsage / 100) (default 3.36 hours). -pruneremoveddocumentsinterval double default=0.0 - -## Age of removed document before it can be pruned. -## -## Default value is 2 weeks (1209600 seconds). -pruneremoveddocumentsage double default=1209600.0 - -## Minimum size of packets to compress (0 means no compression) -## -packetcompresslimit int default=1024 - -## Compression level for packets -## -## Default value is 3 -packetcompresslevel int default=3 - -## Compression type for packets -## -## Default is LZ4 -packetcompresstype enum {NONE, LZ4} default=LZ4 - -## Interval between considering if lid space compaction should be done (in seconds). -## -## Default value is 10 seconds. -lidspacecompaction.interval double default=10.0 - -## The allowed lid bloat (in docs) before considering lid space compaction. -## -## When considering compaction the lid bloat is calculated as (docIdLimit - numDocs). -## The lid bloat must be >= allowedlidbloat before considering compaction. -lidspacecompaction.allowedlidbloat int default=1000 - -## The allowed lid bloat factor (relative) before considering lid space compaction. -## -## When considering compaction the lid bloat factor is calculated as (docIdLimit - numDocs)/docIdLimit. -## The lid bloat factor must be >= allowedlidbloatfactor before considering compaction. -lidspacecompaction.allowedlidbloatfactor double default=0.01 - -## DEPRECATED (no longer used): Remove on Vespa 8 -## The delay (in seconds) for when the last remove batch operation would be considered to block lid space compaction. -## -## When considering compaction, if the document meta store has received a remove batch operation in the last delay seconds, -## the lid space compaction job is blocked. It is considered again at the next regular interval (see above). -## -## Remove batch operations are used when deleting buckets on a content node. -## This functionality ensures that during massive deleting of buckets (e.g. as part of redistribution of data to a new node), -## lid space compaction do not interfere, but instead is applied after deleting of buckets is complete. -lidspacecompaction.removebatchblockdelay double default=2.0 - -## The rate (ops / second) of remove batch operations for when to block lid space compaction. -## -## When considering compaction, if the current observed rate of remove batch operations -## is higher than the given block rate, the lid space compaction job is blocked. -## It is considered again at the next regular interval (see above). -## -## Remove batch operations are used when deleting buckets on a content node. -## This functionality ensures that during massive deleting of buckets (e.g. as part of redistribution of data to a new node), -## lid space compaction do not interfere, but instead is applied after deleting of buckets is complete. -lidspacecompaction.removebatchblockrate double default=0.5 - -## The rate (ops / second) of remove operations for when to block lid space compaction. -## -## When considering compaction, if the current observed rate of remove operations -## is higher than the given block rate, the lid space compaction job is blocked. -## It is considered again at the next regular interval (see above). -lidspacecompaction.removeblockrate double default=100.0 - -## This is the maximum value visibilitydelay you can have. -## A to higher value here will cost more memory while not improving too much. -maxvisibilitydelay double default=1.0 - -## You can set this to a number above zero for visit to shortcut expensive serialize size computation. -## This value will be provided instead. -## negative number will compute it accurately. -## This is only used for weakly consistent visiting, like streaming search. -visit.defaultserializedsize long default=1 - -## This will ignore the maxbytes limit advised from above. -## This is only used for weakly consistent visiting, like streaming search. -visit.ignoremaxbytes bool default=true - -## Number of initializer threads used for loading structures from disk at proton startup. -## The threads are shared between document databases when value is larger than 0. -## When set to 0 (default) we use 1 separate thread per document database. -initialize.threads int default = 0 - -## Portion of enumstore address space that can be used before put and update -## portion of feed is blocked. -writefilter.attribute.enumstorelimit double default = 0.9 - -## Portion of attribute multivalue mapping address space that can be used -## before put and update portion of feed is blocked. -writefilter.attribute.multivaluelimit double default = 0.9 - -## Portion of physical memory that can be resident memory in anonymous mapping -## by the proton process before put and update portion of feed is blocked. -writefilter.memorylimit double default = 0.8 - -## Portion of space on disk partition that can be used or reserved before -## put and update portion of feed is blocked. -writefilter.disklimit double default = 0.8 - -## Interval between sampling of disk and memory usage. Default is 10 seconds. -writefilter.sampleinterval double default = 10.0 - -## The size of the disk partition (in bytes) on which proton basedir is located. -## If set to 0, the disk size is sampled by looking at the filesystem space info. -## The disk size is used when determining if feed should be blocked in writefilter. -hwinfo.disk.size long default = 0 restart - -## Whether the disk partition is shared among several instances of proton (e.g. when using docker). -## If shared, disk usage is sampled by doing a recursive directory scan in proton basedir. -## If not, disk usage is sampled by looking at the filesystem space info. -hwinfo.disk.shared bool default = false restart - -## Override for disk write speed, measured in MiB/s. When zero, the -## actual disk write speed is sampled by writing data to a temporary file. -hwinfo.disk.writespeed double default = 200.0 restart - -## Amount of data to write to temporary file when sampling disk write speed. -## Default is 1 GiB. -hwinfo.disk.samplewritesize long default = 1073741824 restart - -## Minimum write speed needed to avoid disk being considered slow. -## Unit is MiB/s, default is 100.0 MiB/s. -hwinfo.disk.slowwritespeedlimit double default = 100.0 restart - -## The size of physical memory (in bytes) available to proton. -## If set to 0, the memory size is sampled as _SC_PHYS_PAGES * _SC_PAGESIZE by using sysconf to do the lookup. -## The memory size is used when determining if feed should be blocked in writefilter. -hwinfo.memory.size long default = 0 restart - -## The number of cores on the cpu. -## If set to 0, this is sampled by using std::thread::hardware_concurrency(). -hwinfo.cpu.cores int default = 0 restart - -## A number between 0.0 and 1.0 that specifies the concurrency when handling feed operations. -## When set to 1.0 all cores on the cpu is utilized. -## -## 4 thread pools used for various aspect of feeding are configured based on this setting: -## 1) Compressing and compacting documents -## 2) Writing changes to attribute fields -## 3) Inverting index fields -## 4) Writing changes to index fields -## -## The number of threads in pool 1 is calculated as: -## max(ceil(hwinfo.cpu.cores * feeding.concurrency), summary.log.numthreads) -## The number of threads in each of pools 2-4 is calculated as: -## max(ceil((hwinfo.cpu.cores * feeding.concurrency)/3), indexing.threads) -## Deprecated -> Use documentdb.feeding.concurrency -feeding.concurrency double default = 0.2 restart - -## Adjustment to resource limit when determining if maintenance jobs can run. -## -## Currently used by 'lid_space_compaction' and 'move_buckets' jobs. -maintenancejobs.resourcelimitfactor double default = 1.05 - -## The max outstanding move operations a maintenance job can have before being blocked. -## -## The job is unblocked (and executed again) when this goes under the limit again. -## Currently used by 'lid_space_compaction' job. -maintenancejobs.maxoutstandingmoveops int default=10 - -## Controls the type of bucket checksum used. Do not change unless -## in depth understanding is present. -bucketdb.checksumtype enum {LEGACY, XXHASH64} default = LEGACY restart diff --git a/searchcore/src/vespa/searchcore/config/vespa.config.search.core.ranking-constants.def b/searchcore/src/vespa/searchcore/config/vespa.config.search.core.ranking-constants.def deleted file mode 100644 index 3b55eda3308..00000000000 --- a/searchcore/src/vespa/searchcore/config/vespa.config.search.core.ranking-constants.def +++ /dev/null @@ -1,6 +0,0 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -namespace=vespa.config.search.core - -constant[].name string -constant[].fileref file -constant[].type string diff --git a/searchlib/src/vespa/searchlib/config/searchlib.translogserver.def b/searchlib/src/vespa/searchlib/config/searchlib.translogserver.def deleted file mode 100644 index f822fc80fc1..00000000000 --- a/searchlib/src/vespa/searchlib/config/searchlib.translogserver.def +++ /dev/null @@ -1,38 +0,0 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -namespace=searchlib - -## Port number to use for listening. -listenport int default=13700 restart - -## Max file size (50M) -filesizemax int default=50000000 - -## Server name to identify server. -servername string default="tls" restart - -## Base directory. The default is not used as it is decided by the model. -basedir string default="tmp" restart - -## Use fsync after each commit. -## If not the below interval is used. -usefsync bool default=false restart - -##Number of threads available for visiting/subscription. -maxthreads int default=4 restart - -##Default crc method used -crcmethod enum {ccitt_crc32, xxh64} default=xxh64 - -## Control compression type. -compression.type enum {NONE, NONE_MULTI, LZ4, ZSTD} default=LZ4 - -## Control compression level -## LZ4 has normal range 1..9 while ZSTD has range 1..19 -## 9 is a reasonable default for both -compression.level int default=9 - -## How large a chunk can grow in memory before beeing flushed -chunk.sizelimit int default = 256000 # 256k - -## How long a chunk can reside in memory befor ebeeing flushed to disk. -chunk.agelimit double default = 0.010 # 10 milliseconds diff --git a/searchlib/src/vespa/searchlib/config/translogserver.def b/searchlib/src/vespa/searchlib/config/translogserver.def new file mode 100644 index 00000000000..f822fc80fc1 --- /dev/null +++ b/searchlib/src/vespa/searchlib/config/translogserver.def @@ -0,0 +1,38 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +namespace=searchlib + +## Port number to use for listening. +listenport int default=13700 restart + +## Max file size (50M) +filesizemax int default=50000000 + +## Server name to identify server. +servername string default="tls" restart + +## Base directory. The default is not used as it is decided by the model. +basedir string default="tmp" restart + +## Use fsync after each commit. +## If not the below interval is used. +usefsync bool default=false restart + +##Number of threads available for visiting/subscription. +maxthreads int default=4 restart + +##Default crc method used +crcmethod enum {ccitt_crc32, xxh64} default=xxh64 + +## Control compression type. +compression.type enum {NONE, NONE_MULTI, LZ4, ZSTD} default=LZ4 + +## Control compression level +## LZ4 has normal range 1..9 while ZSTD has range 1..19 +## 9 is a reasonable default for both +compression.level int default=9 + +## How large a chunk can grow in memory before beeing flushed +chunk.sizelimit int default = 256000 # 256k + +## How long a chunk can reside in memory befor ebeeing flushed to disk. +chunk.agelimit double default = 0.010 # 10 milliseconds diff --git a/searchsummary/src/vespa/searchsummary/config/juniperrc.def b/searchsummary/src/vespa/searchsummary/config/juniperrc.def new file mode 100644 index 00000000000..52f8104aa46 --- /dev/null +++ b/searchsummary/src/vespa/searchsummary/config/juniperrc.def @@ -0,0 +1,79 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +namespace=vespa.config.search.summary + +## Set the length (in #characters) of the dynamically generated +## summaries. This is a hint to the module that generates the +## dynamic summary - the actual size will depend on the available text +## and the query. You may find that you need to set this length +## substantially longer than your actual desired average length - +## setting this about twice as long is often appropriate. +length int default=256 + +## The number of (possibly partial) set of keywords +## matching the query, to attempt to include in the summary. The larger this +## value compared is set relative to the length parameter, the more +## dense the keywords may appear in the summary. +max_matches int default=3 + +## Minimal desired length of the generated summary in +## bytes. This is the shortest summary length for which the number of +## matches will be respected. Eg. if +## a summary appear to become shorter than min_length bytes with +## max_matches matches, then additional matches will be used if available. +min_length int default=128 + +## Make sure the prefix (length controlled by 'juniper.dynsum.length') +## of all fields with summary: dynamic are returned in the dynamic +## summary if a query does not hit in those fields +prefix bool default=true + +## The maximal number of bytes of context to prepend and append to +## each of the selected query keyword hits. This parameter defines the +## max size a summary would become if there are few keyword hits +## (max_matches set low or document contained few matches of the keywords). +surround_max int default=128 + +## The size of the sliding window used to determine if +## multiple query terms occur together. The larger the value, the more +## likely the system will find (and present in dynamic summary) complete +## matches containing all the search terms. The downside is a potential +## performance overhead of keeping candidates for matches longer during +## matching, and consequently updating more candidates that eventually +## gets thrown. +winsize int default=200 + +## This value multiplied with the winsize gives the size of a fallback +## window used to break out when searching for phrase term matches. +winsize_fallback_multiplier double default=10.0 + +## This value specifies the maximum number of match candidates that are +## managed for a non-leaf query node when matching the query against the +## input text. +max_match_candidates int default=1000 + +## The minimal number of bytes in a query keyword for +## it to be subject to the simple Juniper stemming algorithm. Keywords +## that are shorter than or equal to this limit will only yield exact +## matches in the dynamic summaries. +stem_min_length int default=5 + +## The maximal number of bytes that a word in the document +## can be longer than the keyword itself to yield a match. Eg. for +## the default values, if the keyword is 7 bytes long, it will match any +## word with length less than or equal to 10 for which the keyword is a prefix. +stem_max_extend int default=3 + + +## The parameters above may also be overriden on a per-field basis +## using the following array. +override[].fieldname string +override[].length int default=256 +override[].max_matches int default=3 +override[].min_length int default=128 +override[].prefix bool default=true +override[].surround_max int default=128 +override[].winsize int default=200 +override[].winsize_fallback_multiplier double default=10.0 +override[].max_match_candidates int default=1000 +override[].stem_min_length int default=5 +override[].stem_max_extend int default=3 diff --git a/searchsummary/src/vespa/searchsummary/config/vespa.config.search.summary.juniperrc.def b/searchsummary/src/vespa/searchsummary/config/vespa.config.search.summary.juniperrc.def deleted file mode 100644 index 52f8104aa46..00000000000 --- a/searchsummary/src/vespa/searchsummary/config/vespa.config.search.summary.juniperrc.def +++ /dev/null @@ -1,79 +0,0 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -namespace=vespa.config.search.summary - -## Set the length (in #characters) of the dynamically generated -## summaries. This is a hint to the module that generates the -## dynamic summary - the actual size will depend on the available text -## and the query. You may find that you need to set this length -## substantially longer than your actual desired average length - -## setting this about twice as long is often appropriate. -length int default=256 - -## The number of (possibly partial) set of keywords -## matching the query, to attempt to include in the summary. The larger this -## value compared is set relative to the length parameter, the more -## dense the keywords may appear in the summary. -max_matches int default=3 - -## Minimal desired length of the generated summary in -## bytes. This is the shortest summary length for which the number of -## matches will be respected. Eg. if -## a summary appear to become shorter than min_length bytes with -## max_matches matches, then additional matches will be used if available. -min_length int default=128 - -## Make sure the prefix (length controlled by 'juniper.dynsum.length') -## of all fields with summary: dynamic are returned in the dynamic -## summary if a query does not hit in those fields -prefix bool default=true - -## The maximal number of bytes of context to prepend and append to -## each of the selected query keyword hits. This parameter defines the -## max size a summary would become if there are few keyword hits -## (max_matches set low or document contained few matches of the keywords). -surround_max int default=128 - -## The size of the sliding window used to determine if -## multiple query terms occur together. The larger the value, the more -## likely the system will find (and present in dynamic summary) complete -## matches containing all the search terms. The downside is a potential -## performance overhead of keeping candidates for matches longer during -## matching, and consequently updating more candidates that eventually -## gets thrown. -winsize int default=200 - -## This value multiplied with the winsize gives the size of a fallback -## window used to break out when searching for phrase term matches. -winsize_fallback_multiplier double default=10.0 - -## This value specifies the maximum number of match candidates that are -## managed for a non-leaf query node when matching the query against the -## input text. -max_match_candidates int default=1000 - -## The minimal number of bytes in a query keyword for -## it to be subject to the simple Juniper stemming algorithm. Keywords -## that are shorter than or equal to this limit will only yield exact -## matches in the dynamic summaries. -stem_min_length int default=5 - -## The maximal number of bytes that a word in the document -## can be longer than the keyword itself to yield a match. Eg. for -## the default values, if the keyword is 7 bytes long, it will match any -## word with length less than or equal to 10 for which the keyword is a prefix. -stem_max_extend int default=3 - - -## The parameters above may also be overriden on a per-field basis -## using the following array. -override[].fieldname string -override[].length int default=256 -override[].max_matches int default=3 -override[].min_length int default=128 -override[].prefix bool default=true -override[].surround_max int default=128 -override[].winsize int default=200 -override[].winsize_fallback_multiplier double default=10.0 -override[].max_match_candidates int default=1000 -override[].stem_min_length int default=5 -override[].stem_max_extend int default=3 diff --git a/storage/src/vespa/storage/bucketdb/stor-bucket-init.def b/storage/src/vespa/storage/bucketdb/stor-bucket-init.def new file mode 100644 index 00000000000..3517afabd44 --- /dev/null +++ b/storage/src/vespa/storage/bucketdb/stor-bucket-init.def @@ -0,0 +1,35 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +namespace=vespa.config.content.core + +# The maximum number of alien files found during init that should be stored in +# memory so they can be viewed on status page. +max_alien_files_logged int default=10 restart + +# The maximum number of pending info reads to each disk during initialization. +max_pending_info_reads_per_disk int default=20 restart + +# The minimum number of pending info reads to each disk during initialization. +# When pending falls below this, we will resume database scan to add more +# pending up to the maximum setting. +min_pending_info_reads_per_disk int default=4 restart + +# The priority of the read bucket info requests the initializer sends to the +# persistence layer. Currently chosen so that such operations will not pre- +# empt any regular external load or ideal state operations, but they will block +# very low priority background operations such as periodic GC (default pri of +# 200). A tradeoff must be made between fast initialization and the availability +# of data on the initializing node. +info_read_priority int default=185 restart + +# The priority of the list bucket requests the initializer sends to the +# persistence layer. Should always be lower than the read priority to ensure +# starting to read wont make listing wait. However, listing is currently pretty +# much required to be done before starting anyhow, so this option does little +# unless your directories are not hardware independent. +list_priority int default=100 restart + +# Whether the initializer should complete listing before starting to read +# bucket information. Shouldnt matter much performance wise so always set to +# true as it is now. Setting it false, disks done listing first will start +# to process info requests a bit earlier than otherwise. +complete_list_before_starting_read bool default=false restart diff --git a/storage/src/vespa/storage/bucketdb/stor-bucketdb.def b/storage/src/vespa/storage/bucketdb/stor-bucketdb.def new file mode 100644 index 00000000000..470dd3afbf0 --- /dev/null +++ b/storage/src/vespa/storage/bucketdb/stor-bucketdb.def @@ -0,0 +1,9 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +namespace=vespa.config.content.core + +## Number of elements to retrieve in one bucket info chunk +bucketinfobatchsize int default=128 restart + +## Chunk level. Set what level of the path which defines one chunk. +## (See doxygen info in bucketmanager.h for more info) +chunklevel int default=1 restart diff --git a/storage/src/vespa/storage/bucketdb/vespa.config.content.core.stor-bucket-init.def b/storage/src/vespa/storage/bucketdb/vespa.config.content.core.stor-bucket-init.def deleted file mode 100644 index 3517afabd44..00000000000 --- a/storage/src/vespa/storage/bucketdb/vespa.config.content.core.stor-bucket-init.def +++ /dev/null @@ -1,35 +0,0 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -namespace=vespa.config.content.core - -# The maximum number of alien files found during init that should be stored in -# memory so they can be viewed on status page. -max_alien_files_logged int default=10 restart - -# The maximum number of pending info reads to each disk during initialization. -max_pending_info_reads_per_disk int default=20 restart - -# The minimum number of pending info reads to each disk during initialization. -# When pending falls below this, we will resume database scan to add more -# pending up to the maximum setting. -min_pending_info_reads_per_disk int default=4 restart - -# The priority of the read bucket info requests the initializer sends to the -# persistence layer. Currently chosen so that such operations will not pre- -# empt any regular external load or ideal state operations, but they will block -# very low priority background operations such as periodic GC (default pri of -# 200). A tradeoff must be made between fast initialization and the availability -# of data on the initializing node. -info_read_priority int default=185 restart - -# The priority of the list bucket requests the initializer sends to the -# persistence layer. Should always be lower than the read priority to ensure -# starting to read wont make listing wait. However, listing is currently pretty -# much required to be done before starting anyhow, so this option does little -# unless your directories are not hardware independent. -list_priority int default=100 restart - -# Whether the initializer should complete listing before starting to read -# bucket information. Shouldnt matter much performance wise so always set to -# true as it is now. Setting it false, disks done listing first will start -# to process info requests a bit earlier than otherwise. -complete_list_before_starting_read bool default=false restart diff --git a/storage/src/vespa/storage/bucketdb/vespa.config.content.core.stor-bucketdb.def b/storage/src/vespa/storage/bucketdb/vespa.config.content.core.stor-bucketdb.def deleted file mode 100644 index 470dd3afbf0..00000000000 --- a/storage/src/vespa/storage/bucketdb/vespa.config.content.core.stor-bucketdb.def +++ /dev/null @@ -1,9 +0,0 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -namespace=vespa.config.content.core - -## Number of elements to retrieve in one bucket info chunk -bucketinfobatchsize int default=128 restart - -## Chunk level. Set what level of the path which defines one chunk. -## (See doxygen info in bucketmanager.h for more info) -chunklevel int default=1 restart diff --git a/storage/src/vespa/storage/config/rpc-provider.def b/storage/src/vespa/storage/config/rpc-provider.def new file mode 100644 index 00000000000..f54b0e00fe4 --- /dev/null +++ b/storage/src/vespa/storage/config/rpc-provider.def @@ -0,0 +1,4 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +namespace=vespa.config.content.core + +connectspec string default="tcp/localhost:17777" restart diff --git a/storage/src/vespa/storage/config/stor-bouncer.def b/storage/src/vespa/storage/config/stor-bouncer.def new file mode 100644 index 00000000000..6af5ee078e9 --- /dev/null +++ b/storage/src/vespa/storage/config/stor-bouncer.def @@ -0,0 +1,30 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +namespace=vespa.config.content.core + +## Whether or not the bouncer should stop external load from +## entering node when the cluster state is down. +stop_external_load_when_cluster_down bool default=true + +## Sets what node states the node will allow incoming commands +## in. +stop_all_load_when_nodestate_not_in string default="uri" + +## Sets whether to just use (self) reported node state or to use wanted state +## if wanted state is worse than the current reported state. +use_wanted_state_if_possible bool default=true + +## The maximum clock skew allowed in the system. Any messages received +## that have a timestamp longer in the future than this will be failed. +max_clock_skew_seconds int default=5 + +## If this config value is != -1, the node will reject any external feed +## operations with a priority lower than that specified here. Note that since +## we map priorities in such a way that 0 is the _highest_ priority and 255 the +## _lowest_ priority, for two operations A and B, if B has a lower priority +## than A it will have a higher priority _integer_ value. +## +## Only mutating external feed operations will be blocked. Read-only operations +## and internal operations are always let through. +## +## Default is -1 (i.e. rejection is disabled and load is allowed through) +feed_rejection_priority_threshold int default=-1 diff --git a/storage/src/vespa/storage/config/stor-bucketmover.def b/storage/src/vespa/storage/config/stor-bucketmover.def new file mode 100644 index 00000000000..1fc200f83ca --- /dev/null +++ b/storage/src/vespa/storage/config/stor-bucketmover.def @@ -0,0 +1,37 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +namespace=vespa.config.content.core + +## Minimum time between bucket database iterations in the bucket mover. The +## minumum time is used when disks starts to get pretty full and we have plenty +## stuff we can move. +## restart flag was added automatically and needs to be verified. +minimum_recheck_interval_in_seconds int default=60 restart + +## Maximum time between bucket database iterations in the bucket mover. The +## maximum time is used when disks have plenty free space, so moving data is +## not critical. +## restart flag was added automatically and needs to be verified. +maximum_recheck_interval_in_seconds int default=3600 restart + +## Number of buckets to cache at a time when reading the bucket database +## restart flag was added automatically and needs to be verified. +bucket_iteration_chunk int default=1000 restart + +## Maximum fill rate above average fill rate for a target disk to be eligible +## as a target for a bucket move operation. +## restart flag was added automatically and needs to be verified. +max_target_fill_rate_above_average double default=0.01 restart + +## Number of bucket mover runs to keep in history vector +## restart flag was added automatically and needs to be verified. +max_history_size int default=10 restart + +## Max concurrent pending bucket move operations scheduled in total. +## restart flag was added automatically and needs to be verified. +max_pending int default=5 restart + +## Operation delay. If set, the bucket mover will wait for this amount of +## milliseconds between each operation. Useful in testing to make move run go +## slow enough to view without that much data. +## restart flag was added automatically and needs to be verified. +operation_delay int default=0 restart diff --git a/storage/src/vespa/storage/config/stor-communicationmanager.def b/storage/src/vespa/storage/config/stor-communicationmanager.def new file mode 100644 index 00000000000..3e4b1fd6515 --- /dev/null +++ b/storage/src/vespa/storage/config/stor-communicationmanager.def @@ -0,0 +1,57 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +namespace=vespa.config.content.core + +mbusport int default=-1 restart + +rpcport int default=6000 restart + +# Limits for max pending concurrent number of operations towards a node before +# MessageBus starts busy-bouncing messages. Distributor and content nodes are +# treated separately. 0 means no limit. +mbus_distributor_node_max_pending_count int default=5000 +mbus_content_node_max_pending_count int default=0 + +# Limits for max total amount of memory (in bytes) used by operations towards +# a node before MessageBus starts busy-bouncing messages. Distributor and +# content nodes are treated separately. 0 means no limit. +mbus_distributor_node_max_pending_size int default=0 +mbus_content_node_max_pending_size int default=0 + +# Minimum size of packets to compress (0 means no compression) +mbus.compress.limit int default=1024 + +## Compression level for packets +mbus.compress.level int default=3 + +## Compression type for packets. +mbus.compress.type enum {NONE, LZ4, ZSTD} default=LZ4 + +## TTL for rpc target cache +mbus.rpctargetcache.ttl double default = 600 + +## Number of threads for mbus threadpool +## Any value below 1 will be 1. +mbus.num_threads int default=4 + +mbus.optimize_for enum {LATENCY, THROUGHPUT, ADAPTIVE} default = LATENCY + +## Enable to use above thread pool for encoding replies +## False will use network(fnet) thread +mbus.dispatch_on_encode bool default=true + +## Enable to use above thread pool for decoding replies +## False will use network(fnet) thread +## Todo: Change default once verified in large scale deployment. +mbus.dispatch_on_decode bool default=false + +## Skip messenger thread on reply +## Experimental +mbus.skip_reply_thread bool default=false + +## Skip messenger thread on reply +## Experimental +mbus.skip_request_thread bool default=false + +## Skip communication manager thread on mbus requests +## Experimental +skip_thread bool default=false diff --git a/storage/src/vespa/storage/config/stor-distributormanager.def b/storage/src/vespa/storage/config/stor-distributormanager.def new file mode 100644 index 00000000000..db2bfb61376 --- /dev/null +++ b/storage/src/vespa/storage/config/stor-distributormanager.def @@ -0,0 +1,246 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +namespace=vespa.config.content.core + +## Maximum number of ideal state operations scheduled by a distributor. +maxpendingidealstateoperations int default=100 + +## The total size of unique documents in a bucket before we split it due to +## being too big. By default this is now 32 MB. +splitsize int default=33544432 + +## The maximum amount of entries in a file before we should attempt to split it. +## A meta data entry in a slotfile currently uses 40 bytes. It is probably +## good to have the split size, such that all meta data entries are normally +## read when you do the initial read. With the default of 1024, meta data will +## take up around 40 kB, and the default initial read is 64 kB, allowing the +## file to grow a bit above max and still all be read in initial read. +splitcount int default=1024 + +## The maximum size of unique documents that allows the system to reduce +## the number of split bits on the bucket, or join two buckets together. +## The size must be lower than this number, and the count must be lower than +## joincount. +joinsize int default=16000000 + +## The maximum number of unique documents that allows for joining (see +## joinsize). +joincount int default=512 + +## Minimum level of splitting for buckets +minsplitcount int default=16 + +## If non-empty, continuously delete all the documents matching this selection. +garbagecollection.selectiontoremove string default="" + +## The interval with which each bucket is purged using the selection above. +## If 0, no garbage collection is done. +garbagecollection.interval int default=0 + +## If false, dont do splits inline with feeding. +inlinebucketsplitting bool default=true + +## List of state checkers (ideal state generators) that should be ignored in the cluster. +## One or more of the following (case insensitive): +## +## SynchronizeAndMove +## DeleteExtraCopies +## JoinBuckets +## SplitBucket +## SplitInconsistentBuckets +## SetBucketState +## GarbageCollection +blockedstatecheckers[] string restart + +## Whether or not distributor should issue reverts when operations partially +## fail. +enable_revert bool default=true + +## Maximum nodes involved in a merge operation. Currently, this can not be more +## than 16 nodes due to protocol limitations. However, decreasing the max may +## be useful if 16 node merges ends up too expensive. +maximum_nodes_per_merge int default=16 + +## For internal in process debugging, it may be useful to not start the +## distributor thread to be able to call tick() manually and run single threaded +start_distributor_thread bool default=true restart + +## The number of ticks calls done before a wait is done. This can be +## set higher than 10 for the distributor to improve speed of bucket iterations +## while still keep CPU load low/moderate. +ticks_before_wait int default=10 + +## The sleep time between ticks if there are no more queued tasks. +ticks_wait_time_ms int default=1 + +## Max processing time used by deadlock detector. +max_process_time_ms int default=5000 + +## Allow overriding default priorities of certain maintenance operations. +## This is an advanced feature, do not touch this unless you have a very good +## reason to do so! Configuring these values wrongly may cause starvation of +## important operations, leading to unpredictable behavior and/or data loss. +## +## Merge used to move data to ideal location +priority_merge_move_to_ideal_node int default=165 + +## Merge for copies that have gotten out of sync with each other +priority_merge_out_of_sync_copies int default=120 + +## Merge for restoring redundancy of copies +priority_merge_too_few_copies int default=120 + +## Copy activation when there are no other active copies (likely causing +## lack of search coverage for that bucket) +priority_activate_no_existing_active int default=100 + +## Copy activation when there is already an active copy for the bucket. +priority_activate_with_existing_active int default=100 + +## Deletion of bucket copy. Cheap on VDS, not necessarily so on indexed search. +priority_delete_bucket_copy int default=100 + +## Joining caused by bucket siblings getting sufficiently small to fit into a +## single bucket. +priority_join_buckets int default=155 + +## Splitting caused by system increasing its minimum distribution bit count. +priority_split_distribution_bits int default=200 + +## Splitting due to bucket exceeding max document count or byte size (see +## splitcount and splitsize config values) +priority_split_large_bucket int default=175 + +## Splitting due to buckets being inconsistently split. Should be higher +## priority than the vast majority of external load. +priority_split_inconsistent_bucket int default=110 + +## Background garbage collection. Should be lower priority than external load +## and other ideal state operations (aside from perhaps minimum bit splitting). +priority_garbage_collection int default=200 + +## The distributor can send joins that "lift" a bucket without any siblings +## higher up in the bucket tree hierarchy. The assumption is that if this +## is done for all sibling-less buckets, they will all eventually reach a +## level in the tree where they do in fact have a sibling and may (if their +## sizes allow) be joined into a single bucket. +enable_join_for_sibling_less_buckets bool default=false + +## There exists a distribution edge case where bucket siblings end up having +## non-equal ideal locations. This will normally inhibit join operations, as +## these are only allowed when all nodes have all source buckets involved in +## the join operation. Setting this property to true means such buckets may +## still be joined at the cost of transient inconsistencies for the buckets +## being joined into. +enable_inconsistent_join bool default=false + +## The distributor host info reporter may be disabled entirely, in which case +## no per-node statistics for merges, latencies or bucket replication factors +## will be reported back to the cluster controller. Disabling this may make +## sense in large clusters that do not make use of these reports directly or +## indirectly, as it causes potentially significant processing overhead on the +## cluster controller. +## This host reporter must never be disabled on a Hosted Vespa system, or +## automatic upgrades will stall. +enable_host_info_reporting bool default=true + +## For each available node, the distributor will report back to the cluster +## controller a value which indicates the minimum replication factor for any +## bucket contained on said node. This config exposes a way to alter how this +## replication factor is computed. +## +## Valid enum values and their semantics: +## +## TRUSTED - only trusted replicas are counted. +## ANY - any replica present is counted. This may return an overly optimistic +## view of the system. E.g. if there are 3 replicas, 1 having 1000 docs +## and 2 having 1 doc, all being out of sync, counting with ANY will still +## treat this as a minimum replication factor of 3. Conversely, with +## TRUSTED such a bucket would most likely have a factor of 0 (or 1 iff +## the trusted status for the replica with 1000 docs is known). +minimum_replica_counting_mode enum { TRUSTED, ANY } default=TRUSTED + +## Bucket activation only makes sense for indexed search clusters, but Proton +## may also be run in store-only or streaming mode, in which case it does not +## actually require any activations. If the model infers that Proton is running +## in such a mode, activation will be explicitly disabled. +## +## Activation is always disabled entirely for clusters using VDS as their +## engine, regardless of the value of this setting. +disable_bucket_activation bool default=false + + +## Maximum clock skew across nodes in the cluster, in whole seconds. +## Used to prevent timestamp collisions during distributor bucket ownership +## transfers. +## Zero means this mechanism is disabled. +max_cluster_clock_skew_sec int default=1 + +## If set, a distributor will only allow one active operation per document ID +## for puts, updates and removes. This helps prevent issues caused by concurrent +## modifications to documents when sent from multiple feed clients. +sequence_mutating_operations bool default=true + +## Number of seconds that scheduling of new merge operations should be inhibited +## towards a node if it has indicated that its merge queues are full or it is +## suffering from resource exhaustion. +inhibit_merge_sending_on_busy_node_duration_sec int default=10 + +## If set, enables potentially stale reads during cluster state transitions where +## buckets change ownership. This also implicitly enables support for two-phase +## cluster state transitions on the distributor. +## For this option to take effect, the cluster controller must also have two-phase +## states enabled. +allow_stale_reads_during_cluster_state_transitions bool default=false + +## If greater than zero, injects a thread sleep into certain parts of the bucket +## processing logic. This allows for easier testing of racing edge cases where the +## main distributor thread is CPU-blocked processing large amounts of buckets, but +## without actually needing to use a lot of buckets in the test itself. +## Setting any of these values only makes sense for testing! +simulated_db_pruning_latency_msec int default=0 +simulated_db_merging_latency_msec int default=0 + +## Whether to use a B-tree data structure for the distributor bucket database instead +## of the legacy database. Setting this option may trigger alternate code paths for +## read only operations, as the B-tree database is thread safe for concurrent reads. +use_btree_database bool default=true restart + +## If a bucket is inconsistent and an Update operation is received, a two-phase +## write-repair path is triggered in which a Get is sent to all diverging replicas. +## Once received, the update is applied on the distributor and pushed out to the +## content nodes as Puts. +## Iff this config is set to true AND all Gets return the same timestamp from all +## content nodes, the two-phase update path reverts back to the regular fast path. +## Since all replicas of the document were in sync, applying the update in-place +## shall be considered safe. +restart_with_fast_update_path_if_all_get_timestamps_are_consistent bool default=false + +## If set, no merge operations may be generated for any reason by a distributor. +## This is ONLY intended for system testing of certain transient edge cases and +## MUST NOT be set to true in a production environment. +merge_operations_disabled bool default=false + +## If set, Get operations that are initiated by the client (i.e. _not_ Get operations +## that are initiated by the distributor) will be forwarded to the backend with +## a flag signalling that weak read consistency may be used. This allows the +## backend to minimize internal locking. The downside is that it's not guaranteed +## to observe the most recent writes to the document, nor to observe an atomically +## consistent view of fields across document versions. +## This is mostly useful in a system that is effectively read-only. +use_weak_internal_read_consistency_for_client_gets bool default=false + +## If true, adds an initial metadata-only fetch phase to updates that touch buckets +## with inconsistent replicas. Metadata timestamps are compared and a single full Get +## is sent _only_ to one node with the highest timestamp. Without a metadata phase, +## full gets would be sent to _all_ nodes. +## Setting this option to true always implicitly enables the fast update restart +## feature, so it's not required to set that config to true, nor will setting it +## to false actually disable the feature. +enable_metadata_only_fetch_phase_for_inconsistent_updates bool default=false + +## If a distributor main thread tick is constantly processing requests or responses +## originating from other nodes, setting this value above zero will prevent implicit +## maintenance scans from being done as part of the tick for up to N rounds of ticking. +## This is to reduce the amount of CPU spent on ideal state calculations and bucket DB +## accesses when the distributor is heavily loaded with feed operations. +max_consecutively_inhibited_maintenance_ticks int default=20 diff --git a/storage/src/vespa/storage/config/stor-integritychecker.def b/storage/src/vespa/storage/config/stor-integritychecker.def new file mode 100644 index 00000000000..657537ac015 --- /dev/null +++ b/storage/src/vespa/storage/config/stor-integritychecker.def @@ -0,0 +1,38 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +namespace=vespa.config.content.core + +## Minutes after midnight when integrity checker is allowed to start running. +## 0 means it will start/continue run at midnight. +dailycyclestart int default=0 + +## Minutes after midnight when integrity checker is not allowed to run anymore. +## If this equals dailycyclestart it is allowed to run all day. dailycyclestop +## is allowed to be less than dailycyclestart. +dailycyclestop int default=0 + +## Status of what is allowed done on what weekdays. Should be a string with +## seven characters, where the first represent sunday, the seventh saturday. +## The possible options are RrCc- which means: +## R - If state becomes R, and current cycle does not verify file content, +## abort current cycle, otherwise continue it. Start new cycle verifying +## all content of all files. +## r - Continue current cycle. Start new cycle using cheap partial file +## verification. +## c - Continue current cycle. Dont start a new cycle. +weeklycycle string default="Rrrrrrr" + +## Max concurrent pending bucket verifications. For max speed, each disk thread +## should have one to work with all the time. Default is 1, to ensure little +## resources are consumed by this process by default. Once request priority +## has been introduced, this default may become higher. +maxpending int default=2 + +## Minimum time since last cycle before starting a new one in minutes. +## Defaults to 24 hours. +mincycletime int default=1440 + +## Minimum time in seconds between each request. To throttle the system even +## slower if continuous one pending puts on more load on the system than you +## want. Works with multiple pending messages, though it doesnt make much sense +## unless maxpending equals 1. +requestdelay int default=0 diff --git a/storage/src/vespa/storage/config/stor-messageforwarder.def b/storage/src/vespa/storage/config/stor-messageforwarder.def new file mode 100644 index 00000000000..4a3f481e659 --- /dev/null +++ b/storage/src/vespa/storage/config/stor-messageforwarder.def @@ -0,0 +1,4 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +namespace=vespa.config.content.core + +route string default="" restart diff --git a/storage/src/vespa/storage/config/stor-opslogger.def b/storage/src/vespa/storage/config/stor-opslogger.def new file mode 100644 index 00000000000..3ba2c621427 --- /dev/null +++ b/storage/src/vespa/storage/config/stor-opslogger.def @@ -0,0 +1,4 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +namespace=vespa.config.content.core + +targetfile string default="" restart diff --git a/storage/src/vespa/storage/config/stor-prioritymapping.def b/storage/src/vespa/storage/config/stor-prioritymapping.def new file mode 100644 index 00000000000..decf4c68ee2 --- /dev/null +++ b/storage/src/vespa/storage/config/stor-prioritymapping.def @@ -0,0 +1,20 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +namespace=vespa.config.content.core + +highest int default=50 +very_high int default=60 +high_1 int default=70 +high_2 int default=80 +high_3 int default=90 +normal_1 int default=100 +normal_2 int default=110 +normal_3 int default=120 +normal_4 int default=130 +normal_5 int default=140 +normal_6 int default=150 +low_1 int default=160 +low_2 int default=170 +low_3 int default=180 +very_low int default=190 +lowest int default=200 + diff --git a/storage/src/vespa/storage/config/stor-server.def b/storage/src/vespa/storage/config/stor-server.def new file mode 100644 index 00000000000..e1446aa8ed1 --- /dev/null +++ b/storage/src/vespa/storage/config/stor-server.def @@ -0,0 +1,90 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +namespace=vespa.config.content.core + +## Root directory for all files related to this storage node. +## Will typically be "$VESPA_HOME/var/db/vespa/vds/// +root_folder string restart + +## VDS cluster +cluster_name string default="storage" restart + +## The index of this node. Each node of the same type in the same cluster need +## to have unique indexes. This should not be changed, as this is what we use +## to identify the node, and to decide what data should be on it. +node_index int default=0 restart + +## Set whether this is a distributor or a storage node. This will decide what +## storage links are set up. +is_distributor bool restart + +## Capacity of the node. How much data and load this node will get relative to +## other nodes. +node_capacity double default=1.0 restart + +## Capacity of the disks on this node. How much data and load will each disk +## get relative to the other disks on this node. +disk_capacity[] double restart + +## Reliability of this node. How much of the cluster redundancy factor can this +## node make up for. +node_reliability int default=1 restart + +## The upper bound of merges that any storage node can have active. +## A merge operation will be chained through all nodes involved in the +## merge, only actually starting the operation when every node has +## allowed it to pass through. +max_merges_per_node int default=16 +max_merge_queue_size int default=1024 + +## If the persistence provider indicates that it has exhausted one or more +## of its internal resources during a mutating operation, new merges will +## be bounced for this duration. Not allowing further merges helps take +## load off the node while it e.g. compacts its data stores or memory in +## the background. +## Note: this does not affect merges where the current node is marked as +## "source only", as merges do not cause mutations on such nodes. +resource_exhaustion_merge_back_pressure_duration_secs double default=30.0 + +## Whether the deadlock detector should be enabled or not. If disabled, it will +## still run, but it will never actually abort the process it is running in. +enable_dead_lock_detector bool default=false restart + +## Whether to enable deadlock detector warnings in log or not. If enabled, +## warnings will be written even if dead lock detecting is not enabled. +enable_dead_lock_detector_warnings bool default=true restart + +## Each thread registers how often it will at minimum register ticks (given that +## the system is not overloaded. If you are running Vespa on overloaded nodes, +## you can use this slack timeout to add to the thread timeouts in order to +## allow for more slack before dead lock detector kicks in. The value is in seconds. +dead_lock_detector_timeout_slack double default=240 restart + +## If set to 0, storage will attempt to auto-detect the number of VDS mount +## points to use. If set to a number, force this number. This number only makes +## sense on a storage node of course +disk_count int default=0 restart + +## Configure persistence provider. Temporary here to test. +persistence_provider.type enum {STORAGE, DUMMY, RPC } default=STORAGE restart +persistence_provider.rpc.connectspec string default="tcp/localhost:27777" restart + +## Whether or not to use the new metadata flow implementation. Default to not +## as it is currently in development and not even functional +switch_new_meta_data_flow bool default=false restart + +## When the content layer receives a set of changed buckets from the persistence +## layer, it must recheck all of these. Each such recheck results in an +## operation scheduled against the persistence queust and since the total +## number of buckets to recheck may reach hundreds of thousands in a large +## system, we send these in chunks to avoid saturating the queues with +## operations. +bucket_rechecking_chunk_size int default=100 + +## If greater than zero, simulates added latency caused by CPU processing during +## full bucket info requests. The latency is added per batch of operations processed. +## Only useful for testing! +simulated_bucket_request_latency_msec int default=0 + +## If set, content node processes will use a B-tree backed bucket database implementation +## instead of the legacy Judy-based implementation. +use_content_node_btree_bucket_db bool default=false restart diff --git a/storage/src/vespa/storage/config/stor-status.def b/storage/src/vespa/storage/config/stor-status.def new file mode 100644 index 00000000000..640a03299c2 --- /dev/null +++ b/storage/src/vespa/storage/config/stor-status.def @@ -0,0 +1,4 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +namespace=vespa.config.content.core + +httpport int default=0 restart diff --git a/storage/src/vespa/storage/config/stor-visitordispatcher.def b/storage/src/vespa/storage/config/stor-visitordispatcher.def new file mode 100644 index 00000000000..2e418c97989 --- /dev/null +++ b/storage/src/vespa/storage/config/stor-visitordispatcher.def @@ -0,0 +1,19 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +namespace=vespa.config.content.core + +# For any given client visitor operation, this specifies a maximum fan-out +# factor for concurrent content node visitor operations towards a particular +# content node. +# Having several visitor operations running concurrently increases the +# potential data processing parallelism on the content nodes at the expense +# of using additional resources. +maxvisitorspernodeperclientvisitor int default=16 + +# Minimum number of buckets that have to be present on a given content node +# before more than one visitor operation may be sent in parallel towards it. +# This config is directly related to maxvisitorspernodeperclientvisitor. +# Example: with max visitors of 4, min buckets of 5 and total of 40 buckets on +# a content node, a total of 4 visitors of 10 buckets each will be sent to the +# node. If min buckets were 20, only 2 visitors of 20 buckets each would be +# sent. +minbucketspervisitor int default=1 diff --git a/storage/src/vespa/storage/config/vespa.config.content.core.rpc-provider.def b/storage/src/vespa/storage/config/vespa.config.content.core.rpc-provider.def deleted file mode 100644 index f54b0e00fe4..00000000000 --- a/storage/src/vespa/storage/config/vespa.config.content.core.rpc-provider.def +++ /dev/null @@ -1,4 +0,0 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -namespace=vespa.config.content.core - -connectspec string default="tcp/localhost:17777" restart diff --git a/storage/src/vespa/storage/config/vespa.config.content.core.stor-bouncer.def b/storage/src/vespa/storage/config/vespa.config.content.core.stor-bouncer.def deleted file mode 100644 index 6af5ee078e9..00000000000 --- a/storage/src/vespa/storage/config/vespa.config.content.core.stor-bouncer.def +++ /dev/null @@ -1,30 +0,0 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -namespace=vespa.config.content.core - -## Whether or not the bouncer should stop external load from -## entering node when the cluster state is down. -stop_external_load_when_cluster_down bool default=true - -## Sets what node states the node will allow incoming commands -## in. -stop_all_load_when_nodestate_not_in string default="uri" - -## Sets whether to just use (self) reported node state or to use wanted state -## if wanted state is worse than the current reported state. -use_wanted_state_if_possible bool default=true - -## The maximum clock skew allowed in the system. Any messages received -## that have a timestamp longer in the future than this will be failed. -max_clock_skew_seconds int default=5 - -## If this config value is != -1, the node will reject any external feed -## operations with a priority lower than that specified here. Note that since -## we map priorities in such a way that 0 is the _highest_ priority and 255 the -## _lowest_ priority, for two operations A and B, if B has a lower priority -## than A it will have a higher priority _integer_ value. -## -## Only mutating external feed operations will be blocked. Read-only operations -## and internal operations are always let through. -## -## Default is -1 (i.e. rejection is disabled and load is allowed through) -feed_rejection_priority_threshold int default=-1 diff --git a/storage/src/vespa/storage/config/vespa.config.content.core.stor-bucketmover.def b/storage/src/vespa/storage/config/vespa.config.content.core.stor-bucketmover.def deleted file mode 100644 index 1fc200f83ca..00000000000 --- a/storage/src/vespa/storage/config/vespa.config.content.core.stor-bucketmover.def +++ /dev/null @@ -1,37 +0,0 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -namespace=vespa.config.content.core - -## Minimum time between bucket database iterations in the bucket mover. The -## minumum time is used when disks starts to get pretty full and we have plenty -## stuff we can move. -## restart flag was added automatically and needs to be verified. -minimum_recheck_interval_in_seconds int default=60 restart - -## Maximum time between bucket database iterations in the bucket mover. The -## maximum time is used when disks have plenty free space, so moving data is -## not critical. -## restart flag was added automatically and needs to be verified. -maximum_recheck_interval_in_seconds int default=3600 restart - -## Number of buckets to cache at a time when reading the bucket database -## restart flag was added automatically and needs to be verified. -bucket_iteration_chunk int default=1000 restart - -## Maximum fill rate above average fill rate for a target disk to be eligible -## as a target for a bucket move operation. -## restart flag was added automatically and needs to be verified. -max_target_fill_rate_above_average double default=0.01 restart - -## Number of bucket mover runs to keep in history vector -## restart flag was added automatically and needs to be verified. -max_history_size int default=10 restart - -## Max concurrent pending bucket move operations scheduled in total. -## restart flag was added automatically and needs to be verified. -max_pending int default=5 restart - -## Operation delay. If set, the bucket mover will wait for this amount of -## milliseconds between each operation. Useful in testing to make move run go -## slow enough to view without that much data. -## restart flag was added automatically and needs to be verified. -operation_delay int default=0 restart diff --git a/storage/src/vespa/storage/config/vespa.config.content.core.stor-communicationmanager.def b/storage/src/vespa/storage/config/vespa.config.content.core.stor-communicationmanager.def deleted file mode 100644 index 3e4b1fd6515..00000000000 --- a/storage/src/vespa/storage/config/vespa.config.content.core.stor-communicationmanager.def +++ /dev/null @@ -1,57 +0,0 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -namespace=vespa.config.content.core - -mbusport int default=-1 restart - -rpcport int default=6000 restart - -# Limits for max pending concurrent number of operations towards a node before -# MessageBus starts busy-bouncing messages. Distributor and content nodes are -# treated separately. 0 means no limit. -mbus_distributor_node_max_pending_count int default=5000 -mbus_content_node_max_pending_count int default=0 - -# Limits for max total amount of memory (in bytes) used by operations towards -# a node before MessageBus starts busy-bouncing messages. Distributor and -# content nodes are treated separately. 0 means no limit. -mbus_distributor_node_max_pending_size int default=0 -mbus_content_node_max_pending_size int default=0 - -# Minimum size of packets to compress (0 means no compression) -mbus.compress.limit int default=1024 - -## Compression level for packets -mbus.compress.level int default=3 - -## Compression type for packets. -mbus.compress.type enum {NONE, LZ4, ZSTD} default=LZ4 - -## TTL for rpc target cache -mbus.rpctargetcache.ttl double default = 600 - -## Number of threads for mbus threadpool -## Any value below 1 will be 1. -mbus.num_threads int default=4 - -mbus.optimize_for enum {LATENCY, THROUGHPUT, ADAPTIVE} default = LATENCY - -## Enable to use above thread pool for encoding replies -## False will use network(fnet) thread -mbus.dispatch_on_encode bool default=true - -## Enable to use above thread pool for decoding replies -## False will use network(fnet) thread -## Todo: Change default once verified in large scale deployment. -mbus.dispatch_on_decode bool default=false - -## Skip messenger thread on reply -## Experimental -mbus.skip_reply_thread bool default=false - -## Skip messenger thread on reply -## Experimental -mbus.skip_request_thread bool default=false - -## Skip communication manager thread on mbus requests -## Experimental -skip_thread bool default=false diff --git a/storage/src/vespa/storage/config/vespa.config.content.core.stor-distributormanager.def b/storage/src/vespa/storage/config/vespa.config.content.core.stor-distributormanager.def deleted file mode 100644 index db2bfb61376..00000000000 --- a/storage/src/vespa/storage/config/vespa.config.content.core.stor-distributormanager.def +++ /dev/null @@ -1,246 +0,0 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -namespace=vespa.config.content.core - -## Maximum number of ideal state operations scheduled by a distributor. -maxpendingidealstateoperations int default=100 - -## The total size of unique documents in a bucket before we split it due to -## being too big. By default this is now 32 MB. -splitsize int default=33544432 - -## The maximum amount of entries in a file before we should attempt to split it. -## A meta data entry in a slotfile currently uses 40 bytes. It is probably -## good to have the split size, such that all meta data entries are normally -## read when you do the initial read. With the default of 1024, meta data will -## take up around 40 kB, and the default initial read is 64 kB, allowing the -## file to grow a bit above max and still all be read in initial read. -splitcount int default=1024 - -## The maximum size of unique documents that allows the system to reduce -## the number of split bits on the bucket, or join two buckets together. -## The size must be lower than this number, and the count must be lower than -## joincount. -joinsize int default=16000000 - -## The maximum number of unique documents that allows for joining (see -## joinsize). -joincount int default=512 - -## Minimum level of splitting for buckets -minsplitcount int default=16 - -## If non-empty, continuously delete all the documents matching this selection. -garbagecollection.selectiontoremove string default="" - -## The interval with which each bucket is purged using the selection above. -## If 0, no garbage collection is done. -garbagecollection.interval int default=0 - -## If false, dont do splits inline with feeding. -inlinebucketsplitting bool default=true - -## List of state checkers (ideal state generators) that should be ignored in the cluster. -## One or more of the following (case insensitive): -## -## SynchronizeAndMove -## DeleteExtraCopies -## JoinBuckets -## SplitBucket -## SplitInconsistentBuckets -## SetBucketState -## GarbageCollection -blockedstatecheckers[] string restart - -## Whether or not distributor should issue reverts when operations partially -## fail. -enable_revert bool default=true - -## Maximum nodes involved in a merge operation. Currently, this can not be more -## than 16 nodes due to protocol limitations. However, decreasing the max may -## be useful if 16 node merges ends up too expensive. -maximum_nodes_per_merge int default=16 - -## For internal in process debugging, it may be useful to not start the -## distributor thread to be able to call tick() manually and run single threaded -start_distributor_thread bool default=true restart - -## The number of ticks calls done before a wait is done. This can be -## set higher than 10 for the distributor to improve speed of bucket iterations -## while still keep CPU load low/moderate. -ticks_before_wait int default=10 - -## The sleep time between ticks if there are no more queued tasks. -ticks_wait_time_ms int default=1 - -## Max processing time used by deadlock detector. -max_process_time_ms int default=5000 - -## Allow overriding default priorities of certain maintenance operations. -## This is an advanced feature, do not touch this unless you have a very good -## reason to do so! Configuring these values wrongly may cause starvation of -## important operations, leading to unpredictable behavior and/or data loss. -## -## Merge used to move data to ideal location -priority_merge_move_to_ideal_node int default=165 - -## Merge for copies that have gotten out of sync with each other -priority_merge_out_of_sync_copies int default=120 - -## Merge for restoring redundancy of copies -priority_merge_too_few_copies int default=120 - -## Copy activation when there are no other active copies (likely causing -## lack of search coverage for that bucket) -priority_activate_no_existing_active int default=100 - -## Copy activation when there is already an active copy for the bucket. -priority_activate_with_existing_active int default=100 - -## Deletion of bucket copy. Cheap on VDS, not necessarily so on indexed search. -priority_delete_bucket_copy int default=100 - -## Joining caused by bucket siblings getting sufficiently small to fit into a -## single bucket. -priority_join_buckets int default=155 - -## Splitting caused by system increasing its minimum distribution bit count. -priority_split_distribution_bits int default=200 - -## Splitting due to bucket exceeding max document count or byte size (see -## splitcount and splitsize config values) -priority_split_large_bucket int default=175 - -## Splitting due to buckets being inconsistently split. Should be higher -## priority than the vast majority of external load. -priority_split_inconsistent_bucket int default=110 - -## Background garbage collection. Should be lower priority than external load -## and other ideal state operations (aside from perhaps minimum bit splitting). -priority_garbage_collection int default=200 - -## The distributor can send joins that "lift" a bucket without any siblings -## higher up in the bucket tree hierarchy. The assumption is that if this -## is done for all sibling-less buckets, they will all eventually reach a -## level in the tree where they do in fact have a sibling and may (if their -## sizes allow) be joined into a single bucket. -enable_join_for_sibling_less_buckets bool default=false - -## There exists a distribution edge case where bucket siblings end up having -## non-equal ideal locations. This will normally inhibit join operations, as -## these are only allowed when all nodes have all source buckets involved in -## the join operation. Setting this property to true means such buckets may -## still be joined at the cost of transient inconsistencies for the buckets -## being joined into. -enable_inconsistent_join bool default=false - -## The distributor host info reporter may be disabled entirely, in which case -## no per-node statistics for merges, latencies or bucket replication factors -## will be reported back to the cluster controller. Disabling this may make -## sense in large clusters that do not make use of these reports directly or -## indirectly, as it causes potentially significant processing overhead on the -## cluster controller. -## This host reporter must never be disabled on a Hosted Vespa system, or -## automatic upgrades will stall. -enable_host_info_reporting bool default=true - -## For each available node, the distributor will report back to the cluster -## controller a value which indicates the minimum replication factor for any -## bucket contained on said node. This config exposes a way to alter how this -## replication factor is computed. -## -## Valid enum values and their semantics: -## -## TRUSTED - only trusted replicas are counted. -## ANY - any replica present is counted. This may return an overly optimistic -## view of the system. E.g. if there are 3 replicas, 1 having 1000 docs -## and 2 having 1 doc, all being out of sync, counting with ANY will still -## treat this as a minimum replication factor of 3. Conversely, with -## TRUSTED such a bucket would most likely have a factor of 0 (or 1 iff -## the trusted status for the replica with 1000 docs is known). -minimum_replica_counting_mode enum { TRUSTED, ANY } default=TRUSTED - -## Bucket activation only makes sense for indexed search clusters, but Proton -## may also be run in store-only or streaming mode, in which case it does not -## actually require any activations. If the model infers that Proton is running -## in such a mode, activation will be explicitly disabled. -## -## Activation is always disabled entirely for clusters using VDS as their -## engine, regardless of the value of this setting. -disable_bucket_activation bool default=false - - -## Maximum clock skew across nodes in the cluster, in whole seconds. -## Used to prevent timestamp collisions during distributor bucket ownership -## transfers. -## Zero means this mechanism is disabled. -max_cluster_clock_skew_sec int default=1 - -## If set, a distributor will only allow one active operation per document ID -## for puts, updates and removes. This helps prevent issues caused by concurrent -## modifications to documents when sent from multiple feed clients. -sequence_mutating_operations bool default=true - -## Number of seconds that scheduling of new merge operations should be inhibited -## towards a node if it has indicated that its merge queues are full or it is -## suffering from resource exhaustion. -inhibit_merge_sending_on_busy_node_duration_sec int default=10 - -## If set, enables potentially stale reads during cluster state transitions where -## buckets change ownership. This also implicitly enables support for two-phase -## cluster state transitions on the distributor. -## For this option to take effect, the cluster controller must also have two-phase -## states enabled. -allow_stale_reads_during_cluster_state_transitions bool default=false - -## If greater than zero, injects a thread sleep into certain parts of the bucket -## processing logic. This allows for easier testing of racing edge cases where the -## main distributor thread is CPU-blocked processing large amounts of buckets, but -## without actually needing to use a lot of buckets in the test itself. -## Setting any of these values only makes sense for testing! -simulated_db_pruning_latency_msec int default=0 -simulated_db_merging_latency_msec int default=0 - -## Whether to use a B-tree data structure for the distributor bucket database instead -## of the legacy database. Setting this option may trigger alternate code paths for -## read only operations, as the B-tree database is thread safe for concurrent reads. -use_btree_database bool default=true restart - -## If a bucket is inconsistent and an Update operation is received, a two-phase -## write-repair path is triggered in which a Get is sent to all diverging replicas. -## Once received, the update is applied on the distributor and pushed out to the -## content nodes as Puts. -## Iff this config is set to true AND all Gets return the same timestamp from all -## content nodes, the two-phase update path reverts back to the regular fast path. -## Since all replicas of the document were in sync, applying the update in-place -## shall be considered safe. -restart_with_fast_update_path_if_all_get_timestamps_are_consistent bool default=false - -## If set, no merge operations may be generated for any reason by a distributor. -## This is ONLY intended for system testing of certain transient edge cases and -## MUST NOT be set to true in a production environment. -merge_operations_disabled bool default=false - -## If set, Get operations that are initiated by the client (i.e. _not_ Get operations -## that are initiated by the distributor) will be forwarded to the backend with -## a flag signalling that weak read consistency may be used. This allows the -## backend to minimize internal locking. The downside is that it's not guaranteed -## to observe the most recent writes to the document, nor to observe an atomically -## consistent view of fields across document versions. -## This is mostly useful in a system that is effectively read-only. -use_weak_internal_read_consistency_for_client_gets bool default=false - -## If true, adds an initial metadata-only fetch phase to updates that touch buckets -## with inconsistent replicas. Metadata timestamps are compared and a single full Get -## is sent _only_ to one node with the highest timestamp. Without a metadata phase, -## full gets would be sent to _all_ nodes. -## Setting this option to true always implicitly enables the fast update restart -## feature, so it's not required to set that config to true, nor will setting it -## to false actually disable the feature. -enable_metadata_only_fetch_phase_for_inconsistent_updates bool default=false - -## If a distributor main thread tick is constantly processing requests or responses -## originating from other nodes, setting this value above zero will prevent implicit -## maintenance scans from being done as part of the tick for up to N rounds of ticking. -## This is to reduce the amount of CPU spent on ideal state calculations and bucket DB -## accesses when the distributor is heavily loaded with feed operations. -max_consecutively_inhibited_maintenance_ticks int default=20 diff --git a/storage/src/vespa/storage/config/vespa.config.content.core.stor-integritychecker.def b/storage/src/vespa/storage/config/vespa.config.content.core.stor-integritychecker.def deleted file mode 100644 index 657537ac015..00000000000 --- a/storage/src/vespa/storage/config/vespa.config.content.core.stor-integritychecker.def +++ /dev/null @@ -1,38 +0,0 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -namespace=vespa.config.content.core - -## Minutes after midnight when integrity checker is allowed to start running. -## 0 means it will start/continue run at midnight. -dailycyclestart int default=0 - -## Minutes after midnight when integrity checker is not allowed to run anymore. -## If this equals dailycyclestart it is allowed to run all day. dailycyclestop -## is allowed to be less than dailycyclestart. -dailycyclestop int default=0 - -## Status of what is allowed done on what weekdays. Should be a string with -## seven characters, where the first represent sunday, the seventh saturday. -## The possible options are RrCc- which means: -## R - If state becomes R, and current cycle does not verify file content, -## abort current cycle, otherwise continue it. Start new cycle verifying -## all content of all files. -## r - Continue current cycle. Start new cycle using cheap partial file -## verification. -## c - Continue current cycle. Dont start a new cycle. -weeklycycle string default="Rrrrrrr" - -## Max concurrent pending bucket verifications. For max speed, each disk thread -## should have one to work with all the time. Default is 1, to ensure little -## resources are consumed by this process by default. Once request priority -## has been introduced, this default may become higher. -maxpending int default=2 - -## Minimum time since last cycle before starting a new one in minutes. -## Defaults to 24 hours. -mincycletime int default=1440 - -## Minimum time in seconds between each request. To throttle the system even -## slower if continuous one pending puts on more load on the system than you -## want. Works with multiple pending messages, though it doesnt make much sense -## unless maxpending equals 1. -requestdelay int default=0 diff --git a/storage/src/vespa/storage/config/vespa.config.content.core.stor-messageforwarder.def b/storage/src/vespa/storage/config/vespa.config.content.core.stor-messageforwarder.def deleted file mode 100644 index 4a3f481e659..00000000000 --- a/storage/src/vespa/storage/config/vespa.config.content.core.stor-messageforwarder.def +++ /dev/null @@ -1,4 +0,0 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -namespace=vespa.config.content.core - -route string default="" restart diff --git a/storage/src/vespa/storage/config/vespa.config.content.core.stor-opslogger.def b/storage/src/vespa/storage/config/vespa.config.content.core.stor-opslogger.def deleted file mode 100644 index 3ba2c621427..00000000000 --- a/storage/src/vespa/storage/config/vespa.config.content.core.stor-opslogger.def +++ /dev/null @@ -1,4 +0,0 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -namespace=vespa.config.content.core - -targetfile string default="" restart diff --git a/storage/src/vespa/storage/config/vespa.config.content.core.stor-prioritymapping.def b/storage/src/vespa/storage/config/vespa.config.content.core.stor-prioritymapping.def deleted file mode 100644 index decf4c68ee2..00000000000 --- a/storage/src/vespa/storage/config/vespa.config.content.core.stor-prioritymapping.def +++ /dev/null @@ -1,20 +0,0 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -namespace=vespa.config.content.core - -highest int default=50 -very_high int default=60 -high_1 int default=70 -high_2 int default=80 -high_3 int default=90 -normal_1 int default=100 -normal_2 int default=110 -normal_3 int default=120 -normal_4 int default=130 -normal_5 int default=140 -normal_6 int default=150 -low_1 int default=160 -low_2 int default=170 -low_3 int default=180 -very_low int default=190 -lowest int default=200 - diff --git a/storage/src/vespa/storage/config/vespa.config.content.core.stor-server.def b/storage/src/vespa/storage/config/vespa.config.content.core.stor-server.def deleted file mode 100644 index e1446aa8ed1..00000000000 --- a/storage/src/vespa/storage/config/vespa.config.content.core.stor-server.def +++ /dev/null @@ -1,90 +0,0 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -namespace=vespa.config.content.core - -## Root directory for all files related to this storage node. -## Will typically be "$VESPA_HOME/var/db/vespa/vds/// -root_folder string restart - -## VDS cluster -cluster_name string default="storage" restart - -## The index of this node. Each node of the same type in the same cluster need -## to have unique indexes. This should not be changed, as this is what we use -## to identify the node, and to decide what data should be on it. -node_index int default=0 restart - -## Set whether this is a distributor or a storage node. This will decide what -## storage links are set up. -is_distributor bool restart - -## Capacity of the node. How much data and load this node will get relative to -## other nodes. -node_capacity double default=1.0 restart - -## Capacity of the disks on this node. How much data and load will each disk -## get relative to the other disks on this node. -disk_capacity[] double restart - -## Reliability of this node. How much of the cluster redundancy factor can this -## node make up for. -node_reliability int default=1 restart - -## The upper bound of merges that any storage node can have active. -## A merge operation will be chained through all nodes involved in the -## merge, only actually starting the operation when every node has -## allowed it to pass through. -max_merges_per_node int default=16 -max_merge_queue_size int default=1024 - -## If the persistence provider indicates that it has exhausted one or more -## of its internal resources during a mutating operation, new merges will -## be bounced for this duration. Not allowing further merges helps take -## load off the node while it e.g. compacts its data stores or memory in -## the background. -## Note: this does not affect merges where the current node is marked as -## "source only", as merges do not cause mutations on such nodes. -resource_exhaustion_merge_back_pressure_duration_secs double default=30.0 - -## Whether the deadlock detector should be enabled or not. If disabled, it will -## still run, but it will never actually abort the process it is running in. -enable_dead_lock_detector bool default=false restart - -## Whether to enable deadlock detector warnings in log or not. If enabled, -## warnings will be written even if dead lock detecting is not enabled. -enable_dead_lock_detector_warnings bool default=true restart - -## Each thread registers how often it will at minimum register ticks (given that -## the system is not overloaded. If you are running Vespa on overloaded nodes, -## you can use this slack timeout to add to the thread timeouts in order to -## allow for more slack before dead lock detector kicks in. The value is in seconds. -dead_lock_detector_timeout_slack double default=240 restart - -## If set to 0, storage will attempt to auto-detect the number of VDS mount -## points to use. If set to a number, force this number. This number only makes -## sense on a storage node of course -disk_count int default=0 restart - -## Configure persistence provider. Temporary here to test. -persistence_provider.type enum {STORAGE, DUMMY, RPC } default=STORAGE restart -persistence_provider.rpc.connectspec string default="tcp/localhost:27777" restart - -## Whether or not to use the new metadata flow implementation. Default to not -## as it is currently in development and not even functional -switch_new_meta_data_flow bool default=false restart - -## When the content layer receives a set of changed buckets from the persistence -## layer, it must recheck all of these. Each such recheck results in an -## operation scheduled against the persistence queust and since the total -## number of buckets to recheck may reach hundreds of thousands in a large -## system, we send these in chunks to avoid saturating the queues with -## operations. -bucket_rechecking_chunk_size int default=100 - -## If greater than zero, simulates added latency caused by CPU processing during -## full bucket info requests. The latency is added per batch of operations processed. -## Only useful for testing! -simulated_bucket_request_latency_msec int default=0 - -## If set, content node processes will use a B-tree backed bucket database implementation -## instead of the legacy Judy-based implementation. -use_content_node_btree_bucket_db bool default=false restart diff --git a/storage/src/vespa/storage/config/vespa.config.content.core.stor-status.def b/storage/src/vespa/storage/config/vespa.config.content.core.stor-status.def deleted file mode 100644 index 640a03299c2..00000000000 --- a/storage/src/vespa/storage/config/vespa.config.content.core.stor-status.def +++ /dev/null @@ -1,4 +0,0 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -namespace=vespa.config.content.core - -httpport int default=0 restart diff --git a/storage/src/vespa/storage/config/vespa.config.content.core.stor-visitordispatcher.def b/storage/src/vespa/storage/config/vespa.config.content.core.stor-visitordispatcher.def deleted file mode 100644 index 2e418c97989..00000000000 --- a/storage/src/vespa/storage/config/vespa.config.content.core.stor-visitordispatcher.def +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -namespace=vespa.config.content.core - -# For any given client visitor operation, this specifies a maximum fan-out -# factor for concurrent content node visitor operations towards a particular -# content node. -# Having several visitor operations running concurrently increases the -# potential data processing parallelism on the content nodes at the expense -# of using additional resources. -maxvisitorspernodeperclientvisitor int default=16 - -# Minimum number of buckets that have to be present on a given content node -# before more than one visitor operation may be sent in parallel towards it. -# This config is directly related to maxvisitorspernodeperclientvisitor. -# Example: with max visitors of 4, min buckets of 5 and total of 40 buckets on -# a content node, a total of 4 visitors of 10 buckets each will be sent to the -# node. If min buckets were 20, only 2 visitors of 20 buckets each would be -# sent. -minbucketspervisitor int default=1 diff --git a/storage/src/vespa/storage/visiting/stor-visitor.def b/storage/src/vespa/storage/visiting/stor-visitor.def new file mode 100644 index 00000000000..72b3699fe2d --- /dev/null +++ b/storage/src/vespa/storage/visiting/stor-visitor.def @@ -0,0 +1,63 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +namespace=vespa.config.content.core + +## Number of separate threads that runs visitors. +visitorthreads int default=16 restart + +## Default timeout of visitors that loses contact with client (in seconds) +disconnectedvisitortimeout int default=0 restart + +## Time period (in seconds) in which to ignore requests to visitors that doesnt +## exist anymore. (Normal for visitors to get some messages right after +## aborting, logging them as faults instead after this timeout has passed.) +ignorenonexistingvisitortimelimit int default=300 restart + +## The number of buckets that are visited in parallel in a visitor visiting +## multiple buckets. Default is 8, meaning if you send a create visitor to visit +## 100 buckets, 8 of them will be visited in parallel. +defaultparalleliterators int default=8 + +## Default number of maximum client replies pending. +defaultpendingmessages int default=32 + +## Default size of docblocks used to transfer visitor data. +defaultdocblocksize int default=4190208 + +## Default docblock timeout in ms used to transfer visitor data. +## Currently defaults to a day. This is to avoid slow visitor target problems, +## getting data resent faster than it can process, and since there are very few +## reasons to actually time out +defaultdocblocktimeout int default=180000 + +## Default timeout of visitor info messages: Progress and error reports. +## If these time out, the visitor will be aborted on the storage node. +defaultinfotimeout int default=60000 + +## Max concurrent visitors (legacy) +maxconcurrentvisitors int default=64 + +## Priority-based max concurrent visitors. Fixed is the total number of +## concurrent visitors that can run for any priorities. Variable +## increases the concurrency limit for higher priorities, the limit +## being linear with a messages priority. Example: if Fixed is 16 +## and Variable is 64, maxconcurrent for a pri 255 message is 16 and +## maxconcurrent for a pri 0 message is 16+64=80. +## If fixed is left as 0, it will take the value of maxconcurrentvisitors, +## allowing backwards compatability +maxconcurrentvisitors_fixed int default=16 +maxconcurrentvisitors_variable int default=64 + +## Max size of visitor priority queue +maxvisitorqueuesize int default=1024 + +# Limit of memory used _per visitor_ in bytes. +# Due to optimistic parallelization, it is possible for this limit to be +# initially violated when the visitor is first started. This can happen since +# the visitor does not know the size of the bucket contents before fetching +# any data from it and it will do so based on parallelization factors specified +# in the CreateVisitor command. If 3 buckets are initially visited in parallel +# and these both contain a single 100 MiB document, the memory usage of the +# visitor will peak at 300 MiB even if the configured limit is e.g. 20 MiB. +# Default value is set to 20 MiB, which attempts to keep a reasonably safe +# level in the face of a default number of max concurrent visitors (64). +visitor_memory_usage_limit int default=25165824 diff --git a/storage/src/vespa/storage/visiting/vespa.config.content.core.stor-visitor.def b/storage/src/vespa/storage/visiting/vespa.config.content.core.stor-visitor.def deleted file mode 100644 index 72b3699fe2d..00000000000 --- a/storage/src/vespa/storage/visiting/vespa.config.content.core.stor-visitor.def +++ /dev/null @@ -1,63 +0,0 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -namespace=vespa.config.content.core - -## Number of separate threads that runs visitors. -visitorthreads int default=16 restart - -## Default timeout of visitors that loses contact with client (in seconds) -disconnectedvisitortimeout int default=0 restart - -## Time period (in seconds) in which to ignore requests to visitors that doesnt -## exist anymore. (Normal for visitors to get some messages right after -## aborting, logging them as faults instead after this timeout has passed.) -ignorenonexistingvisitortimelimit int default=300 restart - -## The number of buckets that are visited in parallel in a visitor visiting -## multiple buckets. Default is 8, meaning if you send a create visitor to visit -## 100 buckets, 8 of them will be visited in parallel. -defaultparalleliterators int default=8 - -## Default number of maximum client replies pending. -defaultpendingmessages int default=32 - -## Default size of docblocks used to transfer visitor data. -defaultdocblocksize int default=4190208 - -## Default docblock timeout in ms used to transfer visitor data. -## Currently defaults to a day. This is to avoid slow visitor target problems, -## getting data resent faster than it can process, and since there are very few -## reasons to actually time out -defaultdocblocktimeout int default=180000 - -## Default timeout of visitor info messages: Progress and error reports. -## If these time out, the visitor will be aborted on the storage node. -defaultinfotimeout int default=60000 - -## Max concurrent visitors (legacy) -maxconcurrentvisitors int default=64 - -## Priority-based max concurrent visitors. Fixed is the total number of -## concurrent visitors that can run for any priorities. Variable -## increases the concurrency limit for higher priorities, the limit -## being linear with a messages priority. Example: if Fixed is 16 -## and Variable is 64, maxconcurrent for a pri 255 message is 16 and -## maxconcurrent for a pri 0 message is 16+64=80. -## If fixed is left as 0, it will take the value of maxconcurrentvisitors, -## allowing backwards compatability -maxconcurrentvisitors_fixed int default=16 -maxconcurrentvisitors_variable int default=64 - -## Max size of visitor priority queue -maxvisitorqueuesize int default=1024 - -# Limit of memory used _per visitor_ in bytes. -# Due to optimistic parallelization, it is possible for this limit to be -# initially violated when the visitor is first started. This can happen since -# the visitor does not know the size of the bucket contents before fetching -# any data from it and it will do so based on parallelization factors specified -# in the CreateVisitor command. If 3 buckets are initially visited in parallel -# and these both contain a single 100 MiB document, the memory usage of the -# visitor will peak at 300 MiB even if the configured limit is e.g. 20 MiB. -# Default value is set to 20 MiB, which attempts to keep a reasonably safe -# level in the face of a default number of max concurrent visitors (64). -visitor_memory_usage_limit int default=25165824 diff --git a/vsm/src/vespa/vsm/config/vespa.config.search.vsm.vsm.def b/vsm/src/vespa/vsm/config/vespa.config.search.vsm.vsm.def deleted file mode 100644 index c446efe5588..00000000000 --- a/vsm/src/vespa/vsm/config/vespa.config.search.vsm.vsm.def +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -namespace=vespa.config.search.vsm - -## The document model for the documents used as input for the VSM -doctype reference - -## Configuration for storage client used by VSM -storagecfg reference - -## Config defining what search method should be applied to different -## fields in the documents. It also contains a mapping from index name -## to a set of fields making up that index. -vsmfields reference diff --git a/vsm/src/vespa/vsm/config/vespa.config.search.vsm.vsmfields.def b/vsm/src/vespa/vsm/config/vespa.config.search.vsm.vsmfields.def deleted file mode 100644 index ca48692b526..00000000000 --- a/vsm/src/vespa/vsm/config/vespa.config.search.vsm.vsmfields.def +++ /dev/null @@ -1,31 +0,0 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -namespace=vespa.config.search.vsm - -## Level of verification applied to the documents received. -documentverificationlevel int default=0 - -## Set if one should ignore limit hits. -searchall int default=1 - -## The name of a field for which we are assigning a search method. -## The field name refers directly to a field in the document model. -fieldspec[].name string - -## The search method for a given field. Note: same field in 2 different document types must match on type if not a random result might be expected. -fieldspec[].searchmethod enum { NONE, BOOL, AUTOUTF8, UTF8, SSE2UTF8, INT8, INT16, INT32, INT64, FLOAT16, FLOAT, DOUBLE } default=AUTOUTF8 -fieldspec[].arg1 string default="" - -## Maximum number of chars to search per field. -fieldspec[].maxlength int default=1048576 - -## Type of the field -fieldspec[].fieldtype enum {ATTRIBUTE, INDEX} default=INDEX - -## The name of a documenttype for which we are assigning a set of indexes. -documenttype[].name string -## The name of an index of a documenttype for which we are assigning a set of fields. -documenttype[].index[].name string - -## The name of a field part of an index. -## The field name refers directly to a field in the document model. -documenttype[].index[].field[].name string diff --git a/vsm/src/vespa/vsm/config/vespa.config.search.vsm.vsmsummary.def b/vsm/src/vespa/vsm/config/vespa.config.search.vsm.vsmsummary.def deleted file mode 100644 index 4664371faba..00000000000 --- a/vsm/src/vespa/vsm/config/vespa.config.search.vsm.vsmsummary.def +++ /dev/null @@ -1,21 +0,0 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -namespace=vespa.config.search.vsm - -## The name of the result class that should be generated for documents -## returned from the VSM. If this value is empty, the first found -## result class will be used. -outputclass string default="" - -## Mapping of field names between the result class and the document -## model. This value represents the name in the result class. Fields -## not mentioned here will get the identity mapping. -fieldmap[].summary string - -## Mapping of field names between the result class and the document -## model. This field vector represents the names in the document model -## that should be used as input when generating the summary field. -fieldmap[].document[].field string - -## This command specifies how the document fields should be combined -## when generating the summary field. -fieldmap[].command enum { NONE, FLATTENJUNIPER, FLATTENSPACE } default=NONE diff --git a/vsm/src/vespa/vsm/config/vsm.def b/vsm/src/vespa/vsm/config/vsm.def new file mode 100644 index 00000000000..c446efe5588 --- /dev/null +++ b/vsm/src/vespa/vsm/config/vsm.def @@ -0,0 +1,13 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +namespace=vespa.config.search.vsm + +## The document model for the documents used as input for the VSM +doctype reference + +## Configuration for storage client used by VSM +storagecfg reference + +## Config defining what search method should be applied to different +## fields in the documents. It also contains a mapping from index name +## to a set of fields making up that index. +vsmfields reference diff --git a/vsm/src/vespa/vsm/config/vsmfields.def b/vsm/src/vespa/vsm/config/vsmfields.def new file mode 100644 index 00000000000..ca48692b526 --- /dev/null +++ b/vsm/src/vespa/vsm/config/vsmfields.def @@ -0,0 +1,31 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +namespace=vespa.config.search.vsm + +## Level of verification applied to the documents received. +documentverificationlevel int default=0 + +## Set if one should ignore limit hits. +searchall int default=1 + +## The name of a field for which we are assigning a search method. +## The field name refers directly to a field in the document model. +fieldspec[].name string + +## The search method for a given field. Note: same field in 2 different document types must match on type if not a random result might be expected. +fieldspec[].searchmethod enum { NONE, BOOL, AUTOUTF8, UTF8, SSE2UTF8, INT8, INT16, INT32, INT64, FLOAT16, FLOAT, DOUBLE } default=AUTOUTF8 +fieldspec[].arg1 string default="" + +## Maximum number of chars to search per field. +fieldspec[].maxlength int default=1048576 + +## Type of the field +fieldspec[].fieldtype enum {ATTRIBUTE, INDEX} default=INDEX + +## The name of a documenttype for which we are assigning a set of indexes. +documenttype[].name string +## The name of an index of a documenttype for which we are assigning a set of fields. +documenttype[].index[].name string + +## The name of a field part of an index. +## The field name refers directly to a field in the document model. +documenttype[].index[].field[].name string diff --git a/vsm/src/vespa/vsm/config/vsmsummary.def b/vsm/src/vespa/vsm/config/vsmsummary.def new file mode 100644 index 00000000000..4664371faba --- /dev/null +++ b/vsm/src/vespa/vsm/config/vsmsummary.def @@ -0,0 +1,21 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +namespace=vespa.config.search.vsm + +## The name of the result class that should be generated for documents +## returned from the VSM. If this value is empty, the first found +## result class will be used. +outputclass string default="" + +## Mapping of field names between the result class and the document +## model. This value represents the name in the result class. Fields +## not mentioned here will get the identity mapping. +fieldmap[].summary string + +## Mapping of field names between the result class and the document +## model. This field vector represents the names in the document model +## that should be used as input when generating the summary field. +fieldmap[].document[].field string + +## This command specifies how the document fields should be combined +## when generating the summary field. +fieldmap[].command enum { NONE, FLATTENJUNIPER, FLATTENSPACE } default=NONE -- cgit v1.2.3