diff options
author | Jon Bratseth <bratseth@yahoo-inc.com> | 2016-06-15 23:09:44 +0200 |
---|---|---|
committer | Jon Bratseth <bratseth@yahoo-inc.com> | 2016-06-15 23:09:44 +0200 |
commit | 72231250ed81e10d66bfe70701e64fa5fe50f712 (patch) | |
tree | 2728bba1131a6f6e5bdf95afec7d7ff9358dac50 /configdefinitions |
Publish
Diffstat (limited to 'configdefinitions')
44 files changed, 1599 insertions, 0 deletions
diff --git a/configdefinitions/.gitignore b/configdefinitions/.gitignore new file mode 100644 index 00000000000..aa75679bac6 --- /dev/null +++ b/configdefinitions/.gitignore @@ -0,0 +1,5 @@ +configdefinitions.iml +target +/pom.xml.build +Makefile +Testing diff --git a/configdefinitions/CMakeLists.txt b/configdefinitions/CMakeLists.txt new file mode 100644 index 00000000000..83a33e54b84 --- /dev/null +++ b/configdefinitions/CMakeLists.txt @@ -0,0 +1,10 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_define_module( + DEPENDS + fastos + vespalib + config_cloudconfig + + LIBS + src/vespa +) diff --git a/configdefinitions/OWNERS b/configdefinitions/OWNERS new file mode 100644 index 00000000000..e0a00db5f4f --- /dev/null +++ b/configdefinitions/OWNERS @@ -0,0 +1 @@ +musum diff --git a/configdefinitions/README b/configdefinitions/README new file mode 100644 index 00000000000..8ee691fca78 --- /dev/null +++ b/configdefinitions/README @@ -0,0 +1 @@ +Common config definitions for cross-module properties. diff --git a/configdefinitions/pom.xml b/configdefinitions/pom.xml new file mode 100644 index 00000000000..8b50f7c8d3e --- /dev/null +++ b/configdefinitions/pom.xml @@ -0,0 +1,65 @@ +<?xml version="1.0"?> +<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd"> + <modelVersion>4.0.0</modelVersion> + <parent> + <groupId>com.yahoo.vespa</groupId> + <artifactId>parent</artifactId> + <version>6-SNAPSHOT</version> + <relativePath>../parent/pom.xml</relativePath> + </parent> + <artifactId>configdefinitions</artifactId> + <packaging>container-plugin</packaging> + <version>6-SNAPSHOT</version> + <name>configdefinitions</name> + <dependencies> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>config-lib</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> + </dependencies> + <build> + <plugins> + <plugin> + <groupId>com.yahoo.vespa</groupId> + <artifactId>bundle-plugin</artifactId> + <extensions>true</extensions> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-compiler-plugin</artifactId> + <configuration> + <compilerArgs> + <arg>-Xlint:all</arg> + <arg>-Werror</arg> + </compilerArgs> + </configuration> + </plugin> + <plugin> + <groupId>com.yahoo.vespa</groupId> + <artifactId>config-class-plugin</artifactId> + <version>${project.version}</version> + <configuration> + <defFilesDirectories>src/vespa/</defFilesDirectories> + </configuration> + <executions> + <execution> + <id>config-gen</id> + <goals> + <goal>config-gen</goal> + </goals> + </execution> + </executions> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-install-plugin</artifactId> + <configuration> + <updateReleaseInfo>true</updateReleaseInfo> + </configuration> + </plugin> + </plugins> + </build> +</project> diff --git a/configdefinitions/src/.gitignore b/configdefinitions/src/.gitignore new file mode 100644 index 00000000000..02d2647f806 --- /dev/null +++ b/configdefinitions/src/.gitignore @@ -0,0 +1,4 @@ +Makefile.ini +config_command.sh +configdefinitions.mak +project.dsw diff --git a/configdefinitions/src/main/java/com/yahoo/cloud/config/package-info.java b/configdefinitions/src/main/java/com/yahoo/cloud/config/package-info.java new file mode 100644 index 00000000000..d9bfa3565ef --- /dev/null +++ b/configdefinitions/src/main/java/com/yahoo/cloud/config/package-info.java @@ -0,0 +1,5 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.cloud.config; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/configdefinitions/src/main/java/com/yahoo/vespa/config/content/package-info.java b/configdefinitions/src/main/java/com/yahoo/vespa/config/content/package-info.java new file mode 100644 index 00000000000..51f5c733378 --- /dev/null +++ b/configdefinitions/src/main/java/com/yahoo/vespa/config/content/package-info.java @@ -0,0 +1,5 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.vespa.config.content; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/configdefinitions/src/main/java/com/yahoo/vespa/config/search/package-info.java b/configdefinitions/src/main/java/com/yahoo/vespa/config/search/package-info.java new file mode 100644 index 00000000000..80250e890fd --- /dev/null +++ b/configdefinitions/src/main/java/com/yahoo/vespa/config/search/package-info.java @@ -0,0 +1,5 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.vespa.config.search; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/configdefinitions/src/main/java/com/yahoo/vespa/config/storage/package-info.java b/configdefinitions/src/main/java/com/yahoo/vespa/config/storage/package-info.java new file mode 100644 index 00000000000..1a8208af963 --- /dev/null +++ b/configdefinitions/src/main/java/com/yahoo/vespa/config/storage/package-info.java @@ -0,0 +1,5 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.vespa.config.storage; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/configdefinitions/src/main/java/com/yahoo/vespa/configdefinition/package-info.java b/configdefinitions/src/main/java/com/yahoo/vespa/configdefinition/package-info.java new file mode 100644 index 00000000000..ed363c90a86 --- /dev/null +++ b/configdefinitions/src/main/java/com/yahoo/vespa/configdefinition/package-info.java @@ -0,0 +1,6 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.vespa.configdefinition; + +import com.yahoo.api.annotations.PublicApi; +import com.yahoo.osgi.annotation.ExportPackage;
\ No newline at end of file diff --git a/configdefinitions/src/vespa/.gitignore b/configdefinitions/src/vespa/.gitignore new file mode 100644 index 00000000000..a9aed4826f0 --- /dev/null +++ b/configdefinitions/src/vespa/.gitignore @@ -0,0 +1,6 @@ +*.So +*.so.* +.depend +Makefile +config-*.cpp +config-*.h diff --git a/configdefinitions/src/vespa/CMakeLists.txt b/configdefinitions/src/vespa/CMakeLists.txt new file mode 100644 index 00000000000..910c8da80ef --- /dev/null +++ b/configdefinitions/src/vespa/CMakeLists.txt @@ -0,0 +1,66 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_library(configdefinitions + SOURCES + INSTALL lib64 + DEPENDS +) +vespa_generate_config(configdefinitions application-id.def) +install(FILES application-id.def DESTINATION var/db/vespa/config_server/serverdb/classes) +vespa_generate_config(configdefinitions attributes.def) +install(FILES attributes.def DESTINATION var/db/vespa/config_server/serverdb/classes) +vespa_generate_config(configdefinitions cluster-info.def) +install(FILES cluster-info.def DESTINATION var/db/vespa/config_server/serverdb/classes) +vespa_generate_config(configdefinitions cluster-list.def) +install(FILES cluster-list.def DESTINATION var/db/vespa/config_server/serverdb/classes) +vespa_generate_config(configdefinitions configserver.def) +install(FILES configserver.def DESTINATION var/db/vespa/config_server/serverdb/classes) +vespa_generate_config(configdefinitions dispatch.def) +install(FILES dispatch.def DESTINATION var/db/vespa/config_server/serverdb/classes) +vespa_generate_config(configdefinitions elk.def) +install(FILES elk.def DESTINATION var/db/vespa/config_server/serverdb/classes) +vespa_generate_config(configdefinitions fleetcontroller.def) +install(FILES fleetcontroller.def DESTINATION var/db/vespa/config_server/serverdb/classes) +vespa_generate_config(configdefinitions ilscripts.def) +install(FILES ilscripts.def DESTINATION var/db/vespa/config_server/serverdb/classes) +vespa_generate_config(configdefinitions indexschema.def) +install(FILES indexschema.def DESTINATION var/db/vespa/config_server/serverdb/classes) +vespa_generate_config(configdefinitions lb-services.def) +install(FILES lb-services.def DESTINATION var/db/vespa/config_server/serverdb/classes) +vespa_generate_config(configdefinitions load-type.def) +install(FILES load-type.def DESTINATION var/db/vespa/config_server/serverdb/classes) +vespa_generate_config(configdefinitions messagetyperouteselectorpolicy.def) +install(FILES messagetyperouteselectorpolicy.def DESTINATION var/db/vespa/config_server/serverdb/classes) +vespa_generate_config(configdefinitions model.def) +install(FILES model.def DESTINATION var/db/vespa/config_server/serverdb/classes) +vespa_generate_config(configdefinitions persistence.def) +install(FILES persistence.def DESTINATION var/db/vespa/config_server/serverdb/classes) +vespa_generate_config(configdefinitions rank-profiles.def) +install(FILES rank-profiles.def DESTINATION var/db/vespa/config_server/serverdb/classes) +vespa_generate_config(configdefinitions routing-provider.def) +install(FILES routing-provider.def DESTINATION var/db/vespa/config_server/serverdb/classes) +vespa_generate_config(configdefinitions sentinel.def) +install(FILES sentinel.def DESTINATION var/db/vespa/config_server/serverdb/classes) +vespa_generate_config(configdefinitions slobroks.def) +install(FILES slobroks.def DESTINATION var/db/vespa/config_server/serverdb/classes) +vespa_generate_config(configdefinitions specialtokens.def) +install(FILES specialtokens.def DESTINATION var/db/vespa/config_server/serverdb/classes) +vespa_generate_config(configdefinitions stor-devices.def) +install(FILES stor-devices.def DESTINATION var/db/vespa/config_server/serverdb/classes) +vespa_generate_config(configdefinitions stor-distribution.def) +install(FILES stor-distribution.def DESTINATION var/db/vespa/config_server/serverdb/classes) +vespa_generate_config(configdefinitions stor-filestor.def) +install(FILES stor-filestor.def DESTINATION var/db/vespa/config_server/serverdb/classes) +vespa_generate_config(configdefinitions stor-memfilepersistence.def) +install(FILES stor-memfilepersistence.def DESTINATION var/db/vespa/config_server/serverdb/classes) +vespa_generate_config(configdefinitions summary.def) +install(FILES summary.def DESTINATION var/db/vespa/config_server/serverdb/classes) +vespa_generate_config(configdefinitions summarymap.def) +install(FILES summarymap.def DESTINATION var/db/vespa/config_server/serverdb/classes) +vespa_generate_config(configdefinitions upgrading.def) +install(FILES upgrading.def DESTINATION var/db/vespa/config_server/serverdb/classes) +vespa_generate_config(configdefinitions ymon.def) +install(FILES ymon.def DESTINATION var/db/vespa/config_server/serverdb/classes) +vespa_generate_config(configdefinitions zookeeper-server.def) +install(FILES zookeeper-server.def DESTINATION var/db/vespa/config_server/serverdb/classes) +vespa_generate_config(configdefinitions zookeepers.def) +install(FILES zookeepers.def DESTINATION var/db/vespa/config_server/serverdb/classes) diff --git a/configdefinitions/src/vespa/application-id.def b/configdefinitions/src/vespa/application-id.def new file mode 100644 index 00000000000..7c312774856 --- /dev/null +++ b/configdefinitions/src/vespa/application-id.def @@ -0,0 +1,8 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +# Application Id config (tenant, application, instance) + +namespace=cloud.config + +tenant string default="(unknowntenant)" +application string default="(unknownapplication)" +instance string default="(unknowninstance)" diff --git a/configdefinitions/src/vespa/attributes.def b/configdefinitions/src/vespa/attributes.def new file mode 100644 index 00000000000..7ff4c797b01 --- /dev/null +++ b/configdefinitions/src/vespa/attributes.def @@ -0,0 +1,28 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +namespace=vespa.config.search + +attribute[].name string +attribute[].datatype enum { STRING, UINT1, UINT2, UINT4, INT8, INT16, INT32, INT64, FLOAT, DOUBLE, PREDICATE, TENSOR, NONE } default=NONE +attribute[].collectiontype enum { SINGLE, ARRAY, WEIGHTEDSET } default=SINGLE +attribute[].removeifzero bool default=false +attribute[].createifnonexistent bool default=false +attribute[].fastsearch bool default=false +attribute[].huge bool default=false +attribute[].sortascending bool default=true +attribute[].sortfunction enum { RAW, LOWERCASE, UCA } default=UCA +attribute[].sortstrength enum { PRIMARY, SECONDARY, TERTIARY, QUATERNARY, IDENTICAL } default=PRIMARY +attribute[].sortlocale string default="" +# Allow bitvector postings in addition to btree postings ? +attribute[].enablebitvectors bool default=false +# Allow only bitvector postings, i.e. drop btree postings to save memory.? +attribute[].enableonlybitvector bool default=false +# Allow fast access to this attribute at all times. +# If so, attribute is kept in memory also for non-searchable documents. +attribute[].fastaccess bool default=false +attribute[].arity int default=8 +attribute[].lowerbound long default=-9223372036854775808 +attribute[].upperbound long default=9223372036854775807 +# The threshold is given as a ratio of the corpus size +attribute[].densepostinglistthreshold double default=0.40 +# Specification of tensor type if this attribute is of type TENSOR. +attribute[].tensortype string default="" diff --git a/configdefinitions/src/vespa/cluster-info.def b/configdefinitions/src/vespa/cluster-info.def new file mode 100644 index 00000000000..fcdf2636165 --- /dev/null +++ b/configdefinitions/src/vespa/cluster-info.def @@ -0,0 +1,12 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +# Information about the cluster the subscriber is a part of +namespace=cloud.config + +# clusterId is same as clustername in model +clusterId string +nodeCount int + +services[].index int default=-1 +services[].hostname string default="(nohostname)" +services[].ports[].number int default=-1 +services[].ports[].tags string default="(notags)" diff --git a/configdefinitions/src/vespa/cluster-list.def b/configdefinitions/src/vespa/cluster-list.def new file mode 100644 index 00000000000..b1c1009e0bd --- /dev/null +++ b/configdefinitions/src/vespa/cluster-list.def @@ -0,0 +1,5 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +namespace=cloud.config + +storage[].name string +storage[].configid string default="" diff --git a/configdefinitions/src/vespa/configserver.def b/configdefinitions/src/vespa/configserver.def new file mode 100644 index 00000000000..48d5321d66d --- /dev/null +++ b/configdefinitions/src/vespa/configserver.def @@ -0,0 +1,34 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +namespace=cloud.config +rpcport int default=19070 +httpport int default=19071 +numthreads int default=16 +zookeepercfg string default="conf/zookeeper/zookeeper.cfg" +zookeeperserver[].hostname string +zookeeperserver[].port int default=2181 +# in seconds +zookeeper.barrierTimeout long default=120 +configModelPluginDir[] string +configServerDBDir string default="var/db/vespa/config_server/serverdb/" +maxgetconfigclients int default=1000000 +maxoutputbuffersize int default=65536 +# in seconds +sessionLifetime long default=3600 +applicationDirectory string default="conf/configserver-app" +masterGeneration long default=0 +multitenant bool default=false +numDelayedResponseThreads int default=1 +payloadCompressionType enum { UNCOMPRESSED, LZ4 } default=LZ4 +serverId string default="localhost" +hostedVespa bool default=false +numParallelTenantLoaders int default=1 + +# Zone information +environment string default="prod" +region string default="default" +defaultFlavor string default="default" +defaultAdminFlavor string default="default" +defaultContainerFlavor string default="default" +defaultContentFlavor string default="default" + +useVespaVersionInRequest bool default=false diff --git a/configdefinitions/src/vespa/dispatch.def b/configdefinitions/src/vespa/dispatch.def new file mode 100644 index 00000000000..4e966fae843 --- /dev/null +++ b/configdefinitions/src/vespa/dispatch.def @@ -0,0 +1,14 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +# Configuration of dispatch from container nodes to search clusters + +namespace=vespa.config.search + +# The unique key of a search node +node[].key int + +# The host name of this search node +node[].host string + +# The port of this search node +node[].port int + diff --git a/configdefinitions/src/vespa/elk.def b/configdefinitions/src/vespa/elk.def new file mode 100644 index 00000000000..0b144e84293 --- /dev/null +++ b/configdefinitions/src/vespa/elk.def @@ -0,0 +1,24 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +namespace=cloud.config + +# elasticsearch: for future use +elasticsearch[].host string + +# Port 0 (default) means use the Defaults web service port +elasticsearch[].port int default=0 + +# logstash forwarder +logstash.network.servers[].host string +logstash.network.servers[].port int default=5043 +logstash.network.timeout int default=15 +logstash.files[].paths[] string +logstash.files[].fields{} string + +# A relative path will be prepended by vespa home. +# An absolute path will be used as-is. +logstash.config_file string default="conf/logstash-forwarder/config.json" + +logstash.source_field string default="index_source" +logstash.spool_size int default=100 +# Note: verbose=true actually means "not noisy" in the forwarder +logstash.forwarder_command_line_params string default = "-verbose=true" diff --git a/configdefinitions/src/vespa/fleetcontroller.def b/configdefinitions/src/vespa/fleetcontroller.def new file mode 100644 index 00000000000..919e9fc2209 --- /dev/null +++ b/configdefinitions/src/vespa/fleetcontroller.def @@ -0,0 +1,140 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +namespace=vespa.config.content + +## Name of VDS cluster +cluster_name string restart + +## The fleetcontroller index. Each fleetcontroller should have a unique index +## which can be used to identify them. +index int restart + +## Number of fleet controllers. If more than one, the fleetcontroller needs to +## do master election in order to know which one is active. It then needs to +## know how many exist. +fleet_controller_count int default=1 + +## The number of seconds we can attempt to talk to zookeeper before our session +## time out and we lose our connection. +zookeeper_session_timeout double default=30.0 + +## When a master candidate see the master disappearing from ZooKeeper, it wants +## to take over as master. But, before taking over, the master should be aware +## that it has lost its zookeeper connection, so it will stop reacting as +## master. Suggests setting this to 2-3 times the zookeeper session timeout. +## (Set in number of seconds) +master_zookeeper_cooldown_period double default=60.0 + +## Sets how many fleetcontrollers will gather state. A fleetcontroller +## gathering state can take over quicker should ## the master fail. +## If set to 1, only master will gather state. If set higher, others will +## also do so, prioritizing those fleetcontrollers likely to be the ones to +## take over if the master fails. +state_gather_count int default=2 + +## Location of ZooKeeper servers +zookeeper_server string restart + +## RPC Port used by fleetcontroller +rpc_port int default=6500 restart + +## Port where fleetcontroller listens for HTTP status request +http_port int default=0 restart + +## Maximum number of milliseconds a storage will be automatically reported in +## maintenance due to node recently being availble. (If 0, no time at all) +storage_transition_time int default=30000 + +## Maximum number of milliseconds a distributor will be automatically reported +## in maintenance due to node recently being availble. (If 0, no time at all) +## +## Currently default is 0.. Should probably be more then we know it is working +## correctly +distributor_transition_time int default=0 + +## Maximum number of milliseconds allowed between progress increase during +## initialization. If no progress have been made during this time period, the +## node will be considered down. +## +## Currently disabled as 5.0 nodes gets load while initializing which may be +## higher pri than initializing, so nodes can stay in init for a long time. +init_progress_time int default=0 + +## Minimum time (in ms) between system state updates. To limit updates in a +## system where a lot is happening at the same time, this value will make sure +## we dont change the state too often. +min_time_between_new_systemstates int default=10000 + +## Sets how many milliseconds to wait between each state poll for old nodes +## requiring state polling. (4.1 or older) +state_polling_frequency int default=5000 + +## The maximum amount of premature crashes a node is allowed to have in a row +## before the fleetcontroller disables that node. +max_premature_crashes int default=100000 + +## If a node has been down or up this many milliseconds, clear the premature +## crash count of a node and consider the node as stable +stable_state_time_period int default=7200000 + +## The maximum number of events to keep in the event log +event_log_max_size int default=1024 + +## The maximum number of node events to keep in the node event log per node +event_node_log_max_size int default=1024 + +## The total number of distributor nodes that can exist. If 0, we dont know and +## will use the highest distributor index number we have ever seen + 1. +total_distributor_count int default=0 + +## The total number of storage nodes that can exist. If 0, we dont know and +## will use the highest storage index number we have ever seen + 1. +total_storage_count int default=0 + +## The minimum number of distributor nodes that should be up for the cluster +## state to be up. (Retired nodes counts as up in this case) +min_distributors_up_count int default=1 + +## The minimum number of storage nodes that should be up for the cluster state +## to be up (Retired nodes counts as up in this case) +min_storage_up_count int default=1 + +## The minimum ratio of known distributor nodes that should be up (or retired) +## for the cluster state to stay up. +min_distributor_up_ratio double default=0.01 + +## The minimum ratio of known storage nodes that should be up (or retired) for +## the cluster state to stay up. +min_storage_up_ratio double default=0.01 + +## Seconds to sleep after doing a work cycle where we did no work. Some +## events do not interrupt the sleeping, such as slobrok changes, so shouldnt +## set this too high +cycle_wait_time double default=0.1 + +## Minimum time to pass in seconds before broadcasting our first systemstate as +## a new fleetcontroller. (Will broadcast earlier than this if we have gathered +## state from all before this). To prevent disturbance when taking over as +## fleetcontroller, give nodes a bit of time to answer so we dont temporarily +## report nodes as down. +min_time_before_first_system_state_broadcast double default=5.0 + +## Request timeout of node state requests. Keeping a high timeout allows us to +## always have a pending operation with very low cost. Keeping a low timeout is +## good to detect issues like packet loss. The default tries to balance the two +## by not resending too often, but detecting packet loss within a minute at +## least. If we can guarantuee RPC layer to fail on packet loss within +## reasonable time we should increase this default. +get_node_state_request_timeout double default=120.0 + +## If a node is out of slobrok longer than this time period, assume the node +## is down, even if we have a pending node state request to it. Slobrok does +## a bit more keep alive checking than fleetcontroller, so it is possible that +## the node disappears from slobrok while it still looks ok in fleetcontroller. +max_slobrok_disconnect_grace_period double default=60.0 + +## Whether to show system states that have never been sent to storage nodes in +## the event log. +show_local_systemstates_in_event_log bool default=true + +## The ideal number of distribution bits this system should have +ideal_distribution_bits int default=16 diff --git a/configdefinitions/src/vespa/ilscripts.def b/configdefinitions/src/vespa/ilscripts.def new file mode 100644 index 00000000000..1ac0723eb75 --- /dev/null +++ b/configdefinitions/src/vespa/ilscripts.def @@ -0,0 +1,10 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +version=5 +namespace=vespa.configdefinition + +## The maximum number of occurrences of a given term to index per field +maxtermoccurrences int default=100 + +ilscript[].doctype string +ilscript[].docfield[] string +ilscript[].content[] string diff --git a/configdefinitions/src/vespa/indexschema.def b/configdefinitions/src/vespa/indexschema.def new file mode 100644 index 00000000000..c6879f5673c --- /dev/null +++ b/configdefinitions/src/vespa/indexschema.def @@ -0,0 +1,26 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +## Config specifying the index fields and field collections that are part of an index schema. +version=7 +namespace=vespa.config.search + +## The name of the index field. +indexfield[].name string +## This is the type of an index. Currently only rise and vespa are supported +indexfield[].indextype enum {VESPA, RISE} default=VESPA +## The data type of the index field. +indexfield[].datatype enum { STRING, INT64, BOOLEANTREE } default=STRING +## The collection type of the index field. +indexfield[].collectiontype enum { SINGLE, ARRAY, WEIGHTEDSET } default=SINGLE +## Whether the index should support prefix searches. +indexfield[].prefix bool default=false +## Whether the index should support phrase searches. +indexfield[].phrases bool default=false +## Whether the index should have posting lists with word positions. +indexfield[].positions bool default=true +## Average element length +indexfield[].averageelementlen int default=512 + +## The name of the field collection (aka logical view). +fieldset[].name string +## The name of the index field that are part of this collection. +fieldset[].field[].name string diff --git a/configdefinitions/src/vespa/lb-services.def b/configdefinitions/src/vespa/lb-services.def new file mode 100644 index 00000000000..2dc57723a8f --- /dev/null +++ b/configdefinitions/src/vespa/lb-services.def @@ -0,0 +1,20 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +# Config for load balancer that needs to know about all services for +# all tenants and applications in a config server cluster + +namespace=cloud.config + +# Active rotation given as flag 'active' for a prod region in deployment.xml +# Default true for now (since code in config-model to set it is not ready yet), should have no default value +tenants{}.applications{}.activeRotation bool default=true + +tenants{}.applications{}.hosts{}.hostname string default="(unknownhostname)" +tenants{}.applications{}.hosts{}.services{}.type string default="(noservicetype)" +tenants{}.applications{}.hosts{}.services{}.clustertype string default="(unknownclustertype)" +tenants{}.applications{}.hosts{}.services{}.clustername string default="(unknownclustername)" +tenants{}.applications{}.hosts{}.services{}.configId string +tenants{}.applications{}.hosts{}.services{}.index int default=0 +tenants{}.applications{}.hosts{}.services{}.ports[].number int default=-1 +tenants{}.applications{}.hosts{}.services{}.ports[].tags string default="(notags)" +tenants{}.applications{}.hosts{}.services{}.servicealiases[] string +tenants{}.applications{}.hosts{}.services{}.endpointaliases[] string diff --git a/configdefinitions/src/vespa/load-type.def b/configdefinitions/src/vespa/load-type.def new file mode 100644 index 00000000000..5d11584fc77 --- /dev/null +++ b/configdefinitions/src/vespa/load-type.def @@ -0,0 +1,30 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +## This config is used to define load types in VESPA. Load types are merely +## a way to distinguish types of load, as to present such types differently. +## For instance, some metrics will be presented per load type, such that you +## can see how that specific type is being handled. Load types are especially +## useful for separating loads of different priority. + +namespace=vespa.config.content + +## The identifying index of this type. 0 cannot be used as it is +## reserved for the default type. Other types should be +## non-overlapping from 1 and upwards. No gaps are allowed in the +## index range. (For performance reasons we want to use arrays +## internally where the id is the index) +## +## The identifier is used for efficient transfer of type information. +## The types will be identified by names instead in more textual or +## verbose interfaces. +## restart flag was added automatically and needs to be verified. +type[].id int restart + +## The name of a given load type. Each load type must have a unique +## name. The name "default" is reserved for the default type of id 0. +## restart flag was added automatically and needs to be verified. +type[].name string restart + +## The default priority for the load type. +## restart flag was added automatically and needs to be verified. +type[].priority string restart + diff --git a/configdefinitions/src/vespa/messagetyperouteselectorpolicy.def b/configdefinitions/src/vespa/messagetyperouteselectorpolicy.def new file mode 100644 index 00000000000..f8051277524 --- /dev/null +++ b/configdefinitions/src/vespa/messagetyperouteselectorpolicy.def @@ -0,0 +1,11 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +namespace=vespa.config.content + +# Default route if no override is set for a type. +defaultroute string + +# The name of the route. +route[].name string + +# The document protocol messagetype triggering this route. +route[].messagetype int diff --git a/configdefinitions/src/vespa/model.def b/configdefinitions/src/vespa/model.def new file mode 100644 index 00000000000..fcd73315a89 --- /dev/null +++ b/configdefinitions/src/vespa/model.def @@ -0,0 +1,13 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +namespace=cloud.config + +vespaVersion string default="unknown" +hosts[].name string default="(unknownhostname)" +hosts[].services[].name string default="(noservicename)" +hosts[].services[].type string default="(noservicetype)" +hosts[].services[].configid string default="(noconfigid)" +hosts[].services[].clustertype string default="(unknownclustertype)" +hosts[].services[].clustername string default="(unknownclustername)" +hosts[].services[].index int default=0 +hosts[].services[].ports[].number int default=-1 +hosts[].services[].ports[].tags string default="(notags)" diff --git a/configdefinitions/src/vespa/persistence.def b/configdefinitions/src/vespa/persistence.def new file mode 100644 index 00000000000..90c881c3267 --- /dev/null +++ b/configdefinitions/src/vespa/persistence.def @@ -0,0 +1,60 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +namespace=vespa.config.content + +## If set, causes the service layer to trigger an abort of any queued +## operations for buckets whose ownership changes during a cluster state +## change. This helps ensure consistency between the distributors and +## the storage nodes by discarding operations that were not sent by the +## new owning distributor. The service layer will also block any new +## operations from happening until all operations currently executing for +## buckets with changed ownership have completed. +## Enabling this option has a potential of causing a minor performance +## impact when a node goes up or down due to its blocking semantics, but +## this should be on the order of tens to a few hundred milliseconds. +abort_operations_with_changed_bucket_ownership bool default=true + +## If set, aborts any mutating ideal state operations that arrive to a node +## that were scheduled and sent by distributors that no longer own the buckets +## the operations are bound for. This option complements the above config, +## as this is checked for every arriving message and not when the state changes. +abort_outdated_mutating_ideal_state_ops bool default=true + +## If set, aborts any external (client) operations that arrive to a node from +## a distributor that is no longer the owner of the bucket those operations +## belong to. As with the abort_outdated_mutating_ideal_state_ops option, +## this is a check that happens whenever a message arrives, not when the state +## changes. +abort_outdated_mutating_external_load_ops bool default=false + +## Whether or not to disable partitions when they report I/O errors +fail_partition_on_error bool default=true + +## Time period to keep all updates (given in seconds). One can revert any +## operation done within this time. +revert_time_period int default=300 + +## Minimum number of seconds that document remove (tombstone) entries are +## retained in a bucket before being automatically compacted away. In the case +## where a node has been down for a longer period of time than this value +## and then comes back up, there exists a possibility for documents to be +## resurrected. This happens when documents existing on both the downed node +## and the online nodes are marked as removed on the online nodes and then +## subsequently compacted away along with their remove entries. When the old +## node comes back online, it will appear to the system as if the node has +## documents that do not exist on other nodes (as no remove-entries can be +## found for them) and they will subsequently re-appear. +keep_remove_time_period int default=604800 + +## For providers supporting multiple document versions, this option specifies +## a soft limit on the maximum number of versions that can be stored for a +## single document at any given time. This is considered a soft limit as it +## is possible for more versions to exist concurrently than what this config +## value specifies in a case where the pressure on the containing bucket is not +## yet high enough to warrant a compaction. +## After a compaction has taken place, only the n _newest_ document versions +## will be present in the compacted representation of the bucket. +## This limitation takes precedence over what versions may be stored according +## to revert_time_period. +## Configured value must be at least 1. +maximum_versions_of_single_document_stored int default=5 + diff --git a/configdefinitions/src/vespa/rank-profiles.def b/configdefinitions/src/vespa/rank-profiles.def new file mode 100644 index 00000000000..d47e332daeb --- /dev/null +++ b/configdefinitions/src/vespa/rank-profiles.def @@ -0,0 +1,13 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +version=13 +namespace=vespa.config.search + +## name of this rank profile. maps to table index for internal use. +rankprofile[].name string + +## the name of a generic property available to the feature execution framework and feature plugins +rankprofile[].fef.property[].name string + +## the value of a generic property available to feature plugins +rankprofile[].fef.property[].value string + diff --git a/configdefinitions/src/vespa/routing-provider.def b/configdefinitions/src/vespa/routing-provider.def new file mode 100755 index 00000000000..bda7d5d54b3 --- /dev/null +++ b/configdefinitions/src/vespa/routing-provider.def @@ -0,0 +1,7 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +# Configuration for the routing config provider + +namespace=cloud.config + +# disabled by default, automatically enabled for hosted vespa +enabled bool default=false diff --git a/configdefinitions/src/vespa/routing.def b/configdefinitions/src/vespa/routing.def new file mode 100755 index 00000000000..a1acd2aaf23 --- /dev/null +++ b/configdefinitions/src/vespa/routing.def @@ -0,0 +1,7 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +# List of routing hosts populated from the zone-application in Hosted mode + +namespace=cloud.config + +# Host name(s) of routing/proxy nodes +hosts[] string diff --git a/configdefinitions/src/vespa/sentinel.def b/configdefinitions/src/vespa/sentinel.def new file mode 100644 index 00000000000..d35137b214f --- /dev/null +++ b/configdefinitions/src/vespa/sentinel.def @@ -0,0 +1,50 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +namespace=cloud.config + +## The port where sentinel should run its telnet interface +port.telnet int default=19098 + +## The port where sentinel should run its RPC interface +port.rpc int default=19097 + +## Meta info about the application running on this node. All values are +## exported to child processes as $VESPA_APPLICATION_TENANT, +## $VESPA_APPLICATION_NAME, $VESPA_APPLICATION_ENVIRONMENT, $VESPA_APPLICATION_REGION +## and $VESPA_APPLICATION_INSTANCE +application.tenant string default="default" +application.name string default="default" +application.environment string default="default" +application.instance string default="default" +application.region string default="default" + +## The command to run. This will be run by sh -c, and the following +## environment variables are defined: $ROOT, $VESPA_SERVICE_NAME, +## $VESPA_CONFIG_ID +## +## stdin points to /dev/null +## stdout and stderr connected to sentinel via pipe on startup. +service[].command string + +## The command to run before stopping service. The same properties as for +## startup command holds. +service[].preShutdownCommand string default="" + +## The service name for this service. Exported in VESPA_SERVICE_NAME +## environment variable. All services must have unique names. +## The service name is used by to start/stop services, and to list +## them. It is also logged in the service field in every log message. +service[].name string + +## Should the service be auto-started by configd as soon as it starts up, +## or must it be started manually? Almost always true. +service[].autostart bool default=false + +## Should the service automatically be restarted if it exits? +service[].autorestart bool default=true + +## Config reference to the service being started. +## Exported in the VESPA_CONFIG_ID environment variable. +service[].id reference + +## Affinity of this service towards a CPU socket +service[].affinity.cpuSocket int default=-1 diff --git a/configdefinitions/src/vespa/slobroks.def b/configdefinitions/src/vespa/slobroks.def new file mode 100644 index 00000000000..5570ede7606 --- /dev/null +++ b/configdefinitions/src/vespa/slobroks.def @@ -0,0 +1,14 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +namespace=cloud.config + +### All params must be flagged as 'restart' because this config is manually +### retrieved by ConfiguredApplication.start to init the rpc server in +### com.yahoo.container.Server. + +## The connectionspec for a slobrok is used for connecting to it using +## the FNET Remote Tools framework. It is normally on the form +## tcp/hostname:portnumber where hostname is the same name returned by +## gethostname() on the machine. This string is also used by the +## actual slobrok to figure out which port it should listen to. + +slobrok[].connectionspec string restart diff --git a/configdefinitions/src/vespa/specialtokens.def b/configdefinitions/src/vespa/specialtokens.def new file mode 100644 index 00000000000..55a0c8e03cb --- /dev/null +++ b/configdefinitions/src/vespa/specialtokens.def @@ -0,0 +1,33 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +## Named lists of special tokens - string which should be +## treated as words no matter what characters they consist of. +## A special token can also optionally be replaced by another +## token. +## +## Special tokens are case sensitive, since queries and tokens +## are lowercased they should normally be written in lower case. +## +## The list named default is the only list which is used currently +## +## Order matters in token lists. If one special token is a +## prefix of another token, Vespa will prefer to return the first +## matching token in the list when encountering the longest +## special token. +version=3 +namespace=vespa.configdefinition + +## Path to makefsa binary, needed to create specialtokens-dictionary +## If the path is relative, vespa home is prepended +makefsaPath string default="bin64/makefsa" + +## Use 'default' as the name of the list to use in regular +## indexing and queries +tokenlist[].name string + +## Special tokens, any string is permitted +tokenlist[].tokens[].token string + +## The token which should replace the special token +## If no replace is set for a token, the word token +## will be the special token string itself +tokenlist[].tokens[].replace string default="" diff --git a/configdefinitions/src/vespa/stor-devices.def b/configdefinitions/src/vespa/stor-devices.def new file mode 100644 index 00000000000..dc4bd857e22 --- /dev/null +++ b/configdefinitions/src/vespa/stor-devices.def @@ -0,0 +1,69 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +namespace=vespa.config.storage + +## Root directory for all files related to this storage node. +## Will typically be "$VESPA_HOME/var/db/vespa/vds/<cluster>/<nodetype>/<index>" +## Within this directory the "disks/d<dirindex>" directories/symlinks will +## contain the actual data stored by storage. +root_folder string restart + +## List of disks to store the storage data in. Make an empty directory for +## each disk_path[index] if it does not exist yet, and make a symlink from each +## "<root_folder>/disks/d<index>" to disk_path[index] if it doesn't exist yet. +disk_path[] string restart + +## Capacity of the disks on this node. How much data and load will each disk +## get relative to the other disks on this node. The default disk capacity is +## 1.0. If disk capacity array size does not match disk counts, it will use the +## values that exist, and use 1.0 for the rest. +disk_capacity[] double restart + +## The device mapper is used to detect what storage directories map to what +## partitions and disks, such that then a disk event arise, it arise for all +## directories using that disk. Storage has two different device mappers. One +## that fakes the devices, assuming all VDS directories use unique partitions +## and disks, and a real one trying to detect what is actually being used. +## +## Advantage of simple one is that one can fake multiple disks, which may be +## useful in testing. Problem being that then storage gives clues that it has +## added X bytes to a partition, other directories on the same partition will +## not be adjusted. Thus, one might want to cache stat results less due to +## numbers being incorrect. +## +## If using a real mapper, disk events such as IO failures will mark all +## directories using partitions on that disk bad, rather than just the directory +## containing the file which we had trouble using. We dont detect real physical +## disks. Currently we detect whether they use the same *nix device or not. +device_mapper enum { SIMPLE_MAPPER, REAL_MAPPER } default=REAL_MAPPER restart + +## Whether or not to fail directories on same partition and/or disk as another +## directory on failures. Our goal is to return directory errors on failures +## only affecting one directory, partition failures for errors only affecting a +## single partition and so on. In reality, however, it is very hard to know if +## the problem is partition or disk related for instance. Both a corrupt +## filesystem and a bad disk may show the same IO error to storage. We +## recommend having this enabled, so we can attempt to do logic, but it is +## possible to turn it of, in case you see the logic doesnt work for you (and +## that you get more directories than needed marked down and that troubles +## your system) +fail_multiple_directories_on_common_events bool default=true restart + +## Storage stats file systems regularily to check how full they are. To avoid +## constantly calling this system call storage may cache the result and validate +## changes once in a while. The always policy always calls the real statfs +## function when asked for disk fullness. This may be useful in system testing +## where you want to test disk full situations (but not this policy). The stat +## once policy does a single stat upon start, and depends on storage to give +## clues about changes in disk usage to report correct results (currently not +## recommended since we dont guarantuee the clues are 100% correct). The period +## policy asks for each Nth request, and so does the dynamic policy, but the +## latter one has a dynamic period, being large when disk has low fill rate, but +## small with high fillrate. The dynamic policy is recommended as it doesnt +## require exact change clues requires few stat calls as long as the disks +## arent close to full. +statfs_policy enum { STAT_ALWAYS, STAT_ONCE, STAT_PERIOD, STAT_DYNAMIC } default=STAT_DYNAMIC restart + +## If set to non-null, overrides the code default period for stat period and +## dynamic policies. See PartitionMonitor class for details. +statfs_period int default=0 restart + diff --git a/configdefinitions/src/vespa/stor-distribution.def b/configdefinitions/src/vespa/stor-distribution.def new file mode 100644 index 00000000000..618a79abd5d --- /dev/null +++ b/configdefinitions/src/vespa/stor-distribution.def @@ -0,0 +1,70 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +namespace=vespa.config.content + +## Redundancy decides how many copies of documents will be stored. Documents +## will store copies on enough nodes, such that the sum of the node reliability +## factors is equal or above the redundancy. +redundancy int default=3 + +## Initial redundancy allows put-operations to return as completed after +## a subset of all copies have been persisted (known as initial persistence +## in BCP terms). +## Set to 0 to disable and use normal redundancy behavior instead. +initial_redundancy int default=0 + +## If set, requires that the primary nodes copy should always be persisted +## before an n-of-m initial persistence operation can complete. +ensure_primary_persisted bool default=true + +## The number of copies that should be "ready" to be active. +## Maximum is redundancy. +ready_copies int default=0 + +## If this option is set true, the distributor will try to enforce one active copy +## of buckets per leaf hierarchical group. This is a simple implementation for +## search to be able to setup top level dispatcher to only send search to all +## nodes in one group as they have a static cost per node used. It used, +## hierarchical grouping can not be used for other purposes. Using this option +## implies that: +## - ready_copies == redundancy +## - Only one level of hierarchical grouping defined. +## - That level distributes copies to all defined groups. +active_per_leaf_group bool default=false + +## Search have some unfortunate properties with some queries, adding a static +## query cost, independent of the number of documents searched on a node. For +## these cases they use many small groups to be able to scale. In such cases, +## where groups are small, having distributors able to take over when none are +## available in a group is useful. +distributor_auto_ownership_transfer_on_whole_group_down bool default=true + +## Hierarchical grouping divides the nodes into a tree of groups. Due to config +## liking flat structures. The tree of groups is represented by a single array +## of groups defined here, where index is a string that can have a form like +## "1.2.1" in order to tell where in the tree structure the group is. +group[].index string +## Each group needs to have a name. Makes it easier to operations that they can +## name their groups according to what they want to call them. +group[].name string +## Capacity of the given group +group[].capacity double default=1 +## Partitions define how copies are divided among child groups/nodes. +group[].partitions string default="" +## Leaf groups will have a set of nodes within them. Branch groups will have +## none. +group[].nodes[].index int +# Whether this node is retired. +# The system will migrate all data away from retired nodes such that they can +# eventually be removed without partial data loss +group[].nodes[].retired bool default=false + +## Which disk distribution to use. From the current choices, we very much +## recommend using MODULO_BID. The only reason to use any of the earlier one is +## during an upgrade where you dont want to mess up the distribution in case you +## want to revert the upgrade. +## +## MODULO old one (4.0) +## MODULO_INDEX with node index in seed +## MODULO_KNUTH with random(node index) in seed +## MODULO_BID using all used bits, except count bits, and random(node index) +disk_distribution enum { MODULO, MODULO_INDEX, MODULO_KNUTH, MODULO_BID } default=MODULO_BID diff --git a/configdefinitions/src/vespa/stor-filestor.def b/configdefinitions/src/vespa/stor-filestor.def new file mode 100644 index 00000000000..4cbf160190c --- /dev/null +++ b/configdefinitions/src/vespa/stor-filestor.def @@ -0,0 +1,443 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +namespace=vespa.config.content + +## Use the new storage core +use_new_core bool default=false restart + +## FILE LAYOUT PARAMETERS + +## Number of directories per level to spread files across +## DEPRECATED: see stor-memfilepersistence config instead +dir_spread int default=256 restart + +## Number of directory levels +## DEPRECATED: see stor-memfilepersistence config instead +dir_levels int default=1 restart + +## FILE SIZE PARAMETERS + +## Minimum number of meta data entries in one slotfile. When creating new +## files or resizing files, enforce it to contain at least this many meta +## entries. Set to 512 by default, using 20544 bytes total for metadata in +## new files. +## DEPRECATED: see stor-memfilepersistence config instead +minimum_file_meta_slots int default=512 + +## Maximum number of entries in one slotfile. File must be split before +## accepting more data +## +## Default ensure meta data is less than 512 kB. +## DEPRECATED: see stor-memfilepersistence config instead +maximum_file_meta_slots int default=13106 + +## Minimum size of header block. At least this amount of header space will +## be available in new or resized files. For 512 documents, this will be +## 200 bytes per document. +## DEPRECATED: see stor-memfilepersistence config instead +minimum_file_header_block_size int default=102848 + +## Maximum header block size (in bytes). Since the whole meta data list and +## header block needs to be read into memory for some operations, a limit +## can be set for the header block, to avoid consuming too much memory. +## +## Default is set high, as we dont configure it automatically right now, so we +## would rather people configure it down than up. +## DEPRECATED: see stor-memfilepersistence config instead +maximum_file_header_block_size int default=33554432 + +## Minimum size of a single slotfile. When creating or resizing files, they +## will never become smaller than this. Default of 1 MB, will be 1807 byte +## per doc if we use all 512 meta data entries set as default min meta +## entries. +## DEPRECATED: see stor-memfilepersistence config instead +minimum_file_size int default=1048576 + +## Maximum size of a single slotfile. File must be split before accepting +## more data. Will return file full errors. +## +## Default is set high, as we dont configure it automatically right now, so we +## would rather people configure it down than up. +## DEPRECATED: see stor-memfilepersistence config instead +maximum_file_size int default=268431360 + +## When creating new files, always create files as a multiplum of this size. +## Should be set to the block size on the disk, or possibly a multiplum of +## that, though we know no advantage of setting it to more than the block +## size. If you want more free space in files, you should rather adjust that +## with growfactor or min file sizes. +## DEPRECATED: see stor-memfilepersistence config instead +file_block_size int default=4096 + +## DETECT FAILURE PARAMETERS + +## If persistence operations take more than this amount of milliseconds, it +## will be logged as a warning. +warn_on_slow_operations int default=5000 + +## After seeing given number of errors on a disk, storage will disable the +## disk and restart. If set to 0, storage will never disable disks. Note +## that if you get disk errors, which arent automatically fixed, this will +## make storage keep failing all operations towards this disk, and all +## retries will use the same disk and same file, so you might end up with a +## lot of operations that can never succeed. Thus, it is not recommended to +## turn this off. +fail_disk_after_error_count int default=1 restart + +## If a disk operation takes more time in seconds than the disk operation +## timeout, storage will consider that a disk failure, and mark the disk +## bad, restart and not use the disk anymore. Note that currently, this +## includes CPU time too, so if used, this parameter should be set fairly +## high as we still sometimes have some operations that might take a few +## seconds. Disabled by default as we currently have these long operations +## during normal load. +disk_operation_timeout int default=0 restart + +## Timestamps provided for entries written are normally provided externally. +## To detect clock skew between nodes, storage will warn if timestamps look +## out of sync. Future time clearly indicates clock skew and thus the limit +## here is low. Time in the past might just indicate that the operation has +## been queued for a while, so the default value here is not very strict. +## Time is given in seconds. Note that there are valid usecases for +## inserting data with old timestamps, for instance when synchronizing two +## copies of the same data. Warnings should not be printed in these cases. +time_future_limit int default=5 +time_past_limit int default=3600 + +## Enabling debug verifications will make storage do extra verifications +## to find errors as soon as possible. These extra verifications will use up +## a lot of resources though, and should not be needed in normal operations. +## They are mostly to be used during test phases or when debugging problems. +## The value itself is a bit mask, where you can enable different kinds of +## verifications by setting given bits. +debug_verifications int default=0 restart + +## CONSISTENCY PARAMETERS + +## If true, fsync after all disk operations, to ensure no dirty OS file +## cache afterwards. This is only useful if using cached IO, which is not +## recommended to start with. +## DEPRECATED: see stor-memfilepersistence config instead +fsync_after_each_operation bool default=false restart + +## Time period to keep all updates (given in seconds). One can revert any +## operation done within this time. +## DEPRECATED: see stor-memfilepersistence config instead +revert_time_period int default=300 + +## If a remove is older than the reverttimeperiod, the document it is +## removing may be removed from the file. There are a few advantages to +## keeping removes or a bit longer though. If you use this copy to +## synchronize another copy of data, having the remove entry makes it easy +## to detect that you should delete this entry from the other data copy. +## This is useful for internal synchronization of files within VDS if you +## use multiple copies, or for partial recovery or BCP situations. To +## guarantuee consistency in such situations, a data destination that have +## been unavailable for longer than this amount of time, should have its +## data cleared before being set into the system again. This value is given +## in seconds, with the default being one week +## DEPRECATED: see stor-memfilepersistence config instead +keep_remove_time_period int default=604800 + +## Maximum number of entries to keep for a single document. Heavy resending +## within the revert time period may add loads of entries to a file, ruining +## performance. This setting sets a maximum number of entries for each +## document. This will make entries potentially live shorter than the revert +## time period to avoid a resending worst case. A value of 0 means infinite. +## DEPRECATED: see stor-memfilepersistence config instead +maximum_versions_of_single_document_stored int default=0 + +## PERFORMANCE PARAMETERS + +## Number of threads to use for each mountpoint. VDS needs memory per thread +## to perform disk operations, so increasing this number will increase +## memory usage, but it will also make the disk queue on a given disk be +## able to be larger, such that the disk can choose operations to optimize +## seek time. +## See benchmarks for performance/memory tradeoff. +threads[].lowestpri int default=255 restart + +## Pause operations (and block new ones from starting) with priority +## lower than this value when executing operations with higher pri than +## min_priority_to_be_blocking +max_priority_to_block int default=255 restart +min_priority_to_be_blocking int default=0 restart + +## Chunksize to use while merging buckets between nodes. +## +## Default is set to 4 MB - 4k. This is to allow for malloc to waste some bytes +## with tracking info or to align to 512b without passing allocation limit, +## while still reading 4k blocks from disk. +bucket_merge_chunk_size int default=4190208 restart + +## When reading a slotfile, one does not know the size of the meta data +## list, so one have to read a static amount of data, and possibly read more +## if one didnt read enough. This value needs to be at least 64 byte to read +## the initial header stating the true size of the meta data list. +## DEPRECATED: see stor-memfilepersistence config instead +initial_index_read int default=61440 + +## Similar to index read, when reading the document identifiers, one does +## not know the length of the name prior to reading. Thus, if you read less +## than the size, you will have to do an extra read to get the rest. If you +## use very long document identifiers you should increase this value to be +## larger than most of your identifiers. +## restart flag was added automatically and needs to be verified. +## DEPRECATED: see stor-memfilepersistence config instead +initial_name_read int default=512 restart + +## When we need to read (or write) multiple entries in a file where we can +## either read a big enough section to cover all of them. But at some +## distance between the areas we need, it becomes beneficial to do multiple +## reads rather than to read over them. This setting set how many sequential +## bytes we dont need that we allow to be read/written in order to join two +## logical IO operations together in the application. Setting this low might +## be ok if system calls are cheap and we manage to queue up next IO +## operation in time such that the disk dont need to spin an extra round. +## Setting it high will make the disk more likely to process them together, +## but the time used to read/write the gap might have been used to do +## something useful instead. +## DEPRECATED: see stor-memfilepersistence config instead +maximum_gap_to_read_through int default=65536 + +## Currently not in here as we dont have append functionality yet. Might +## improve performance in some cases. +## max_file_appends int default=0 + +## When writing with direct IO, we need to align blocks to 512b, and to +## avoid reading we write garbage after each doc to fill 512b block. When +## resizing or splitting the file we can realign the files such that we +## remove the gaps of existing data, as we will rewrite everything anyhow. +## If using very small documents this might improve your disk space +## utilization. For larger documents it doesnt really reduce much, so might +## be useful to turn it off to save CPU. +## DEPRECATED: see stor-memfilepersistence config instead +remove_file_gaps_on_rewrite bool default=true restart + +## The order we try to enforce in file body blocks. Visitors that only need +## to visit some of the data in a bucket will be able to read less if what +## it needs to read is located next to each other on disk. However, as we +## dont enforce order on write operations, this demands that resize/split +## operations do the resorting, which, if we cant do it all in memory is +## very expensive. ANY will not do any reordering. Timestamp will enforce +## timestamp order, which is fairly close to order that will normally be +## written anyway, so it should be cheap to reorder even if we cant do it +## all in memory. This might be useful if you often visit subsets based on +## time. RGID uses reverse GID, which stores data from one location +## together. This is useful if you want to visit data from one user from +## buckets that have many users often. This is a much more expensive sort +## though, and should only be done if we have enough memory. +## DEPRECATED: see stor-memfilepersistence config instead +body_block_order enum { ANY, TIMESTAMP, RGID } default=ANY restart + +## If set to true, we will refuse to do reordering of memory unless we have +## enough memory to do it all in memory. See body_block_order comments. +## DEPRECATED: see stor-memfilepersistence config instead +only_reorder_body_in_memory bool default=true restart + +## Whether or not we should verify checksums of all read data during regular +## operations like put, get & remove. Note that some operations, like merge +## and bucket integrity verify operations will still check checksums even if +## this is set false. +## DEPRECATED: see stor-memfilepersistence config instead +verify_checksum_on_regular_load bool default=true restart + +## For streaming search, visiting is very performance critical. Thus you can +## specifically disable checksum verification for visiting. +## DEPRECATED: see stor-memfilepersistence config instead +verify_checksum_on_visit bool default=true restart + +## Maximum size of index buffer that will be allowed to stay in memory and +## not being reduced back to this size after we no longer need it. +## DEPRECATED: see stor-memfilepersistence config instead +maximum_sustainable_index_buffer_size int default=1044480 restart + +## Maximum size of input buffer that will be allowed to stay in memory and +## not being reduced back to this size after we no longer need it. +## DEPRECATED: see stor-memfilepersistence config instead +maximum_sustainable_input_buffer_size int default=1044480 restart + +## Maximum size of output buffer that will be allowed to stay in memory and +## not being reduced back to this size after we no longer need it. +## DEPRECATED: see stor-memfilepersistence config instead +maximum_sustainable_output_buffer_size int default=1044480 restart + +## Whether to downsize index buffer immediately after usage if its above the +## maximum size. If not, it will not be resized down until someone requests +## to use it that needs less data in the buffer. Index buffer is used all +## the time so there should be little reason for immediately downsizing it. +## DEPRECATED: see stor-memfilepersistence config instead +downsize_index_buffer_immediately_after_use bool default=false restart + +## Whether to downsize input buffer immediately after usage if its above the +## maximum size. If not, it will not be resized down until someone requests +## to use it that needs less data in the buffer. Input buffer is not used +## that often, so downsizing it immediately might save some memory. +## DEPRECATED: see stor-memfilepersistence config instead +downsize_input_buffer_immediately_after_use bool default=true restart + +## Whether to downsize output buffer immediately after usage if its above +## the maximum size. If not, it will not be resized down until someone +## requests to use it that needs less data in the buffer. Input buffer is +## not used that often, so downsizing it immediately might save some memory. +## DEPRECATED: see stor-memfilepersistence config instead +downsize_output_buffer_immediately_after_use bool default=true restart + +## Minimum size of buffer used to write a continuous file. If maximum amount +## of memory is not available. At least this amount will be allocated. +## DEPRECATED: see stor-memfilepersistence config instead +minimum_continuous_file_write_buffer_size int default=1044480 restart + +## Maximum size of buffer used to write a continuous file. If set above max +## file size we will always write new files in one go, which probably makes +## for the least chance of getting fragmentation, but will also consume the +## most memory. Default of writing a MB at a time, should make for little +## performance loss because of disk seek time, and hopefully get little +## fragmentation while keeping memory usage down. +## DEPRECATED: see stor-memfilepersistence config instead +maximum_continuous_file_write_buffer_size int default=1044480 restart + +## Minimum amount of memory allocated to read source data during join. This +## amount of memory will be forced allocated. +## DEPRECATED: see stor-memfilepersistence config instead +minimum_join_source_body_read_buffer_size int default=1044480 restart + +## This sets the maximum size of the buffer used in join to read source +## data. Join uses the least IO if this buffer is as big as the body block +## of the source file. Due to the memory manager, each join might get that +## much memory though. +## DEPRECATED: see stor-memfilepersistence config instead +maximum_join_source_body_read_buffer_size int default=16773120 restart + +## Minimum amount of memory allocated to read source data during export. This +## amount of memory will be forced allocated. +## DEPRECATED: see stor-memfilepersistence config instead +minimum_export_source_body_read_buffer_size int default=1044480 restart + +## This sets the maximum size of the buffer used in export to read source +## data. Export uses the least IO if this buffer is as big as the body block +## of the source file. In addition, reordering of body block might not be +## feasibly unless the buffer is big enough to include the whole body block. +## DEPRECATED: see stor-memfilepersistence config instead +maximum_export_source_body_read_buffer_size int default=33550336 restart + +## Minimum size of buffer used to read data during defragmentation. +## DEPRECATED: see stor-memfilepersistence config instead +minimum_defrag_source_body_read_buffer_size int default=1044480 restart + +## This sets the maximum size of the buffer used in defragmentation to read +## source data. Defragmentation uses the least IO if this buffer is as big +## as the body block of the source file, but this might consume some memory. +## Defragmentation is not enabled by default. +## DEPRECATED: see stor-memfilepersistence config instead +maximum_defrag_source_body_read_buffer_size int default=1044480 restart + +## When merging, if we find more than this number of documents that exist on all +## of the same copies, send a separate apply bucket diff with these entries +## to an optimized merge chain that guarantuees minimum data transfer. +common_merge_chain_optimalization_minimum_size int default=64 restart + +## When merging, it is possible to send more metadata than needed in order to +## let local nodes in merge decide which entries fits best to add this time +## based on disk location. Toggle this option on to use it. Note that memory +## consumption might increase in a 4.1 to 4.2 upgrade due to this, as 4.1 +## dont support to only fill in part of the metadata provided and will always +## fill all. +enable_merge_local_node_choose_docs_optimalization bool default=true restart + +## If set, when we need to cache the entire header, and we already have cached +## all the metadata, read the metadata to find max header position, and only +## read the part containing information. +## DEPRECATED: see stor-memfilepersistence config instead +read_only_used_header_part_when_possible bool default=true restart + +## Enable the slotfile read cache. The cache holds recent metadata and header +## blocks read from disks, and even if small, is very useful for localized +## access. +## DEPRECATED: see stor-memfilepersistence config instead +enable_slotfile_cache bool default=true restart + +## Let the slotfile cache the whole header on a header only operation not +## needing the entire header, if this amount of header only accesses needing +## part of the header has already happened. +## +## Set very high to begin with to never reduce performance. If you have heavy +## header only access to some files, you may get better performance by tuning +## this value. +## DEPRECATED: see stor-memfilepersistence config instead +slotfile_precache_header_after_access_count int default=512 restart + +## Whether or not to enable the multibit split optimalization. This is useful +## if splitting is expensive, but listing document identifiers is fairly cheap. +## This is true for memfile persistence layer, but not for vespa search. +enable_multibit_split_optimalization bool default=true restart + +## STORAGE SPACE vs IO/CPU PERFORMANCE OPTIONS + +## If true, use direct IO, bypassing OS caches for disk access. This is very +## useful as VDS does a random distribution dividing load, so it is unlikely +## that the OS cache will ever hit, and thus it is a huge performance drain. +## DEPRECATED: see stor-memfilepersistence config instead +use_direct_io bool default=true restart + +## All IO operations will be aligned to this amount of bytes if direct IO is +## enabled. +## DEPRECATED: see stor-memfilepersistence config instead +block_alignment_size int default=512 restart + +## When a disk is fuller than this factor, we will not allow more data to be +## written to the system, unless this data is written in order to reduce +## storage consumption, such as resizing files to become smaller or add +## meta entries to write remove entries into. This value is set high as +## default as we expect a lot of users to have formatted their disks to +## already reserve 8% of the data to root user which is often default. We +## suggest using 0% reserved for root, and rather set this parameter lower +## to reserve space. That way, VDS have more space available in worst case +## in order to resize files to become smaller. +## DEPRECATED: see stor-memfilepersistence config instead +disk_full_factor double default=0.98 restart + +## The grow factor sets how much free space to add to a file when we resize +## it, whether we are making it smaller or larger. If the space we need +## after the operation triggering the resize is X, then the file will be +## resized to be of size X * growfactor. In a growing system with no +## deletes, a growfactor of 2 will make the files have 25% free space on +## average. Reducing it to 1.5 will reduce the average free space to 16.7%. +## DEPRECATED: see stor-memfilepersistence config instead +grow_factor double default=2.0 + +## If files fall below this fill rate, resize file to become smaller. +## Note that this parameter is tightly coupled with the growfactor. For +## instance, with a growfactor of 2.0, a file will contain 50 % free space +## after a resize. If the min fill rate then is 0.50, that means that if a +## single doc is deleted from this file, we need to resize it to +## become smaller straight away. +## DEPRECATED: see stor-memfilepersistence config instead +min_fill_rate double default=0.1 + +## Minimum part of defragmented space one need to reclaim to allow +## defragmentation of file. This value is given as a ratio of reclaimed +## space compared to the total size of the data block. +## Example: A body block of 100 MB, has 15 MB free space, with largest +## continuos free space of 5 MB. Gain of defragmentation will then be 0.1. +## DEPRECATED: see stor-memfilepersistence config instead +defrag_minimum_gain double default=1.0 restart + +## When creating/resizing slotfiles, one uses average document sizes to +## decide how much free space to add to metadata, header and body portions. +## This option can be used to allocate extra free space to meta data in +## order to reduce the chance of the file needing resize due to lack of free +## meta data entries. +## DEPRECATED: see stor-memfilepersistence config instead +overrepresent_meta_data_factor double default=1.2 + +## When creating/resizing slotfiles, one uses average document sizes to +## decide how much free space to add to metadata, header and body portions. +## This option can be used to allocate extra free space to header data in +## order to reduce the chance of the file needing resize due to lack of +## header block space. +## DEPRECATED: see stor-memfilepersistence config instead +overrepresent_header_block_factor double default=1.1 + +## Load types to use cache for. If empty, cache for all. +## DEPRECATED: see stor-memfilepersistence config instead +load_types_to_cache[] string restart diff --git a/configdefinitions/src/vespa/stor-memfilepersistence.def b/configdefinitions/src/vespa/stor-memfilepersistence.def new file mode 100644 index 00000000000..ec45788e20a --- /dev/null +++ b/configdefinitions/src/vespa/stor-memfilepersistence.def @@ -0,0 +1,183 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +namespace=vespa.config.storage + +## FILE LAYOUT PARAMETERS + +## Number of directories per level to spread files across +dir_spread int default=256 restart + +## Number of directory levels +dir_levels int default=1 restart + +## FILE SIZE PARAMETERS + +## Minimum number of meta data entries in one slotfile. When creating new +## files or resizing files, enforce it to contain at least this many meta +## entries. Set to 512 by default, using 20544 bytes total for metadata in +## new files. +minimum_file_meta_slots int default=512 + +## Maximum number of entries in one slotfile. File must be split before +## accepting more data +## +## Default ensure meta data is less than 512 kB. +maximum_file_meta_slots int default=13106 + +## Minimum size of header block. At least this amount of header space will +## be available in new or resized files. For 512 documents, this will be +## 200 bytes per document. +minimum_file_header_block_size int default=102848 + +## Maximum header block size (in bytes). Since the whole meta data list and +## header block needs to be read into memory for some operations, a limit +## can be set for the header block, to avoid consuming too much memory. +## +## Default is set high, as we dont configure it automatically right now, so we +## would rather people configure it down than up. +maximum_file_header_block_size int default=33554432 + +## Minimum size of a single slotfile. When creating or resizing files, they +## will never become smaller than this. Default of 1 MB, will be 1807 byte +## per doc if we use all 512 meta data entries set as default min meta +## entries. +minimum_file_size int default=1048576 + +## Maximum size of a single slotfile. File must be split before accepting +## more data. Will return file full errors. +## +## Default is set high, as we dont configure it automatically right now, so we +## would rather people configure it down than up. +maximum_file_size int default=268431360 + +## When creating new files, always create files as a multiplum of this size. +## Should be set to the block size on the disk, or possibly a multiplum of +## that, though we know no advantage of setting it to more than the block +## size. If you want more free space in files, you should rather adjust that +## with growfactor or min file sizes. +file_block_size int default=4096 restart + +## CONSISTENCY PARAMETERS + +## If true, fsync after all disk operations, to ensure no dirty OS file +## cache afterwards. This is only useful if using cached IO, which is not +## recommended to start with. +fsync_after_each_operation bool default=false restart + +## Time period to keep all updates (given in seconds). One can revert any +## operation done within this time. +revert_time_period int default=300 + +## If a remove is older than the reverttimeperiod, the document it is +## removing may be removed from the file. There are a few advantages to +## keeping removes or a bit longer though. If you use this copy to +## synchronize another copy of data, having the remove entry makes it easy +## to detect that you should delete this entry from the other data copy. +## This is useful for internal synchronization of files within VDS if you +## use multiple copies, or for partial recovery or BCP situations. To +## guarantuee consistency in such situations, a data destination that have +## been unavailable for longer than this amount of time, should have its +## data cleared before being set into the system again. This value is given +## in seconds, with the default being one week +keep_remove_time_period int default=604800 + +## PERFORMANCE PARAMETERS + +## Number of threads to use for each mountpoint. VDS needs memory per thread +## to perform disk operations, so increasing this number will increase +## memory usage, but it will also make the disk queue on a given disk be +## able to be larger, such that the disk can choose operations to optimize +## seek time. +## See benchmarks for performance/memory tradeoff. +threads[].lowestpri int default=255 restart + +## Pause operations (and block new ones from starting) with priority +## lower than this value when executing operations with higher pri than +## min_priority_to_be_blocking +max_priority_to_block int default=255 restart +min_priority_to_be_blocking int default=0 restart + +## When reading a slotfile, one does not know the size of the meta data +## list, so one have to read a static amount of data, and possibly read more +## if one didnt read enough. This value needs to be at least 64 byte to read +## the initial header stating the true size of the meta data list. +initial_index_read int default=61440 + +## When we need to read (or write) multiple entries in a file where we can +## either read a big enough section to cover all of them. But at some +## distance between the areas we need, it becomes beneficial to do multiple +## reads rather than to read over them. This setting set how many sequential +## bytes we dont need that we allow to be read/written in order to join two +## logical IO operations together in the application. Setting this low might +## be ok if system calls are cheap and we manage to queue up next IO +## operation in time such that the disk dont need to spin an extra round. +## Setting it high will make the disk more likely to process them together, +## but the time used to read/write the gap might have been used to do +## something useful instead. +maximum_gap_to_read_through int default=65536 + +## Size of the cache. +cache_size long default=1073741824 restart + +## Soft limit on bytes used in cache for meta data, written as a percentage +## of the total +cache_size_meta_percentage long default=33 restart + +## Soft limit on bytes used in cache for meta data, written as a percentage +## of the total +cache_size_header_percentage long default=33 restart + +## Soft limit on bytes used in cache for meta data, written as a percentage +## of the total +cache_size_body_percentage long default=34 restart + +## When a disk is fuller than this factor, we will not allow more data to be +## written to the system, unless this data is written in order to reduce +## storage consumption, such as resizing files to become smaller or add +## meta entries to write remove entries into. This value is set high as +## default as we expect a lot of users to have formatted their disks to +## already reserve 8% of the data to root user which is often default. We +## suggest using 0% reserved for root, and rather set this parameter lower +## to reserve space. That way, VDS have more space available in worst case +## in order to resize files to become smaller. +disk_full_factor double default=0.98 restart + +## We don't allow disk moves to any disk that is fuller than this. This is a +## safeguard to avoid disk movement to fill up a disk. +disk_full_factor_move double default=0.80 restart + +## The grow factor sets how much free space to add to a file when we resize +## it, whether we are making it smaller or larger. If the space we need +## after the operation triggering the resize is X, then the file will be +## resized to be of size X * growfactor. In a growing system with no +## deletes, a growfactor of 2 will make the files have 25% free space on +## average. Reducing it to 1.5 will reduce the average free space to 16.7%. +grow_factor double default=2.0 + +## If files fall below this fill rate, resize file to become smaller. +## Note that this parameter is tightly coupled with the growfactor. For +## instance, with a growfactor of 2.0, a file will contain 50 % free space +## after a resize. If the min fill rate then is 0.50, that means that if a +## single doc is deleted from this file, we need to resize it to +## become smaller straight away. +min_fill_rate double default=0.1 + +## When creating/resizing slotfiles, one uses average document sizes to +## decide how much free space to add to metadata, header and body portions. +## This option can be used to allocate extra free space to meta data in +## order to reduce the chance of the file needing resize due to lack of free +## meta data entries. +overrepresent_meta_data_factor double default=1.2 + +## When creating/resizing slotfiles, one uses average document sizes to +## decide how much free space to add to metadata, header and body portions. +## This option can be used to allocate extra free space to header data in +## order to reduce the chance of the file needing resize due to lack of +## header block space. +overrepresent_header_block_factor double default=1.1 + + +## If non-empty, stores remove entries in such a way that the cluster can +## be downgraded to 5.0 without causing incompatibilities. +## Do not use this unless you know exactly why you are doing it! +store_50_backwards_compatible_remove_entries_with_doctype string default="" + diff --git a/configdefinitions/src/vespa/summary.def b/configdefinitions/src/vespa/summary.def new file mode 100644 index 00000000000..2b53872a1be --- /dev/null +++ b/configdefinitions/src/vespa/summary.def @@ -0,0 +1,9 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +version=2 +namespace=vespa.config.search + +defaultsummaryid int default=-1 +classes[].id int +classes[].name string +classes[].fields[].name string +classes[].fields[].type string diff --git a/configdefinitions/src/vespa/summarymap.def b/configdefinitions/src/vespa/summarymap.def new file mode 100644 index 00000000000..7bb5e7df849 --- /dev/null +++ b/configdefinitions/src/vespa/summarymap.def @@ -0,0 +1,15 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +namespace=vespa.config.search + +## The default output summary class id, -1 to use the stored summary class id +defaultoutputclass int default=-1 + +## The name of the summary field which is overridden with a dynamic value +override[].field string + +## The overriding command, one of: +## staticrank, dynamicteaser, dynamicteasermetric, label, ranklog, empty, copy +override[].command string + +## Space-seaparated arguments, dedendent on the given command +override[].arguments string default="" diff --git a/configdefinitions/src/vespa/upgrading.def b/configdefinitions/src/vespa/upgrading.def new file mode 100644 index 00000000000..9020895695b --- /dev/null +++ b/configdefinitions/src/vespa/upgrading.def @@ -0,0 +1,24 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +namespace=vespa.config.content + +## Config specifying cluster global parameters for a storage cluster. + +## Set to true when there exist nodes of previous major version in cluster. +## This will enable major version backward compability mode in the cluster. +upgrading_major_to bool default=false + +## Set to true when there exist nodes of the next major version in the +## cluster. This will usually not be needed, unless specified in the upgrade +## docs. Sometimes though, it may be easier to let old version be forward +## compatable than having new version backwards compatable. +upgrading_major_from bool default=false + +## Set to true when there exist nodes of previous minor version in cluster. +## This should not be needed, but settings exist in case we end up having +## no other choice than to have a compatability mode between minor versions. +upgrading_minor_to bool default=false + +## Set to true when there exist nodes of next minor version in cluster. +## This should not be needed, but settings exist in case we end up having +## no other choice than to have a compatability mode between minor versions. +upgrading_minor_from bool default=false diff --git a/configdefinitions/src/vespa/ymon.def b/configdefinitions/src/vespa/ymon.def new file mode 100644 index 00000000000..efff7a9cd0b --- /dev/null +++ b/configdefinitions/src/vespa/ymon.def @@ -0,0 +1,6 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +namespace=cloud.config + +services[].servicetype string default="(unknownservicetype)" +services[].ymonname string default="(unknownymonname)" +services[].hosts[] string diff --git a/configdefinitions/src/vespa/zookeeper-server.def b/configdefinitions/src/vespa/zookeeper-server.def new file mode 100644 index 00000000000..b2f697a1488 --- /dev/null +++ b/configdefinitions/src/vespa/zookeeper-server.def @@ -0,0 +1,34 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +namespace=cloud.config + +# Vespa home is prepended if the file is relative +zooKeeperConfigFile string default="conf/zookeeper/zookeeper.cfg" + +tickTime int default=2000 +initLimit int default=20 +syncLimit int default=15 +maxClientConnections int default=0 + +# Vespa home is prepended if the file is relative +dataDir string default="var/zookeeper" + +clientPort int default=2181 + +# In the hosted Vespa prod.us-east-3 zone, a snapshotCount of 50000 corresponds +# to about 5 transaction log snapshots per hour. Assuming this is a fairly +# normal zone, a snapRetainCount of 15 gives 3-4 hours of logs before they're +# purged. +snapshotCount int default=50000 +autopurge.purgeInterval int default=1 +autopurge.snapRetainCount int default=15 + +# Vespa home is prepended if the file is relative +myidFile string default="var/zookeeper/myid" +# Change from default of 1 Mb in zookeeper to 50 Mb +juteMaxBuffer int default=52428800 + +myid int +server[].id int +server[].hostname string +server[].quorumPort int default=2182 +server[].electionPort int default=2183 diff --git a/configdefinitions/src/vespa/zookeepers.def b/configdefinitions/src/vespa/zookeepers.def new file mode 100644 index 00000000000..570e59a779f --- /dev/null +++ b/configdefinitions/src/vespa/zookeepers.def @@ -0,0 +1,3 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +namespace=cloud.config +zookeeperserverlist string |