From cc1811b7eab9fd445683835cbced1ab2d042c6f8 Mon Sep 17 00:00:00 2001 From: Henning Baldersheim Date: Thu, 5 Apr 2018 23:43:56 +0200 Subject: No more engine=vds in config-model --- .../yahoo/vespa/model/content/BucketSplitting.java | 14 +- .../vespa/model/content/DistributorCluster.java | 3 +- .../model/content/cluster/ContentCluster.java | 9 - .../content/cluster/EngineFactoryBuilder.java | 4 +- .../vespa/model/content/engines/VDSEngine.java | 83 -------- .../storagecluster/IntegrityCheckerProducer.java | 38 ---- config-model/src/main/javacc/SDParser.jj | 4 +- config-model/src/main/resources/schema/content.rnc | 13 +- .../src/main/resources/schema/services.rnc | 1 - config-model/src/main/resources/schema/storage.rnc | 237 --------------------- 10 files changed, 18 insertions(+), 388 deletions(-) delete mode 100644 config-model/src/main/java/com/yahoo/vespa/model/content/engines/VDSEngine.java delete mode 100644 config-model/src/main/resources/schema/storage.rnc (limited to 'config-model/src/main') diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/BucketSplitting.java b/config-model/src/main/java/com/yahoo/vespa/model/content/BucketSplitting.java index 7f720672ef9..9f7494f470d 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/BucketSplitting.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/BucketSplitting.java @@ -12,13 +12,12 @@ public class BucketSplitting implements StorDistributormanagerConfig.Producer { Integer maxDocuments; Integer splitSize; Integer minSplitCount; - boolean useInlineBucketSplitting; public static class Builder { - public BucketSplitting build(ContentCluster cluster, ModelElement clusterElem) { + public BucketSplitting build(ModelElement clusterElem) { ModelElement tuning = clusterElem.getChild("tuning"); if (tuning == null) { - return new BucketSplitting(cluster.isMemfilePersistence(), null, null, null); + return new BucketSplitting(null, null, null); } ModelElement bucketSplitting = tuning.getChild("bucket-splitting"); @@ -27,18 +26,17 @@ public class BucketSplitting implements StorDistributormanagerConfig.Producer { Integer splitSize = bucketSplitting.getIntegerAttribute("max-size"); Integer minSplitCount = bucketSplitting.getIntegerAttribute("minimum-bits"); - return new BucketSplitting(cluster.isMemfilePersistence(), maxDocuments, splitSize, minSplitCount); + return new BucketSplitting(maxDocuments, splitSize, minSplitCount); } - return new BucketSplitting(cluster.isMemfilePersistence(), null, null, null); + return new BucketSplitting(null, null, null); } } - public BucketSplitting(boolean useInlineBucketSplitting, Integer maxDocuments, Integer splitSize, Integer minSplitCount) { + public BucketSplitting(Integer maxDocuments, Integer splitSize, Integer minSplitCount) { this.maxDocuments = maxDocuments; this.splitSize = splitSize; this.minSplitCount = minSplitCount; - this.useInlineBucketSplitting = useInlineBucketSplitting; } @Override @@ -55,6 +53,6 @@ public class BucketSplitting implements StorDistributormanagerConfig.Producer { builder.minsplitcount(minSplitCount); } - builder.inlinebucketsplitting(useInlineBucketSplitting); + builder.inlinebucketsplitting(false); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/DistributorCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/DistributorCluster.java index 0405f96cd89..e60aabd24e8 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/DistributorCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/DistributorCluster.java @@ -100,8 +100,7 @@ public class DistributorCluster extends AbstractConfigProducer impl final boolean hasIndexedDocumentType = clusterContainsIndexedDocumentType(documentsNode); return new DistributorCluster(parent, - new BucketSplitting.Builder().build( - parent, new ModelElement(producerSpec)), gc, hasIndexedDocumentType); + new BucketSplitting.Builder().build(new ModelElement(producerSpec)), gc, hasIndexedDocumentType); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java index fabd7a0cc1a..9378c8c27f9 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java @@ -38,7 +38,6 @@ import com.yahoo.vespa.model.container.xml.ContainerModelBuilder; import com.yahoo.vespa.model.content.*; import com.yahoo.vespa.model.content.engines.PersistenceEngine; import com.yahoo.vespa.model.content.engines.ProtonEngine; -import com.yahoo.vespa.model.content.engines.VDSEngine; import com.yahoo.vespa.model.content.storagecluster.StorageCluster; import com.yahoo.vespa.model.search.IndexedSearchCluster; import com.yahoo.vespa.model.search.MultilevelDispatchValidator; @@ -134,10 +133,6 @@ public class ContentCluster extends AbstractConfigProducer implements StorDistri throw new RuntimeException("If you have indexed search you need to have proton as engine"); } - if (c.isMemfilePersistence()) { - admin.deployLogger().log(Level.WARNING, "'vds' engine is deprecated and will soon be removed. 'proton' is only recommended engine."); - } - if (documentsElement != null) { ModelElement e = documentsElement.getChild("document-processing"); if (e != null) { @@ -515,10 +510,6 @@ public class ContentCluster extends AbstractConfigProducer implements StorDistri return getPersistence().getDefaultDistributionMode(); } - public boolean isMemfilePersistence() { - return persistenceFactory instanceof VDSEngine.Factory; - } - public static String getClusterName(ModelElement clusterElem) { String clusterName = clusterElem.getStringAttribute("id"); if (clusterName == null) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/EngineFactoryBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/EngineFactoryBuilder.java index 04c5fd4fd72..f8961d81bf4 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/EngineFactoryBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/EngineFactoryBuilder.java @@ -16,9 +16,7 @@ public class EngineFactoryBuilder { } ModelElement e; - if ((e = persistence.getChild("vds")) != null) { - return new VDSEngine.Factory(e); - } else if (persistence.getChild("proton") != null) { + if (persistence.getChild("proton") != null) { return new ProtonEngine.Factory(c.getSearch()); } else if (persistence.getChild("dummy") != null) { return new com.yahoo.vespa.model.content.engines.DummyPersistence.Factory(); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/engines/VDSEngine.java b/config-model/src/main/java/com/yahoo/vespa/model/content/engines/VDSEngine.java deleted file mode 100644 index c3720d52511..00000000000 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/engines/VDSEngine.java +++ /dev/null @@ -1,83 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.model.content.engines; - -import com.yahoo.vespa.config.storage.StorMemfilepersistenceConfig; -import com.yahoo.vespa.config.content.core.StorServerConfig; -import com.yahoo.vespa.model.builder.xml.dom.ModelElement; -import com.yahoo.vespa.model.content.StorageGroup; -import com.yahoo.vespa.model.content.StorageNode; -import com.yahoo.vespa.model.content.cluster.ContentCluster; - -/** - * Configuration class to generate config for the memfile engines provider. - */ -public class VDSEngine extends PersistenceEngine - implements StorMemfilepersistenceConfig.Producer -{ - ModelElement tuning; - - public VDSEngine(StorageNode parent, ModelElement vdsConfig) { - super(parent, "provider"); - - if (vdsConfig != null) { - this.tuning = vdsConfig.getChild("tuning"); - } - - if (parent != null) { - parent.useVdsEngine(); - } - } - - @Override - public void getConfig(StorMemfilepersistenceConfig.Builder builder) { - if (tuning == null) { - return; - } - - ModelElement diskFullRatio = tuning.getChild("disk-full-ratio"); - if (diskFullRatio != null) { - builder.disk_full_factor(diskFullRatio.asDouble()); - } - - ModelElement cacheSize = tuning.getChild("cache-size"); - if (cacheSize != null) { - builder.cache_size(cacheSize.asLong()); - } - } - - @Override - public void getConfig(StorServerConfig.Builder builder) { - builder.persistence_provider( - new StorServerConfig.Persistence_provider.Builder().type( - StorServerConfig.Persistence_provider.Type.Enum.STORAGE) - ); - } - - public static class Factory implements PersistenceFactory { - ModelElement vdsConfig; - - public Factory(ModelElement vdsConfig) { - this.vdsConfig = vdsConfig; - } - - @Override - public PersistenceEngine create(StorageNode storageNode, StorageGroup parentGroup, ModelElement storageNodeElement) { - return new VDSEngine(storageNode, vdsConfig); - } - - @Override - public boolean supportRevert() { - return true; - } - - @Override - public boolean enableMultiLevelSplitting() { - return true; - } - - @Override - public ContentCluster.DistributionMode getDefaultDistributionMode() { - return ContentCluster.DistributionMode.STRICT; - } - } -} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/IntegrityCheckerProducer.java b/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/IntegrityCheckerProducer.java index 7fa4211ade8..46d621981d4 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/IntegrityCheckerProducer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/IntegrityCheckerProducer.java @@ -13,45 +13,7 @@ public class IntegrityCheckerProducer implements StorIntegritycheckerConfig.Prod public static class Builder { protected IntegrityCheckerProducer build(ContentCluster cluster, ModelElement clusterElem) { - if (!cluster.isMemfilePersistence()) { return integrityCheckerDisabled(); - } - - ModelElement tuning = clusterElem.getChild("tuning"); - - if (tuning == null) { - return new IntegrityCheckerProducer(); - } - - ModelElement maintenance = tuning.getChild("maintenance"); - if (maintenance == null) { - return new IntegrityCheckerProducer(); - } - - Integer startTime = null; - Integer stopTime = null; - String weeklyCycle = null; - - String start = maintenance.getStringAttribute("start"); - if (start != null) { - startTime = ConfigModelUtils.getTimeOfDay(start); - } - - String stop = maintenance.getStringAttribute("stop"); - if (stop != null) { - stopTime = ConfigModelUtils.getTimeOfDay(stop); - } - - String high = maintenance.getStringAttribute("high"); - - if (high != null) { - int weekday = ConfigModelUtils.getDayOfWeek(high); - char[] weeklycycle = "rrrrrrr".toCharArray(); - weeklycycle[weekday] = 'R'; - weeklyCycle = String.valueOf(weeklycycle); - } - - return new IntegrityCheckerProducer(startTime, stopTime, weeklyCycle); } } diff --git a/config-model/src/main/javacc/SDParser.jj b/config-model/src/main/javacc/SDParser.jj index 86b136a1dd2..0aff59d459f 100644 --- a/config-model/src/main/javacc/SDParser.jj +++ b/config-model/src/main/javacc/SDParser.jj @@ -1501,7 +1501,7 @@ void body(SDField field) : { } { { - deployLogger.log(Level.WARNING, field + ": 'header/body' is deprecated and has no effect unless using deprecated vds engine."); + deployLogger.log(Level.WARNING, field + ": 'header/body' is deprecated and has no effect."); field.setHeader(false); field.setHeaderOrBodyDefined(true); } @@ -1516,7 +1516,7 @@ void header(SDField field) : { } {
{ - deployLogger.log(Level.WARNING, field + ": 'header/body' is deprecated and has no effect unless using deprecated vds engine."); + deployLogger.log(Level.WARNING, field + ": 'header/body' is deprecated and has no effect."); field.setHeader(true); field.setHeaderOrBodyDefined(true); } diff --git a/config-model/src/main/resources/schema/content.rnc b/config-model/src/main/resources/schema/content.rnc index 8121a85c056..58d22ea9b6f 100644 --- a/config-model/src/main/resources/schema/content.rnc +++ b/config-model/src/main/resources/schema/content.rnc @@ -157,11 +157,7 @@ DocumentProcessing = element document-processing { ## Config for the persistence providers. Engine = element engine { - (Vds | Proton | Dummy) -} - -Vds = element vds { - text + (Proton | Dummy) } Proton = element proton { @@ -375,3 +371,10 @@ TuningCompression = element compression { Experimental = element experimental { element enable-multiple-bucket-spaces { xsd:boolean }? } + +Thread = element thread { + ## The lowest priority this thread should handle. + attribute lowest-priority { xsd:string}? & + ## The number of threads of this type to create + attribute count { xsd:integer }? +} diff --git a/config-model/src/main/resources/schema/services.rnc b/config-model/src/main/resources/schema/services.rnc index c728dd017c8..3897448cb56 100644 --- a/config-model/src/main/resources/schema/services.rnc +++ b/config-model/src/main/resources/schema/services.rnc @@ -4,7 +4,6 @@ include "admin.rnc" include "clients.rnc" include "content.rnc" include "docproc.rnc" -include "storage.rnc" include "routing.rnc" include "containercluster.rnc" include "genericcluster.rnc" diff --git a/config-model/src/main/resources/schema/storage.rnc b/config-model/src/main/resources/schema/storage.rnc deleted file mode 100644 index 24a00cb9ecf..00000000000 --- a/config-model/src/main/resources/schema/storage.rnc +++ /dev/null @@ -1,237 +0,0 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -namespace a = "http://relaxng.org/ns/compatibility/annotations/1.0" - -Storage = Storage30 - -# Specific v2.0 definitions - -Tuning20 = element tuning { - element cache { xsd:nonNegativeInteger }? & - element memorytouse { xsd:nonNegativeInteger }? & - element avgdiskspernode { xsd:nonNegativeInteger }? & - element avgmegabytesperdisk { xsd:nonNegativeInteger }? & - element avgdocsize { xsd:nonNegativeInteger }? -} - -Datadistribution = element datadistribution { - attribute disk { text }? & - element diskbalancer { - attribute start { xsd:string { pattern = "[0-9]{2}:[0-9]{2}" } } ? & - attribute stop { xsd:string { pattern = "[0-9]{2}:[0-9]{2}" } } ? & - attribute cycletime { xsd:string { pattern = "[0-9]{2}:[0-9]{2}" } } ? & - attribute targetskewpercent { xsd:double { minInclusive = "0" maxInclusive = "100"} }? & - attribute max-target-fill-rate-above-average { xsd:double { minInclusive = "-1" maxInclusive = "1"} }? & - attribute stopontarget { xsd:boolean }? - } ? -} - -OperationsLog20 = -## File name of file to log operations to. -[ a:defaultValue = "false" ] -element operationslog { xsd:boolean } - -Priority = element priority { - attribute from { xsd:string} & - attribute to { xsd:integer { minInclusive = "0" maxInclusive = "255" } } -} - -PriorityMapping = element prioritymapping { - Priority+ -} - -Thread = element thread { - ## The lowest priority this thread should handle. - attribute lowest-priority { xsd:string}? & - ## The number of threads of this type to create - attribute count { xsd:integer }? -} - -## Declare which storage threads each disk should have. -Threads = element diskthreads { - ## Operations with priority worse than this can be blocked - attribute highest-priority-to-block { xsd:string } ? & - ## Operations with priority better than this can block others - attribute lowest-priority-to-block-others { xsd:string } ? & - Thread+ -} - -Visitors = element visitors { - attribute threads { xsd:positiveInteger }? & - attribute maxqueuesize { xsd:positiveInteger }? & - element maxconcurrent { - attribute fixed { xsd:positiveInteger } & - attribute variable { xsd:nonNegativeInteger }? - }? -} - -ClusterParamsV30 = - ## Percentage of disk space storage will use storing documents. (Remaining used for resizing) - element diskfullpercentage { xsd:nonNegativeInteger { minInclusive = "1" maxInclusive = "99" } }? & - ## Whether to restart storage marking failed disks down - element faildiskonerror { xsd:boolean }? & - ## Time in seconds to keep all operations in case we need to revert them - element reverttimeperiod { xsd:nonNegativeInteger }? & - ## Time in seconds to keep remove statements for search/VDS recovery - element recoveryperiod { xsd:nonNegativeInteger }? & - ## Number of bits used for initial splitting. - element staticbucketbits { xsd:integer { minInclusive = "16" maxInclusive = "58" } }? & - ## Number of bytes per storage file. - element bucketsplitsize { xsd:positiveInteger }? & - ## Number of documents per storage file. - element bucketsplitcount { xsd:positiveInteger }? & - PriorityMapping? & - element deviceparams { - ## Timeout in seconds of a disk operation where storage assumes IO error (0 = never = default) - attribute diskoperationtimeout { xsd:nonNegativeInteger }? & - ## Not use O_DIRECT when reading from files - element cachedio { - ## Whether to sync after each VDS operation, default false - attribute fsync { xsd:boolean }? - }? & - ## Disk threads in storage - Threads? - }? & - GarbageCollect? & - DeadLockDetector? & - Visitors? - -FleetControllers = element fleetcontrollers { - attribute jvmargs { text }? & - FleetControllerStats.attlist & - FleetController20+ -} - -FleetController20 = element fleetcontroller { - service.attlist & - attribute index { xsd:nonNegativeInteger } -} - -# Common definitions for v2.0 and v3.0 - -StorageMetrics = element metrics { element log { text }* } - -Disk = element disk { - attribute index { xsd:nonNegativeInteger } & - attribute capacity { xsd:double { minExclusive = "0.0" } }? -} - -GarbageCollect = element garbagecollect { - attribute start { xsd:string { pattern = "[0-9]{2}:[0-9]{2}" } }, - attribute stop { xsd:string { pattern = "[0-9]{2}:[0-9]{2}" } }, - attribute full { string "monday" | string "tuesday" | string "wednesday" | - string "thursday" | string "friday" | string "saturday" | string "sunday" } -} - -DeadLockDetector = element deadlockdetector { - attribute enable { xsd:boolean } ? & - attribute slack-time-period { xsd:double { minInclusive = "0.0" } }? -} - -FleetController = element fleetcontroller { - service.attlist & - FleetControllerStats.attlist & - attribute index { xsd:nonNegativeInteger } ? -} - -FleetControllerStats.attlist = - attribute initprogresstime { xsd:double { minExclusive = "0.0" } } ? & - attribute transitiontime { xsd:double { minInclusive = "0.0" } } ? & - attribute mintimebeforesettingoutofreachnodedown { xsd:double { minExclusive = "0.0" } } ? & - attribute maxprematurecrashes { xsd:nonNegativeInteger } ? & - attribute stablestateperiod { xsd:double { minExclusive = "0.0" } } ? & - attribute mindistributorupcount { xsd:nonNegativeInteger } ? & - attribute mindistributorupratio { - xsd:double { minInclusive = "0.0" maxInclusive = "1.0" } } ? & - attribute minstorageupcount { xsd:nonNegativeInteger } ? & - attribute minstorageupratio { - xsd:double { minInclusive = "0.0" maxInclusive = "1.0" } } ? & - attribute mintimebetweenclusterstates { - xsd:double { minInclusive = "0.0" } } ? - -# v3.0 definitions -Storage30 = element storage { - attribute version { string "3.0" } & - GenericConfig* & - element cluster { - # name defaults to 'storage' - attribute name { xsd:NCName { minLength = "1" } }? & - ## Redundancy level - how many copies of each document are made. - attribute redundancy { xsd:positiveInteger }? & - attribute initial-redundancy { xsd:positiveInteger }? & - attribute ensure-primary-persisted { xsd:boolean }? & - attribute distributionbits { xsd:positiveInteger } ? & - attribute distributorbaseport { xsd:unsignedShort }? & - element forwarding { xsd:string }? & - GenericConfig* & - ( FleetController | FleetControllers ) & - ClusterParamsV30? & - StorageMetrics? & - OperationsLog20? & - Tuning20? & - Merges? & - Datadistribution? & - TopStorageGroup - }+ -} - -TopStorageGroup = element group { - attribute index { xsd:nonNegativeInteger }? & - attribute name { xsd:string }? & - element disks { - attribute num { xsd:nonNegativeInteger } & - attribute size { xsd:string } & - attribute iomodel {xsd:string } & - element disk { - attribute index { xsd:nonNegativeInteger } & - attribute size { xsd:string } - } * - } ? & - ( - ( - element distribution { - attribute partitions { xsd:string } - } & - StorageGroup+ - ) | - element node { - service.attlist & - attribute distributorbaseport { xsd:unsignedShort }? & - attribute index { xsd:nonNegativeInteger } & - attribute capacity { xsd:double { minExclusive = "0.0" } }? - } + - ) - } - -StorageGroup = element group { - attribute index { xsd:nonNegativeInteger } & - attribute name { xsd:string } & - element disks { - attribute num { xsd:nonNegativeInteger } & - attribute size { xsd:string } & - attribute iomodel {xsd:string } & - element disk { - attribute index { xsd:nonNegativeInteger } & - attribute size { xsd:string } - } * - } ? & - ( - ( - element distribution { - attribute partitions { xsd:string } - } & - StorageGroup+ - ) | - element node { - service.attlist & - attribute distributorbaseport { xsd:unsignedShort }? & - attribute index { xsd:nonNegativeInteger } & - attribute capacity { xsd:double { minExclusive = "0.0" } }? - } + - ) - } - -Merges = element merges { - attribute maxpernode { xsd:positiveInteger }? & - attribute maxqueuesize { xsd:nonNegativeInteger }? -} -- cgit v1.2.3