diff options
Diffstat (limited to 'config-model')
16 files changed, 378 insertions, 124 deletions
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java index 40c4089a796..b7e6b8080a7 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java @@ -309,10 +309,6 @@ public class VespaMetricSet { metrics.add(new Metric("vds.filestor.alldisks.queuesize.average","diskqueuesize")); metrics.add(new Metric("vds.filestor.alldisks.averagequeuewait.sum.average","diskqueuewait")); - metrics.add(new Metric("vds.memfilepersistence.cache.files.average")); - metrics.add(new Metric("vds.memfilepersistence.cache.body.average")); - metrics.add(new Metric("vds.memfilepersistence.cache.header.average")); - metrics.add(new Metric("vds.memfilepersistence.cache.meta.average")); metrics.add(new Metric("vds.visitor.allthreads.queuesize.count.average")); metrics.add(new Metric("vds.visitor.allthreads.completed.sum.average")); metrics.add(new Metric("vds.visitor.allthreads.created.sum.rate","visit")); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java index 2a208e45732..f390d6368d1 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java @@ -15,6 +15,7 @@ import com.yahoo.vespa.model.application.validation.change.ContentClusterRemoval import com.yahoo.vespa.model.application.validation.change.IndexedSearchClusterChangeValidator; import com.yahoo.vespa.model.application.validation.change.IndexingModeChangeValidator; import com.yahoo.vespa.model.application.validation.change.StartupCommandChangeValidator; +import com.yahoo.vespa.model.application.validation.change.StreamingSearchClusterChangeValidator; import com.yahoo.vespa.model.application.validation.first.AccessControlValidator; import java.time.Instant; @@ -70,6 +71,7 @@ public class Validation { ChangeValidator[] validators = new ChangeValidator[] { new IndexingModeChangeValidator(), new IndexedSearchClusterChangeValidator(), + new StreamingSearchClusterChangeValidator(), new ConfigValueChangeValidator(logger), new StartupCommandChangeValidator(), new ContentClusterRemovalValidator(), diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/StreamingSearchClusterChangeValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/StreamingSearchClusterChangeValidator.java new file mode 100644 index 00000000000..a96275010bf --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/StreamingSearchClusterChangeValidator.java @@ -0,0 +1,124 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.application.validation.change; + +import com.yahoo.config.application.api.ValidationOverrides; +import com.yahoo.config.model.api.ConfigChangeAction; +import com.yahoo.config.model.api.ServiceInfo; +import com.yahoo.documentmodel.NewDocumentType; +import com.yahoo.searchdefinition.derived.AttributeFields; +import com.yahoo.searchdefinition.document.Attribute; +import com.yahoo.vespa.model.VespaModel; +import com.yahoo.vespa.model.application.validation.change.search.ChangeMessageBuilder; +import com.yahoo.vespa.model.application.validation.change.search.DocumentTypeChangeValidator; +import com.yahoo.vespa.model.content.cluster.ContentCluster; +import com.yahoo.vespa.model.search.StreamingSearchCluster; + +import java.time.Instant; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import java.util.stream.Collectors; + +/** + * Validates the changes between all current and next streaming search clusters in a Vespa model. + * + * @author geirst + */ +public class StreamingSearchClusterChangeValidator implements ChangeValidator { + + @Override + public List<ConfigChangeAction> validate(VespaModel current, VespaModel next, ValidationOverrides overrides, Instant now) { + List<ConfigChangeAction> result = new ArrayList<>(); + current.getContentClusters().forEach((clusterName, currentCluster) -> { + ContentCluster nextCluster = next.getContentClusters().get(clusterName); + if (nextCluster != null) { + List<StreamingSearchCluster> nextStreamingClusters = nextCluster.getSearch().getStreamingClusters(); + currentCluster.getSearch().getStreamingClusters().forEach(currentStreamingCluster -> { + Optional<StreamingSearchCluster> nextStreamingCluster = findStreamingCluster(currentStreamingCluster.getClusterName(), nextStreamingClusters); + if (nextStreamingCluster.isPresent()) { + result.addAll(validateStreamingCluster(currentCluster, currentStreamingCluster, + nextCluster, nextStreamingCluster.get(), overrides, now)); + } + }); + } + }); + return result; + } + + private static Optional<StreamingSearchCluster> findStreamingCluster(String clusterName, List<StreamingSearchCluster> clusters) { + return clusters.stream() + .filter(cluster -> cluster.getClusterName().equals(clusterName)) + .findFirst(); + } + + private static List<ConfigChangeAction> validateStreamingCluster(ContentCluster currentCluster, + StreamingSearchCluster currentStreamingCluster, + ContentCluster nextCluster, + StreamingSearchCluster nextStreamingCluster, + ValidationOverrides overrides, + Instant now) { + List<VespaConfigChangeAction> result = new ArrayList<>(); + + result.addAll(validateDocumentTypeChanges(getDocumentType(currentCluster, currentStreamingCluster), + getDocumentType(nextCluster, nextStreamingCluster), overrides, now)); + result.addAll(validateAttributeFastAccessAdded(currentStreamingCluster.getSdConfig().getAttributeFields(), + nextStreamingCluster.getSdConfig().getAttributeFields())); + result.addAll(validateAttributeFastAccessRemoved(currentStreamingCluster.getSdConfig().getAttributeFields(), + nextStreamingCluster.getSdConfig().getAttributeFields())); + + return modifyActions(result, getSearchNodeServices(nextCluster), nextStreamingCluster.getDocTypeName()); + } + + private static List<VespaConfigChangeAction> validateDocumentTypeChanges(NewDocumentType currentDocType, + NewDocumentType nextDocType, + ValidationOverrides overrides, + Instant now) { + return new DocumentTypeChangeValidator(currentDocType, nextDocType).validate(overrides, now); + } + + private static NewDocumentType getDocumentType(ContentCluster cluster, StreamingSearchCluster streamingCluster) { + return cluster.getDocumentDefinitions().get(streamingCluster.getDocTypeName()); + } + + private static List<VespaConfigChangeAction> validateAttributeFastAccessAdded(AttributeFields currentAttributes, + AttributeFields nextAttributes) { + return validateAttributeFastAccessChanged(nextAttributes, currentAttributes, "add"); + } + + private static List<VespaConfigChangeAction> validateAttributeFastAccessRemoved(AttributeFields currentAttributes, + AttributeFields nextAttributes) { + return validateAttributeFastAccessChanged(currentAttributes, nextAttributes, "remove"); + } + + private static List<VespaConfigChangeAction> validateAttributeFastAccessChanged(AttributeFields lhsAttributes, + AttributeFields rhsAttributes, + String change) { + return lhsAttributes.attributes().stream() + .filter(attr -> attr.isFastAccess() && + !hasFastAccessAttribute(attr.getName(), rhsAttributes)) + .map(attr -> new VespaRestartAction(new ChangeMessageBuilder(attr.getName()) + .addChange(change + " fast-access attribute").build())) + .collect(Collectors.toList()); + } + + private static boolean hasFastAccessAttribute(String attrName, AttributeFields attributes) { + Attribute attr = attributes.getAttribute(attrName); + return (attr != null && attr.isFastAccess()); + } + + private static List<ServiceInfo> getSearchNodeServices(ContentCluster cluster) { + return cluster.getSearch().getSearchNodes().stream() + .map(node -> node.getServiceInfo()) + .collect(Collectors.toList()); + } + + private static List<ConfigChangeAction> modifyActions(List<VespaConfigChangeAction> result, + List<ServiceInfo> services, + String docTypeName) { + return result.stream() + .map(action -> action.modifyAction("Document type '" + docTypeName + "': " + action.getMessage(), + services, docTypeName)) + .collect(Collectors.toList()); + } + +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java index c4cfd6d185b..cb306c8cae2 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java @@ -17,6 +17,7 @@ import org.w3c.dom.Element; import java.util.*; +import java.util.stream.Collectors; /** * Encapsulates the various options for search in a content model. @@ -272,6 +273,13 @@ public class ContentSearchCluster extends AbstractConfigProducer implements Prot .findFirst(); } + public List<StreamingSearchCluster> getStreamingClusters() { + return getClusters().values().stream() + .filter(StreamingSearchCluster.class::isInstance) + .map(StreamingSearchCluster.class::cast) + .collect(Collectors.toList()); + } + @Override public void getConfig(ProtonConfig.Builder builder) { double visibilityDelay = hasIndexedCluster() ? getIndexed().getVisibilityDelay() : 0.0; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/StorageNode.java b/config-model/src/main/java/com/yahoo/vespa/model/content/StorageNode.java index c8eceacc36d..74576b45584 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/StorageNode.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/StorageNode.java @@ -3,10 +3,8 @@ package com.yahoo.vespa.model.content; import com.yahoo.vespa.config.content.StorFilestorConfig; import com.yahoo.vespa.config.content.core.StorBucketmoverConfig; -import com.yahoo.vespa.config.storage.StorDevicesConfig; import com.yahoo.vespa.config.content.core.StorServerConfig; import com.yahoo.config.model.producer.AbstractConfigProducer; -import com.yahoo.vespa.config.storage.StorMemfilepersistenceConfig; import com.yahoo.vespa.defaults.Defaults; import com.yahoo.vespa.model.application.validation.RestartConfigs; import com.yahoo.vespa.model.builder.xml.dom.ModelElement; @@ -16,21 +14,16 @@ import com.yahoo.vespa.model.content.engines.ProtonProvider; import com.yahoo.vespa.model.content.storagecluster.StorageCluster; import org.w3c.dom.Element; -import java.util.Arrays; - /** * Class to provide config related to a specific storage node. */ -@RestartConfigs({StorDevicesConfig.class, StorFilestorConfig.class, - StorMemfilepersistenceConfig.class, StorBucketmoverConfig.class}) -public class StorageNode extends ContentNode implements StorServerConfig.Producer, StorDevicesConfig.Producer { +@RestartConfigs({StorFilestorConfig.class, StorBucketmoverConfig.class}) +public class StorageNode extends ContentNode implements StorServerConfig.Producer { - static final String rootFolder = Defaults.getDefaults().underVespaHome("var/db/vespa/vds/"); + static final String rootFolder = Defaults.getDefaults().underVespaHome("var/db/vespa/search/"); private final Double capacity; private final boolean retired; - private final boolean isHostedVespa; - private boolean usesVdsEngine = false; public static class Builder extends VespaDomBuilder.DomConfigProducerBuilder<StorageNode> { @Override @@ -47,7 +40,6 @@ public class StorageNode extends ContentNode implements StorServerConfig.Produce distributionKey); this.retired = retired; this.capacity = capacity; - this.isHostedVespa = cluster.getRoot().getDeployState().getProperties().hostedVespa(); } @Override @@ -57,31 +49,6 @@ public class StorageNode extends ContentNode implements StorServerConfig.Produce : "exec sbin/vespa-storaged -c $VESPA_CONFIG_ID"; } - @Override - public void getConfig(StorDevicesConfig.Builder builder) { - String root_folder = getRootDirectory(); - builder.root_folder(root_folder); - - // For VDS in hosted Vespa, we default to using the root_folder as the disk to store the data in. - // Setting disk_path will then - if (isHostedVespa && usesVdsEngine) { - // VDS looks up the first disk at the directory path root_folder/disks/d0. - builder.disk_path(Arrays.asList(root_folder + "/disks/d0")); - } - } - - // 2015-08-11: Needed because of the following circular dependency: - // 1. StorageNode is created. - // 2. A particular persistence engine is picked depending on things (like the presence of engine/vds element) - // that are hidden from the code creating the StorageNode in (1). - // 3. The persistence engine depends on the StorageNode, e.g. it's a parent node. - // - // If the VDSEngine is picked in (2), we would like to know this in StorageNode::getConfig(). Hence this setter. - public void useVdsEngine() { - usesVdsEngine = true; - } - - public double getCapacity() { if (capacity != null) { return capacity; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/StorageCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/StorageCluster.java index 0fdafa1ddf6..10f39d1f499 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/StorageCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/StorageCluster.java @@ -2,7 +2,6 @@ package com.yahoo.vespa.model.content.storagecluster; import com.yahoo.vespa.config.content.core.StorIntegritycheckerConfig; -import com.yahoo.vespa.config.storage.StorMemfilepersistenceConfig; import com.yahoo.vespa.config.content.core.StorBucketmoverConfig; import com.yahoo.vespa.config.content.core.StorVisitorConfig; import com.yahoo.vespa.config.content.StorFilestorConfig; @@ -22,7 +21,6 @@ import org.w3c.dom.Element; public class StorageCluster extends AbstractConfigProducer<StorageNode> implements StorServerConfig.Producer, StorBucketmoverConfig.Producer, - StorMemfilepersistenceConfig.Producer, StorIntegritycheckerConfig.Producer, StorFilestorConfig.Producer, StorVisitorConfig.Producer, @@ -46,9 +44,6 @@ public class StorageCluster extends AbstractConfigProducer<StorageNode> } private Integer bucketMoverMaxFillAboveAverage = null; - private Long cacheSize = null; - private Double diskFullPercentage = null; - private String clusterName; private FileStorProducer fileStorProducer; private IntegrityCheckerProducer integrityCheckerProducer; @@ -80,18 +75,6 @@ public class StorageCluster extends AbstractConfigProducer<StorageNode> } @Override - public void getConfig(StorMemfilepersistenceConfig.Builder builder) { - if (cacheSize != null) { - builder.cache_size(cacheSize); - } - - if (diskFullPercentage != null) { - builder.disk_full_factor(diskFullPercentage / 100.0); - builder.disk_full_factor_move(diskFullPercentage / 100.0 * 0.9); - } - } - - @Override public void getConfig(MetricsmanagerConfig.Builder builder) { ContentCluster.getMetricBuilder("fleetcontroller", builder). addedmetrics("vds.datastored.alldisks.docs"). diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/SearchNode.java b/config-model/src/main/java/com/yahoo/vespa/model/search/SearchNode.java index 924c282d2e5..547ff354b41 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/SearchNode.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/SearchNode.java @@ -13,7 +13,6 @@ import com.yahoo.vespa.config.content.core.StorCommunicationmanagerConfig; import com.yahoo.vespa.config.content.core.StorServerConfig; import com.yahoo.vespa.config.content.core.StorStatusConfig; import com.yahoo.vespa.config.search.core.ProtonConfig; -import com.yahoo.vespa.config.storage.StorDevicesConfig; import static com.yahoo.vespa.defaults.Defaults.getDefaults; import com.yahoo.vespa.model.AbstractService; import com.yahoo.vespa.model.admin.monitoring.Monitoring; @@ -39,7 +38,7 @@ import java.util.Optional; * @author hmusum */ @RestartConfigs({ProtonConfig.class, MetricsmanagerConfig.class, TranslogserverConfig.class, - StorDevicesConfig.class, StorFilestorConfig.class, StorBucketmoverConfig.class, + StorFilestorConfig.class, StorBucketmoverConfig.class, StorCommunicationmanagerConfig.class, StorStatusConfig.class, StorServerConfig.class, LoadTypeConfig.class}) public class SearchNode extends AbstractService implements diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/StreamingSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/search/StreamingSearchCluster.java index 85f98c27365..e87df3d530b 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/StreamingSearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/StreamingSearchCluster.java @@ -45,11 +45,13 @@ public class StreamingSearchCluster extends SearchCluster implements private final String storageRouteSpec; private final AttributesProducer attributesConfig; + private final String docTypeName; private DerivedConfiguration sdConfig = null; public StreamingSearchCluster(AbstractConfigProducer parent, String clusterName, int index, String docTypeName, String storageRouteSpec) { super(parent, clusterName, index); attributesConfig = new AttributesProducer(parent, docTypeName); + this.docTypeName = docTypeName; this.storageRouteSpec = storageRouteSpec; } @@ -59,6 +61,11 @@ public class StreamingSearchCluster extends SearchCluster implements @Override protected IndexingMode getIndexingMode() { return IndexingMode.STREAMING; } public final String getStorageRouteSpec() { return storageRouteSpec; } + + public String getDocTypeName() { + return docTypeName; + } + @Override public int getRowBits() { return 0; } @@ -92,6 +99,9 @@ public class StreamingSearchCluster extends SearchCluster implements } private void deriveSingleSearchDefinition(com.yahoo.searchdefinition.Search localSearch, List<com.yahoo.searchdefinition.Search> globalSearches) { + if (!localSearch.getName().equals(docTypeName)) { + throw new IllegalStateException("Mismatch between document type name (" + docTypeName + ") and name of search definition (" + localSearch.getName() + ")"); + } this.sdConfig = new DerivedConfiguration(localSearch, globalSearches, deployLogger(), getRoot().getDeployState().rankProfileRegistry(), getRoot().getDeployState().getQueryProfiles().getRegistry()); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ConfigChangeTestUtils.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ConfigChangeTestUtils.java index 3738add1a6d..9572cfebb2a 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ConfigChangeTestUtils.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ConfigChangeTestUtils.java @@ -1,11 +1,16 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.application.validation.change; +import com.yahoo.config.model.api.ConfigChangeAction; import com.yahoo.config.model.api.ServiceInfo; import com.yahoo.config.application.api.ValidationOverrides; import java.time.Instant; import java.util.List; +import java.util.stream.Collectors; + +import static org.hamcrest.CoreMatchers.equalTo; +import static org.junit.Assert.assertThat; public class ConfigChangeTestUtils { @@ -25,4 +30,28 @@ public class ConfigChangeTestUtils { List<ServiceInfo> services, String documentType, Instant now) { return VespaRefeedAction.of(name, overrides, message, services, documentType, now); } + + public static List<ConfigChangeAction> normalizeServicesInActions(List<ConfigChangeAction> result) { + return result.stream() + .map(action -> ((VespaConfigChangeAction) action).modifyAction( + action.getMessage(), + normalizeServices(action.getServices()), + action.getType().equals(ConfigChangeAction.Type.REFEED) ? + ((VespaRefeedAction)action).getDocumentType() : "")) + .collect(Collectors.toList()); + } + + public static List<ServiceInfo> normalizeServices(List<ServiceInfo> services) { + return services.stream() + .map(service -> new ServiceInfo(service.getServiceName(), "null", null, null, + service.getConfigId(), "null")) + .collect(Collectors.toList()); + } + + public static void assertEqualActions(List<ConfigChangeAction> exp, List<ConfigChangeAction> act) { + exp.sort((lhs, rhs) -> lhs.getMessage().compareTo(rhs.getMessage())); + act.sort((lhs, rhs) -> lhs.getMessage().compareTo(rhs.getMessage())); + assertThat(act, equalTo(exp)); + } + } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/IndexedSearchClusterChangeValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/IndexedSearchClusterChangeValidatorTest.java index 387b8912018..43ad1bc0e8a 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/IndexedSearchClusterChangeValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/IndexedSearchClusterChangeValidatorTest.java @@ -13,13 +13,13 @@ import org.junit.Test; import java.time.Instant; import java.util.Arrays; import java.util.List; -import java.util.stream.Collectors; -import static org.hamcrest.CoreMatchers.equalTo; +import static com.yahoo.vespa.model.application.validation.change.ConfigChangeTestUtils.assertEqualActions; +import static com.yahoo.vespa.model.application.validation.change.ConfigChangeTestUtils.newRefeedAction; +import static com.yahoo.vespa.model.application.validation.change.ConfigChangeTestUtils.newRestartAction; +import static com.yahoo.vespa.model.application.validation.change.ConfigChangeTestUtils.normalizeServicesInActions; import static org.hamcrest.CoreMatchers.is; import static org.junit.Assert.assertThat; -import static com.yahoo.vespa.model.application.validation.change.ConfigChangeTestUtils.newRestartAction; -import static com.yahoo.vespa.model.application.validation.change.ConfigChangeTestUtils.newRefeedAction; public class IndexedSearchClusterChangeValidatorTest { @@ -74,27 +74,13 @@ public class IndexedSearchClusterChangeValidatorTest { buildCreator().create(); } - public void assertValidation() { - List<ConfigChangeAction> act = normalizeServicesInActions(validator.validate(currentModel, nextModel, - ValidationOverrides.empty, Instant.now())); - assertThat(act.size(), is(0)); - } - - private static List<ConfigChangeAction> normalizeServicesInActions(List<ConfigChangeAction> result) { - return result.stream(). - map(action -> ((VespaConfigChangeAction) action).modifyAction( - action.getMessage(), - normalizeServices(action.getServices()), - action.getType().equals(ConfigChangeAction.Type.REFEED) ? - ((VespaRefeedAction)action).getDocumentType() : "")). - collect(Collectors.toList()); + private List<ConfigChangeAction> validate() { + return normalizeServicesInActions(validator.validate(currentModel, nextModel, + ValidationOverrides.empty, Instant.now())); } - private static List<ServiceInfo> normalizeServices(List<ServiceInfo> services) { - return services.stream(). - map(service -> new ServiceInfo(service.getServiceName(), "null", null, null, - service.getConfigId(), "null")). - collect(Collectors.toList()); + public void assertValidation() { + assertThat(validate().size(), is(0)); } public void assertValidation(ConfigChangeAction exp) { @@ -102,11 +88,7 @@ public class IndexedSearchClusterChangeValidatorTest { } public void assertValidation(List<ConfigChangeAction> exp) { - List<ConfigChangeAction> act = normalizeServicesInActions(validator.validate(currentModel, nextModel, - ValidationOverrides.empty, Instant.now())); - exp.sort((lhs, rhs) -> lhs.getMessage().compareTo(rhs.getMessage())); - act.sort((lhs, rhs) -> lhs.getMessage().compareTo(rhs.getMessage())); - assertThat(act, equalTo(exp)); + assertEqualActions(exp, validate()); } } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/StreamingSearchClusterChangeValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/StreamingSearchClusterChangeValidatorTest.java new file mode 100644 index 00000000000..c5fee3efa99 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/StreamingSearchClusterChangeValidatorTest.java @@ -0,0 +1,179 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.application.validation.change; + +import com.yahoo.config.model.api.ConfigChangeAction; +import com.yahoo.config.model.api.ServiceInfo; +import com.yahoo.vespa.model.VespaModel; +import com.yahoo.config.application.api.ValidationOverrides; +import com.yahoo.vespa.model.content.utils.ApplicationPackageBuilder; +import com.yahoo.vespa.model.content.utils.ContentClusterBuilder; +import com.yahoo.vespa.model.content.utils.DocType; +import com.yahoo.vespa.model.content.utils.SearchDefinitionBuilder; +import org.junit.Test; + +import java.time.Instant; +import java.util.Arrays; +import java.util.List; + +import static com.yahoo.vespa.model.application.validation.change.ConfigChangeTestUtils.assertEqualActions; +import static com.yahoo.vespa.model.application.validation.change.ConfigChangeTestUtils.normalizeServicesInActions; +import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.assertThat; + +public class StreamingSearchClusterChangeValidatorTest { + + private static class Fixture { + VespaModel currentModel; + VespaModel nextModel; + StreamingSearchClusterChangeValidator validator; + + public Fixture(VespaModel currentModel, VespaModel nextModel) { + this.currentModel = currentModel; + this.nextModel = nextModel; + validator = new StreamingSearchClusterChangeValidator(); + } + + public static Fixture withOneDocType(String currentSd, String nextSd) { + return new Fixture(createOneDocModel(currentSd), createOneDocModel(nextSd)); + } + + public static VespaModel createOneDocModel(String sdContent) { + return new ApplicationPackageBuilder() + .addCluster(new ContentClusterBuilder().name("foo").docTypes(Arrays.asList(DocType.streaming("d1")))) + .addSearchDefinition(new SearchDefinitionBuilder().name("d1").content(sdContent).build()) + .buildCreator().create(); + } + + public static Fixture withTwoDocTypes(String currentSd, String nextSd) { + return new Fixture(createTwoDocModel(currentSd, currentSd), createTwoDocModel(nextSd, nextSd)); + } + + public static VespaModel createTwoDocModel(String d1Content, String d2Content) { + return new ApplicationPackageBuilder() + .addCluster(new ContentClusterBuilder().name("foo").docTypes(Arrays.asList(DocType.streaming("d1"), DocType.streaming("d2")))) + .addSearchDefinition(new SearchDefinitionBuilder().name("d1").content(d1Content).build()) + .addSearchDefinition(new SearchDefinitionBuilder().name("d2").content(d2Content).build()) + .buildCreator().create(); + } + + public static Fixture withTwoClusters(String currentSd, String nextSd) { + return new Fixture(createTwoClusterModel(currentSd, currentSd), createTwoClusterModel(nextSd, nextSd)); + } + + public static VespaModel createTwoClusterModel(String d1Content, String d2Content) { + return new ApplicationPackageBuilder() + .addCluster(new ContentClusterBuilder().name("foo").docTypes(Arrays.asList(DocType.streaming("d1")))) + .addCluster(new ContentClusterBuilder().name("bar").docTypes(Arrays.asList(DocType.streaming("d2")))) + .addSearchDefinition(new SearchDefinitionBuilder().name("d1").content(d1Content).build()) + .addSearchDefinition(new SearchDefinitionBuilder().name("d2").content(d2Content).build()) + .buildCreator().create(); + } + + public List<ConfigChangeAction> validate() { + return normalizeServicesInActions(validator.validate(currentModel, nextModel, + ValidationOverrides.empty, Instant.now())); + } + + public void assertValidation() { + assertThat(validate().size(), is(0)); + } + + public void assertValidation(ConfigChangeAction exp) { + assertValidation(Arrays.asList(exp)); + } + + public void assertValidation(List<ConfigChangeAction> exp) { + assertEqualActions(exp, validate()); + } + } + + private static String STRING_FIELD = "field f1 type string { indexing: summary }"; + private static String INT_FIELD = "field f1 type int { indexing: summary }"; + private static String ATTRIBUTE_INT_FIELD = "field f1 type int { indexing: attribute | summary }"; + private static String ATTRIBUTE_FAST_ACCESS_INT_FIELD = "field f1 type int { indexing: attribute | summary \n attribute: fast-access }"; + private static List<ServiceInfo> FOO_SERVICE = Arrays.asList( + new ServiceInfo("searchnode", "null", null, null, "foo/search/0", "null")); + private static List<ServiceInfo> BAR_SERVICE = Arrays.asList( + new ServiceInfo("searchnode2", "null", null, null, "bar/search/0", "null")); + + @Test + public void changing_field_type_requires_refeed() { + Fixture.withOneDocType(STRING_FIELD, INT_FIELD) + .assertValidation(createFieldTypeChangeRefeedAction("d1", FOO_SERVICE)); + } + + @Test + public void changes_in_multiple_streaming_clusters_are_discovered() { + Fixture.withTwoClusters(STRING_FIELD, INT_FIELD) + .assertValidation(Arrays.asList( + createFieldTypeChangeRefeedAction("d1", FOO_SERVICE), + createFieldTypeChangeRefeedAction("d2", BAR_SERVICE))); + } + + @Test + public void changes_in_multiple_document_types_are_discovered() { + Fixture.withTwoDocTypes(STRING_FIELD, INT_FIELD) + .assertValidation(Arrays.asList( + createFieldTypeChangeRefeedAction("d1", FOO_SERVICE), + createFieldTypeChangeRefeedAction("d2", FOO_SERVICE))); + } + + @Test + public void adding_fast_access_to_an_attribute_requires_restart() { + Fixture.withOneDocType(INT_FIELD, ATTRIBUTE_FAST_ACCESS_INT_FIELD) + .assertValidation(createAddFastAccessRestartAction()); + + Fixture.withOneDocType(ATTRIBUTE_INT_FIELD, ATTRIBUTE_FAST_ACCESS_INT_FIELD) + .assertValidation(createAddFastAccessRestartAction()); + } + + @Test + public void removing_fast_access_from_an_attribute_requires_restart() { + Fixture.withOneDocType(ATTRIBUTE_FAST_ACCESS_INT_FIELD, INT_FIELD) + .assertValidation(createRemoveFastAccessRestartAction()); + + Fixture.withOneDocType(ATTRIBUTE_FAST_ACCESS_INT_FIELD, ATTRIBUTE_INT_FIELD) + .assertValidation(createRemoveFastAccessRestartAction()); + } + + @Test + public void adding_attribute_field_is_ok() { + Fixture.withOneDocType(INT_FIELD, ATTRIBUTE_INT_FIELD).assertValidation(); + } + + @Test + public void removing_attribute_field_is_ok() { + Fixture.withOneDocType(ATTRIBUTE_INT_FIELD, INT_FIELD).assertValidation(); + } + + @Test + public void unchanged_fast_access_attribute_field_is_ok() { + Fixture.withOneDocType(ATTRIBUTE_FAST_ACCESS_INT_FIELD, ATTRIBUTE_FAST_ACCESS_INT_FIELD).assertValidation(); + } + + @Test + public void adding_streaming_cluster_is_ok() { + new Fixture(Fixture.createOneDocModel(STRING_FIELD), Fixture.createTwoClusterModel(STRING_FIELD, STRING_FIELD)).assertValidation(); + } + + @Test + public void removing_streaming_cluster_is_ok() { + new Fixture(Fixture.createTwoClusterModel(STRING_FIELD, STRING_FIELD), Fixture.createOneDocModel(STRING_FIELD)).assertValidation(); + } + + private static VespaConfigChangeAction createFieldTypeChangeRefeedAction(String docType, List<ServiceInfo> service) { + return ConfigChangeTestUtils.newRefeedAction("field-type-change", + ValidationOverrides.empty, + "Document type '" + docType + "': Field 'f1' changed: data type: 'string' -> 'int'", + service, docType, Instant.now()); + } + + private static VespaConfigChangeAction createAddFastAccessRestartAction() { + return ConfigChangeTestUtils.newRestartAction("Document type 'd1': Field 'f1' changed: add fast-access attribute", FOO_SERVICE); + } + + private static VespaConfigChangeAction createRemoveFastAccessRestartAction() { + return ConfigChangeTestUtils.newRestartAction("Document type 'd1': Field 'f1' changed: remove fast-access attribute", FOO_SERVICE); + } + +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java index cc17c46146f..4fcac56d0c0 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java @@ -747,7 +747,7 @@ public class ContentBuilderTest extends DomBuilderTest { " <node hostalias='mockhost' distribution-key='0' />\n" + " </group>\n" + " <engine>\n" + - " <vds/>\n" + + " <dummy/>\n" + " </engine>\n" + " </content>\n"); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/ClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/ClusterTest.java index 1c8f572ce8d..33bd5084138 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/ClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/ClusterTest.java @@ -431,9 +431,6 @@ public class ClusterTest extends ContentBaseTest { cluster = parse( "<content version=\"1.0\" id=\"storage\">\n" + " <documents/>" + - " <engine>" + - " <vds/>" + - " </engine>" + " <group>\n" + " <node distribution-key=\"0\" hostalias=\"mockhost\"/>\n" + " </group>\n" + @@ -463,9 +460,6 @@ public class ClusterTest extends ContentBaseTest { cluster = parse( "<content version=\"1.0\" id=\"storage\">\n" + " <documents/>" + - " <engine>" + - " <vds/>" + - " </engine>" + " <group>\n" + " <node distribution-key=\"0\" hostalias=\"mockhost\"/>\n" + " </group>\n" + diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/DistributorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/DistributorTest.java index bdb64877fea..8ae68468374 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/DistributorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/DistributorTest.java @@ -70,9 +70,6 @@ public class DistributorTest { StorDistributormanagerConfig.Builder builder = new StorDistributormanagerConfig.Builder(); parse("<cluster id=\"storage\">\n" + " <documents/>" + - " <engine>" + - " <vds/>" + - " </engine>" + " <tuning>\n" + " <bucket-splitting max-documents=\"2K\" max-size=\"25M\" minimum-bits=\"8\" />\n" + " </tuning>\n" + @@ -96,9 +93,6 @@ public class DistributorTest { StorDistributormanagerConfig.Builder builder = new StorDistributormanagerConfig.Builder(); ContentCluster cluster = parseCluster("<cluster id=\"storage\">\n" + " <documents/>" + - " <engine>" + - " <vds/>" + - " </engine>" + " <tuning>" + " <distribution type=\"legacy\"/>" + " </tuning>\n" + @@ -120,9 +114,6 @@ public class DistributorTest { cluster = parseCluster("<cluster id=\"storage\">\n" + " <documents/>" + - " <engine>" + - " <vds/>" + - " </engine>" + " <tuning>" + " <distribution type=\"legacy\"/>" + " </tuning>\n" + diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/GenericConfigTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/GenericConfigTest.java index 75ddd8173c3..a65c4e50521 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/GenericConfigTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/GenericConfigTest.java @@ -1,7 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.content; -import com.yahoo.vespa.config.storage.StorMemfilepersistenceConfig; +import com.yahoo.vespa.config.content.StorFilestorConfig; import com.yahoo.searchdefinition.parser.ParseException; import com.yahoo.vespa.model.VespaModel; import com.yahoo.vespa.model.content.cluster.ContentCluster; @@ -28,8 +28,8 @@ public class GenericConfigTest { private String servicesXml() { return "" + "<services version='1.0'>" + - " <config name='vespa.config.storage.stor-memfilepersistence'>" + - " <disk_full_factor>0.001</disk_full_factor> " + + " <config name='vespa.config.content.stor-filestor'>" + + " <num_threads>7</num_threads> " + " </config>" + " <admin version=\"2.0\">" + " <adminserver hostalias=\"node0\" />" + @@ -61,16 +61,16 @@ public class GenericConfigTest { public void config_override_on_root_is_visible_on_storage_cluster() throws Exception { StorageCluster cluster = model.getContentClusters().get("storage").getStorageNodes(); - StorMemfilepersistenceConfig config = model.getConfig(StorMemfilepersistenceConfig.class, cluster.getConfigId()); - assertThat(config.disk_full_factor(), is(0.001)); + StorFilestorConfig config = model.getConfig(StorFilestorConfig.class, cluster.getConfigId()); + assertThat(config.num_threads(), is(7)); } @Test public void config_override_on_root_is_visible_on_content_cluster() throws Exception { ContentCluster cluster = model.getContentClusters().get("storage"); - StorMemfilepersistenceConfig config = model.getConfig(StorMemfilepersistenceConfig.class, cluster.getConfigId()); - assertThat(config.disk_full_factor(), is(0.001)); + StorFilestorConfig config = model.getConfig(StorFilestorConfig.class, cluster.getConfigId()); + assertThat(config.num_threads(), is(7)); } } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/StorageClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/StorageClusterTest.java index d52b07a318a..4f7440e590b 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/StorageClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/StorageClusterTest.java @@ -6,20 +6,17 @@ import com.yahoo.vespa.config.content.core.StorVisitorConfig; import com.yahoo.vespa.config.content.StorFilestorConfig; import com.yahoo.vespa.config.content.core.StorServerConfig; import com.yahoo.vespa.config.content.PersistenceConfig; -import com.yahoo.vespa.config.storage.StorDevicesConfig; import com.yahoo.config.model.test.MockRoot; import com.yahoo.documentmodel.NewDocumentType; -import com.yahoo.text.XML; import static com.yahoo.vespa.defaults.Defaults.getDefaults; import com.yahoo.vespa.model.content.cluster.ContentCluster; import com.yahoo.vespa.model.content.storagecluster.StorageCluster; import com.yahoo.vespa.model.content.utils.ContentClusterUtils; import org.junit.Test; -import org.w3c.dom.Document; -import java.util.Collections; -import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; public class StorageClusterTest { @@ -195,18 +192,11 @@ public class StorageClusterTest { StorageNode node = cluster.getStorageNodes().getChildren().get("0"); { - StorDevicesConfig.Builder builder = new StorDevicesConfig.Builder(); - node.getConfig(builder); - StorDevicesConfig config = new StorDevicesConfig(builder); - assertEquals(getDefaults().underVespaHome("var/db/vespa/vds/storage/storage/0"), config.root_folder()); - } - - { StorServerConfig.Builder builder = new StorServerConfig.Builder(); cluster.getStorageNodes().getConfig(builder); node.getConfig(builder); StorServerConfig config = new StorServerConfig(builder); - assertEquals(getDefaults().underVespaHome("var/db/vespa/vds/storage/storage/0"), config.root_folder()); + assertEquals(getDefaults().underVespaHome("var/db/vespa/search/storage/storage/0"), config.root_folder()); } { @@ -214,7 +204,7 @@ public class StorageClusterTest { cluster.getDistributorNodes().getConfig(builder); cluster.getDistributorNodes().getChildren().get("0").getConfig(builder); StorServerConfig config = new StorServerConfig(builder); - assertEquals(getDefaults().underVespaHome("var/db/vespa/vds/storage/distributor/0"), config.root_folder()); + assertEquals(getDefaults().underVespaHome("var/db/vespa/search/storage/distributor/0"), config.root_folder()); } } |