diff options
Diffstat (limited to 'config-model')
5 files changed, 94 insertions, 12 deletions
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryConsistency.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryConsistency.java index d2c4968ca26..a4ad3964d47 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryConsistency.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryConsistency.java @@ -8,11 +8,14 @@ import com.yahoo.searchdefinition.RankProfileRegistry; import com.yahoo.searchdefinition.document.Attribute; import com.yahoo.document.WeightedSetDataType; import com.yahoo.searchdefinition.Search; +import com.yahoo.searchdefinition.document.ImmutableSDField; import com.yahoo.vespa.documentmodel.DocumentSummary; import com.yahoo.vespa.documentmodel.SummaryField; import com.yahoo.vespa.documentmodel.SummaryTransform; import com.yahoo.vespa.model.container.search.QueryProfiles; +import static com.yahoo.searchdefinition.document.ComplexAttributeFieldUtils.isComplexFieldWithOnlyStructFieldAttributes; + /** * Ensure that summary field transforms for fields having the same name * are consistent across summary classes @@ -33,6 +36,7 @@ public class SummaryConsistency extends Processor { for (SummaryField summaryField : summary.getSummaryFields() ) { assertConsistency(summaryField, search, validate); makeAttributeTransformIfAppropriate(summaryField, search); + makeAttributeCombinerTransformIfAppropriate(summaryField, search); } } } @@ -63,6 +67,17 @@ public class SummaryConsistency extends Processor { summaryField.setTransform(SummaryTransform.ATTRIBUTE); } + /** If the source is a complex field with only struct field attributes then make this use the attribute combiner transform */ + private void makeAttributeCombinerTransformIfAppropriate(SummaryField summaryField,Search search) { + if (summaryField.getTransform() == SummaryTransform.NONE) { + String source_field_name = summaryField.getSingleSource(); + ImmutableSDField source = search.getField(source_field_name); + if (source != null && isComplexFieldWithOnlyStructFieldAttributes(source)) { + summaryField.setTransform(SummaryTransform.ATTRIBUTECOMBINER); + } + } + } + private void assertConsistentTypes(SummaryField existing, SummaryField seen) { if (existing.getDataType() instanceof WeightedSetDataType && seen.getDataType() instanceof WeightedSetDataType && ((WeightedSetDataType)existing.getDataType()).getNestedType().equals(((WeightedSetDataType)seen.getDataType()).getNestedType())) diff --git a/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java b/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java index 31746c39f80..bedf8057b41 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.model; import ai.vespa.rankingexpression.importer.configmodelview.ImportedMlModel; import com.yahoo.collections.Pair; +import com.yahoo.component.Version; import com.yahoo.config.ConfigBuilder; import com.yahoo.config.ConfigInstance; import com.yahoo.config.ConfigInstance.Builder; @@ -96,6 +97,8 @@ public final class VespaModel extends AbstractConfigProducerRoot implements Seri private static final long serialVersionUID = 1L; public static final Logger log = Logger.getLogger(VespaModel.class.getPackage().toString()); + + private final Version version; private final ConfigModelRepo configModelRepo = new ConfigModelRepo(); private final AllocatedHosts allocatedHosts; @@ -155,6 +158,7 @@ public final class VespaModel extends AbstractConfigProducerRoot implements Seri private VespaModel(ConfigModelRegistry configModelRegistry, DeployState deployState, boolean complete, FileDistributor fileDistributor) throws IOException, SAXException { super("vespamodel"); + this.version = deployState.getVespaVersion(); this.validationOverrides = deployState.validationOverrides(); configModelRegistry = new VespaConfigModelRegistry(configModelRegistry); VespaModelBuilder builder = new VespaDomBuilder(); @@ -292,6 +296,11 @@ public final class VespaModel extends AbstractConfigProducerRoot implements Seri return validationOverrides.allows(ValidationId.skipOldConfigModels, now); } + @Override + public Version version() { + return version; + } + /** * Resolves config of the given type and config id, by first instantiating the correct {@link com.yahoo.config.ConfigInstance.Builder}, * calling {@link #getConfig(com.yahoo.config.ConfigInstance.Builder, String)}. The default values used will be those of the config diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java index fdf88124012..270b425fda9 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java @@ -50,7 +50,6 @@ import com.yahoo.vespa.model.content.engines.PersistenceEngine; import com.yahoo.vespa.model.content.engines.ProtonEngine; import com.yahoo.vespa.model.content.storagecluster.StorageCluster; import com.yahoo.vespa.model.search.IndexedSearchCluster; -import com.yahoo.vespa.model.search.MultilevelDispatchValidator; import com.yahoo.vespa.model.search.Tuning; import org.w3c.dom.Element; @@ -318,7 +317,8 @@ public class ContentCluster extends AbstractConfigProducer implements if (clusterControllers == null) { List<HostResource> hosts = admin.getClusterControllerHosts(); if (hosts.size() > 1) { - context.getDeployState().getDeployLogger().log(Level.INFO, "When having content cluster(s) and more than 1 config server it is recommended to configure cluster controllers explicitly."); + context.getDeployState().getDeployLogger().log(Level.INFO, + "When having content cluster(s) and more than 1 config server it is recommended to configure cluster controllers explicitly."); } clusterControllers = createClusterControllers(admin, hosts, "cluster-controllers", false, context.getDeployState()); admin.setClusterControllers(clusterControllers); @@ -348,10 +348,20 @@ public class ContentCluster extends AbstractConfigProducer implements } private List<HostResource> drawControllerHosts(int count, StorageGroup rootGroup, Collection<ContainerModel> containers) { - List<HostResource> hosts = drawContentHostsRecursively(count, rootGroup); + List<HostResource> hosts = drawControllerHosts(count, false, rootGroup, containers); + List<HostResource> retiredHosts = drawControllerHosts(count, true, rootGroup, containers); + + // preserve the cluster state in case all pre-existing controllers are on retired nodes + List<HostResource> all = new ArrayList<>(hosts); + all.addAll(retiredHosts); + return all; + } + + private List<HostResource> drawControllerHosts(int count, boolean retired, StorageGroup rootGroup, Collection<ContainerModel> containers) { + List<HostResource> hosts = drawContentHostsRecursively(count, retired, rootGroup); // if (hosts.size() < count) // supply with containers TODO: Currently disabled due to leading to topology change problems // hosts.addAll(drawContainerHosts(count - hosts.size(), containers, new HashSet<>(hosts))); - if (hosts.size() % 2 == 0) // ZK clusters of even sizes are less available (even in the size=2 case) + if (hosts.size() % 2 == 0 && ! hosts.isEmpty()) // ZK clusters of even sizes are less available (even in the size=2 case) hosts = hosts.subList(0, hosts.size()-1); return hosts; } @@ -425,16 +435,16 @@ public class ContentCluster extends AbstractConfigProducer implements */ // Note: This method cannot be changed to draw different nodes without ensuring that it will draw nodes // which overlaps with previously drawn nodes as that will prevent rolling upgrade - private List<HostResource> drawContentHostsRecursively(int count, StorageGroup group) { + private List<HostResource> drawContentHostsRecursively(int count, boolean retired, StorageGroup group) { Set<HostResource> hosts = new HashSet<>(); if (group.getNodes().isEmpty()) { int hostsPerSubgroup = (int)Math.ceil((double)count / group.getSubgroups().size()); for (StorageGroup subgroup : group.getSubgroups()) - hosts.addAll(drawContentHostsRecursively(hostsPerSubgroup, subgroup)); + hosts.addAll(drawContentHostsRecursively(hostsPerSubgroup, retired, subgroup)); } else { hosts.addAll(group.getNodes().stream() - .filter(node -> ! node.isRetired()) // Avoid retired controllers to avoid surprises on expiry + .filter(node -> node.isRetired() == retired) .map(StorageNode::getHostResource).collect(Collectors.toList())); } diff --git a/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java b/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java index 7c71c399f35..d4457fe59cd 100644 --- a/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java +++ b/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java @@ -636,7 +636,7 @@ public class ModelProvisioningTest { } @Test - public void testClusterControllersAreNotPlacedOnRetiredNodes() { + public void testClusterControllersIncludeNonRetiredNodes() { String services = "<?xml version='1.0' encoding='utf-8' ?>\n" + "<services>" + @@ -662,11 +662,14 @@ public class ModelProvisioningTest { // Check content clusters ContentCluster cluster = model.getContentClusters().get("bar"); ClusterControllerContainerCluster clusterControllers = cluster.getClusterControllers(); - assertEquals(3, clusterControllers.getContainers().size()); + assertEquals(3 + 3, clusterControllers.getContainers().size()); // 3 new + 3 retired assertEquals("bar-controllers", clusterControllers.getName()); - assertEquals("Skipping retired default09", "node-1-3-9-08", clusterControllers.getContainers().get(0).getHostName()); - assertEquals("Skipping retired default06", "node-1-3-9-05", clusterControllers.getContainers().get(1).getHostName()); - assertEquals("Skipping retired default03", "node-1-3-9-02", clusterControllers.getContainers().get(2).getHostName()); + assertEquals("Non-retired", "node-1-3-9-08", clusterControllers.getContainers().get(0).getHostName()); + assertEquals("Non-retired", "node-1-3-9-05", clusterControllers.getContainers().get(1).getHostName()); + assertEquals("Non-retired", "node-1-3-9-02", clusterControllers.getContainers().get(2).getHostName()); + assertEquals("Retired", "node-1-3-9-09", clusterControllers.getContainers().get(3).getHostName()); + assertEquals("Retired", "node-1-3-9-06", clusterControllers.getContainers().get(4).getHostName()); + assertEquals("Retired", "node-1-3-9-03", clusterControllers.getContainers().get(5).getHostName()); } @Test diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/SummaryConsistencyTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/SummaryConsistencyTestCase.java new file mode 100644 index 00000000000..71b4db60886 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/SummaryConsistencyTestCase.java @@ -0,0 +1,45 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.processing; + +import com.yahoo.searchdefinition.Search; +import com.yahoo.searchdefinition.SearchBuilder; +import com.yahoo.searchdefinition.parser.ParseException; +import com.yahoo.vespa.documentmodel.SummaryTransform; +import org.junit.Test; + +import static com.yahoo.config.model.test.TestUtil.joinLines; +import static org.junit.Assert.assertEquals; + +public class SummaryConsistencyTestCase { + + @Test + public void attribute_combiner_transform_is_set_when_source_is_array_of_struct_with_only_struct_field_attributes() throws ParseException { + String sd = joinLines( + "search structmemorysummary {", + " document structmemorysummary {", + " struct elem {", + " field name type string {}", + " field weight type int {}\n", + " }", + " field elem_array type array<elem> {", + " indexing: summary", + " struct-field name {", + " indexing: attribute", + " }", + " struct-field weight {", + " indexing: attribute", + " }", + " }", + " }", + " document-summary unfiltered {", + " summary elem_array_unfiltered type array<elem> {", + " source: elem_array", + " }", + " }", + "", + "}" + ); + Search search = SearchBuilder.createFromString(sd).getSearch(); + assertEquals(SummaryTransform.ATTRIBUTECOMBINER, search.getSummaryField("elem_array_unfiltered").getTransform()); + } +} |