From 96f24c98660281bec88419fc201b4220f36b78d5 Mon Sep 17 00:00:00 2001 From: Henning Baldersheim Date: Thu, 20 Apr 2017 18:29:27 +0200 Subject: Test and implement that only fast access attributes are produced for streaming cluster. --- .../searchdefinition/derived/AttributeFields.java | 92 +++++++++++++--------- .../vespa/model/search/StreamingSearchCluster.java | 5 +- .../search/test/DocumentDatabaseTestCase.java | 19 +++-- .../model/test/utils/ApplicationPackageUtils.java | 3 + 4 files changed, 73 insertions(+), 46 deletions(-) (limited to 'config-model') diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/AttributeFields.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/AttributeFields.java index 9e6d59ed25f..2482c9a1c91 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/AttributeFields.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/AttributeFields.java @@ -26,6 +26,8 @@ import java.util.Map; */ public class AttributeFields extends Derived implements AttributesConfig.Producer { + public enum FieldSet {ALL, FAST_ACCESS} + private Map attributes = new java.util.LinkedHashMap<>(); /** @@ -124,48 +126,62 @@ public class AttributeFields extends Derived implements AttributesConfig.Produce @Override public void getConfig(AttributesConfig.Builder builder) { - for (Attribute attribute : attributes.values()) { - AttributesConfig.Attribute.Builder aaB = new AttributesConfig.Attribute.Builder() + getConfig(builder, FieldSet.ALL); + } + + private boolean isAttributeInFieldSet(Attribute attribute, FieldSet fs) { + return (fs == FieldSet.ALL) || ((fs == FieldSet.FAST_ACCESS) && attribute.isFastAccess()); + } + + private AttributesConfig.Attribute.Builder getConfig(Attribute attribute) { + AttributesConfig.Attribute.Builder aaB = new AttributesConfig.Attribute.Builder() .name(attribute.getName()) .datatype(AttributesConfig.Attribute.Datatype.Enum.valueOf(attribute.getType().getExportAttributeTypeName())) .collectiontype(AttributesConfig.Attribute.Collectiontype.Enum.valueOf(attribute.getCollectionType().getName())); - if (attribute.isRemoveIfZero()) { - aaB.removeifzero(true); - } - if (attribute.isCreateIfNonExistent()) { - aaB.createifnonexistent(true); - } - aaB.enablebitvectors(attribute.isEnabledBitVectors()); - aaB.enableonlybitvector(attribute.isEnabledOnlyBitVector()); - if (attribute.isFastSearch()) { - aaB.fastsearch(true); - } - if (attribute.isFastAccess()) { - aaB.fastaccess(true); - } - if (attribute.isHuge()) { - aaB.huge(true); - } - if (attribute.getSorting().isDescending()) { - aaB.sortascending(false); - } - if (attribute.getSorting().getFunction() != Sorting.Function.UCA) { - aaB.sortfunction(AttributesConfig.Attribute.Sortfunction.Enum.valueOf(attribute.getSorting().getFunction().toString())); - } - if (attribute.getSorting().getStrength() != Sorting.Strength.PRIMARY) { - aaB.sortstrength(AttributesConfig.Attribute.Sortstrength.Enum.valueOf(attribute.getSorting().getStrength().toString())); - } - if (!attribute.getSorting().getLocale().isEmpty()) { - aaB.sortlocale(attribute.getSorting().getLocale()); - } - aaB.arity(attribute.arity()); - aaB.lowerbound(attribute.lowerBound()); - aaB.upperbound(attribute.upperBound()); - aaB.densepostinglistthreshold(attribute.densePostingListThreshold()); - if (attribute.tensorType().isPresent()) { - aaB.tensortype(attribute.tensorType().get().toString()); + if (attribute.isRemoveIfZero()) { + aaB.removeifzero(true); + } + if (attribute.isCreateIfNonExistent()) { + aaB.createifnonexistent(true); + } + aaB.enablebitvectors(attribute.isEnabledBitVectors()); + aaB.enableonlybitvector(attribute.isEnabledOnlyBitVector()); + if (attribute.isFastSearch()) { + aaB.fastsearch(true); + } + if (attribute.isFastAccess()) { + aaB.fastaccess(true); + } + if (attribute.isHuge()) { + aaB.huge(true); + } + if (attribute.getSorting().isDescending()) { + aaB.sortascending(false); + } + if (attribute.getSorting().getFunction() != Sorting.Function.UCA) { + aaB.sortfunction(AttributesConfig.Attribute.Sortfunction.Enum.valueOf(attribute.getSorting().getFunction().toString())); + } + if (attribute.getSorting().getStrength() != Sorting.Strength.PRIMARY) { + aaB.sortstrength(AttributesConfig.Attribute.Sortstrength.Enum.valueOf(attribute.getSorting().getStrength().toString())); + } + if (!attribute.getSorting().getLocale().isEmpty()) { + aaB.sortlocale(attribute.getSorting().getLocale()); + } + aaB.arity(attribute.arity()); + aaB.lowerbound(attribute.lowerBound()); + aaB.upperbound(attribute.upperBound()); + aaB.densepostinglistthreshold(attribute.densePostingListThreshold()); + if (attribute.tensorType().isPresent()) { + aaB.tensortype(attribute.tensorType().get().toString()); + } + return aaB; + } + + public void getConfig(AttributesConfig.Builder builder, FieldSet fs) { + for (Attribute attribute : attributes.values()) { + if (isAttributeInFieldSet(attribute, fs)) { + builder.attribute(getConfig(attribute)); } - builder.attribute(aaB); } } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/StreamingSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/search/StreamingSearchCluster.java index f75d27b4f5b..ef3e279fb64 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/StreamingSearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/StreamingSearchCluster.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.model.search; import com.yahoo.config.model.producer.AbstractConfigProducer; import com.yahoo.prelude.fastsearch.DocumentdbInfoConfig; +import com.yahoo.searchdefinition.derived.AttributeFields; import com.yahoo.searchdefinition.derived.DerivedConfiguration; import com.yahoo.vespa.config.search.AttributesConfig; import com.yahoo.vespa.config.search.RankProfilesConfig; @@ -36,7 +37,9 @@ public class StreamingSearchCluster extends SearchCluster implements @Override public void getConfig(AttributesConfig.Builder builder) { - StreamingSearchCluster.this.getConfig(builder); + if (getSdConfig() != null) { + getSdConfig().getAttributeFields().getConfig(builder, AttributeFields.FieldSet.FAST_ACCESS); + } } } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/search/test/DocumentDatabaseTestCase.java b/config-model/src/test/java/com/yahoo/vespa/model/search/test/DocumentDatabaseTestCase.java index 6d1028e13bc..c15af2ff091 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/search/test/DocumentDatabaseTestCase.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/search/test/DocumentDatabaseTestCase.java @@ -98,8 +98,9 @@ public class DocumentDatabaseTestCase { assertEquals(1, icfg.indexfield().size()); assertEquals(indexField, icfg.indexfield(0).name()); AttributesConfig acfg = model.getConfig(AttributesConfig.class, configId); - assertEquals(1, acfg.attribute().size()); + assertEquals(2, acfg.attribute().size()); assertEquals(attributeField, acfg.attribute(0).name()); + assertEquals(attributeField+"_nfa", acfg.attribute(1).name()); RankProfilesConfig rcfg = model.getConfig(RankProfilesConfig.class, configId); assertEquals(6, rcfg.rankprofile().size()); } @@ -137,11 +138,13 @@ public class DocumentDatabaseTestCase { } { AttributesConfig rac1 = model.getConfig(AttributesConfig.class, type1Id); - assertEquals(1, rac1.attribute().size()); + assertEquals(2, rac1.attribute().size()); assertEquals("f2", rac1.attribute(0).name()); + assertEquals("f2_nfa", rac1.attribute(1).name()); AttributesConfig rac2 = model.getConfig(AttributesConfig.class, type2Id); - assertEquals(1, rac2.attribute().size()); + assertEquals(2, rac2.attribute().size()); assertEquals("f4", rac2.attribute(0).name()); + assertEquals("f4_nfa", rac2.attribute(1).name()); } { IlscriptsConfig icfg = model.getConfig(IlscriptsConfig.class, "test/search/cluster.test"); @@ -189,10 +192,12 @@ public class DocumentDatabaseTestCase { } { // attributes config AttributesConfig acfg = model.getConfig(AttributesConfig.class, searcherId); - assertEquals(2, acfg.attribute().size()); + assertEquals(4, acfg.attribute().size()); assertEquals("f2", acfg.attribute(0).name()); - assertEquals("f4", acfg.attribute(1).name()); - assertEquals("f4", acfg.attribute(1).name()); + assertEquals("f2_nfa", acfg.attribute(1).name()); + assertEquals("f4", acfg.attribute(2).name()); + assertEquals("f4_nfa", acfg.attribute(3).name()); + } } @@ -256,7 +261,7 @@ public class DocumentDatabaseTestCase { public void testThatAttributesConfigIsProducedForIndexed() { assertAttributesConfigIndependentOfMode("index", Arrays.asList("type1"), Arrays.asList("test/search/cluster.test/type1"), - ImmutableMap.of("type1", Arrays.asList("f2"))); + ImmutableMap.of("type1", Arrays.asList("f2", "f2_nfa"))); } @Test public void testThatAttributesConfigIsProducedForStreamingForFastAccessFields() { diff --git a/config-model/src/test/java/com/yahoo/vespa/model/test/utils/ApplicationPackageUtils.java b/config-model/src/test/java/com/yahoo/vespa/model/test/utils/ApplicationPackageUtils.java index f38b7592e31..2653b3f61d2 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/test/utils/ApplicationPackageUtils.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/test/utils/ApplicationPackageUtils.java @@ -30,6 +30,9 @@ public class ApplicationPackageUtils { " attribute: fast-access\n" + " header\n" + " }\n" + + " field " + field2 + "_nfa type int {\n" + + " indexing: attribute \n" + + " }\n" + " }\n" + " rank-profile staticrank inherits default {" + " first-phase { expression: attribute(" + field2 + ") }" + -- cgit v1.2.3