Merge branch 'master' into bratseth/inputs

8 files changed, 29 insertions, 234 deletions

diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java
index cb1e2e047ad..8366dde383b 100644
--- a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java
+++ b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java
@@ -81,7 +81,7 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea
     private boolean useV8GeoPositions = false;
     private List<String> environmentVariables = List.of();
     private boolean avoidRenamingSummaryFeatures = false;
-    private boolean experimentalSdParsing = false;
+    private boolean experimentalSdParsing = true;
     private Architecture adminClusterNodeResourcesArchitecture = Architecture.getDefault();
 
     @Override public ModelContext.FeatureFlags featureFlags() { return this; }
diff --git a/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java b/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java
index 824bf248b5c..1838f1e36b7 100644
--- a/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java
+++ b/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java
@@ -99,7 +99,7 @@ public class MockApplicationPackage implements ApplicationPackage {
     @SuppressWarnings("deprecation") // not redundant
     @Override
     public String getApplicationName() {
-        return "mock application";
+        return "mock-application";
     }
 
     @Override
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/AttributeFields.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/AttributeFields.java
index dad8385e0e1..7acf5557236 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/AttributeFields.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/AttributeFields.java
@@ -100,17 +100,21 @@ public class AttributeFields extends Derived implements AttributesConfig.Produce
         }
     }
 
+    private void applyRanking(ImmutableSDField field, Attribute attribute) {
+        Ranking ranking = field.getRanking();
+        if (ranking != null && ranking.isFilter()) {
+            attribute.setEnableBitVectors(true);
+            attribute.setEnableOnlyBitVector(true);
+        }
+    }
+
     private void deriveAttribute(ImmutableSDField field, Attribute fieldAttribute) {
         Attribute attribute = getAttribute(fieldAttribute.getName());
         if (attribute == null) {
             attributes.put(fieldAttribute.getName(), fieldAttribute);
             attribute = getAttribute(fieldAttribute.getName());
         }
-        Ranking ranking = field.getRanking();
-        if (ranking != null && ranking.isFilter()) {
-            attribute.setEnableBitVectors(true);
-            attribute.setEnableOnlyBitVector(true);
-        }
+        applyRanking(field, attribute);
     }
 
     private void deriveImportedAttributes(ImmutableSDField field) {
@@ -134,6 +138,7 @@ public class AttributeFields extends Derived implements AttributesConfig.Produce
         }
         Attribute attribute = field.getAttributes().get(field.getName());
         if (attribute != null) {
+            applyRanking(field, attribute);
             attributes.put(attribute.getName(), attribute.convertToArray());
         }
     }
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ConvertParsedFields.java b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ConvertParsedFields.java
index dee9b648228..92e099bc1fe 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ConvertParsedFields.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ConvertParsedFields.java
@@ -159,6 +159,15 @@ public class ConvertParsedFields {
         for (var structField : parsed.getStructFields()) {
             convertStructField(field, structField);
         }
+        if (parsed.hasLiteral()) {
+            field.getRanking().setLiteral(true);
+        }
+        if (parsed.hasFilter()) {
+            field.getRanking().setFilter(true);
+        }
+        if (parsed.hasNormal()) {
+            field.getRanking().setNormal(true);
+        }
     }
 
     private void convertStructField(SDField field, ParsedField parsed) {
@@ -196,15 +205,6 @@ public class ConvertParsedFields {
             summaryField.addDestination("default");
             summaryField.setTransform(summaryField.getTransform().bold());
         }
-        if (parsed.hasLiteral()) {
-            field.getRanking().setLiteral(true);
-        }
-        if (parsed.hasFilter()) {
-            field.getRanking().setFilter(true);
-        }
-        if (parsed.hasNormal()) {
-            field.getRanking().setNormal(true);
-        }
     }
 
     static void convertSummaryFieldSettings(SummaryField summary, ParsedSummaryField parsed) {
diff --git a/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentTypes.java b/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentTypes.java
index 630e9f0c097..ba41fa0f5b3 100644
--- a/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentTypes.java
+++ b/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentTypes.java
@@ -27,29 +27,16 @@ public class DocumentTypes {
     }
 
     public DocumenttypesConfig.Builder produce(DocumentModel model, DocumenttypesConfig.Builder builder) {
-        /* later:
-        if (some flag) {
-            return produceDocTypes(model, builder);
-        }
-        */
         builder.usev8geopositions(this.useV8GeoPositions);
         Map<NewDocumentType.Name, NewDocumentType> produced = new HashMap<>();
+        var indexMap = new IdxMap();
         for (NewDocumentType documentType : model.getDocumentManager().getTypes()) {
-            produceInheritOrder(documentType, builder, produced);
+            docTypeInheritOrder(documentType, builder, produced, indexMap);
         }
+        indexMap.verifyAllDone();
         return builder;
     }
 
-    private void produceInheritOrder(NewDocumentType documentType, DocumenttypesConfig.Builder builder, Map<NewDocumentType.Name, NewDocumentType> produced) {
-        if (!produced.containsKey(documentType.getFullName())) {
-            for (NewDocumentType inherited : documentType.getInherited()) {
-                produceInheritOrder(inherited, builder, produced);
-            }
-            buildConfig(documentType, builder);
-            produced.put(documentType.getFullName(), documentType);
-        }
-    }
-
     static private <T> List<T> sortedList(Collection<T> unsorted, Comparator<T> cmp) {
         var list = new ArrayList<T>();
         list.addAll(unsorted);
@@ -57,203 +44,6 @@ public class DocumentTypes {
         return list;
     }
 
-    private void buildConfig(NewDocumentType documentType, DocumenttypesConfig.Builder builder) {
-        if (documentType == VespaDocumentType.INSTANCE) {
-            return;
-        }
-        DocumenttypesConfig.Documenttype.Builder db = new DocumenttypesConfig.Documenttype.Builder();
-        db.
-                id(documentType.getId()).
-                name(documentType.getName()).
-                headerstruct(documentType.getContentStruct().getId());
-        Set<Integer> built = new HashSet<>();
-        for (NewDocumentType inherited : documentType.getInherited()) {
-            db.inherits(new DocumenttypesConfig.Documenttype.Inherits.Builder().id(inherited.getId()));
-            markAsBuilt(built, inherited.getAllTypes());
-        }
-        for (DataType dt : sortedList(documentType.getTypes(), (a,b) -> a.getName().compareTo(b.getName()))) {
-            buildConfig(dt, db, built);
-        }
-        for (AnnotationType annotation : sortedList(documentType.getAnnotations(), (a,b) -> a.getName().compareTo(b.getName()))) {
-            DocumenttypesConfig.Documenttype.Annotationtype.Builder atb = new DocumenttypesConfig.Documenttype.Annotationtype.Builder();
-            db.annotationtype(atb);
-            buildConfig(annotation, atb);
-        }
-        buildConfig(documentType.getFieldSets(), db);
-        buildImportedFieldsConfig(documentType.getImportedFieldNames(), db);
-        builder.documenttype(db);
-    }
-
-    private void buildConfig(Set<FieldSet> fieldSets, DocumenttypesConfig.Documenttype.Builder db) {
-        for (FieldSet fs : fieldSets) {
-            buildConfig(fs, db);
-        }
-    }
-
-    private void buildConfig(FieldSet fs, DocumenttypesConfig.Documenttype.Builder db) {
-        db.fieldsets(fs.getName(), new DocumenttypesConfig.Documenttype.Fieldsets.Builder().fields(fs.getFieldNames()));
-    }
-
-    private void markAsBuilt(Set<Integer> built, DataTypeCollection typeCollection) {
-        for (DataType type : typeCollection.getTypes()) {
-            built.add(type.getId());
-        }
-    }
-
-    private void buildConfig(AnnotationType annotation, DocumenttypesConfig.Documenttype.Annotationtype.Builder builder) {
-        builder.
-                id(annotation.getId()).
-                name(annotation.getName());
-        DataType dt = annotation.getDataType();
-        if (dt != null) {
-            builder.datatype(dt.getId());
-        }
-        for (AnnotationType inherited : annotation.getInheritedTypes()) {
-            builder.inherits(new DocumenttypesConfig.Documenttype.Annotationtype.Inherits.Builder().id(inherited.getId()));
-        }
-    }
-
-    private void buildConfig(DataType type, DocumenttypesConfig.Documenttype.Builder documentBuilder, Set<Integer> built) {
-        if ((VespaDocumentType.INSTANCE.getDataType(type.getId()) == null) && !built.contains(type.getId())) {
-            built.add(type.getId());
-            DocumenttypesConfig.Documenttype.Datatype.Builder dataTypeBuilder = new DocumenttypesConfig.Documenttype.Datatype.Builder();
-            dataTypeBuilder.id(type.getId());
-            if (type instanceof TemporaryUnknownType) {
-                throw new IllegalArgumentException("Can not create config for temporary data type: " + type.getName());
-            }
-            if (type instanceof OwnedTemporaryType) {
-                throw new IllegalArgumentException("Can not create config for temporary data type: " + type.getName());
-            }
-            if (type instanceof StructDataType) {
-                buildConfig((StructDataType) type, dataTypeBuilder, documentBuilder, built);
-            } else if (type instanceof ArrayDataType) {
-                buildConfig((ArrayDataType) type, dataTypeBuilder, documentBuilder, built);
-            } else if (type instanceof WeightedSetDataType) {
-                buildConfig((WeightedSetDataType) type, dataTypeBuilder, documentBuilder, built);
-            } else if (type instanceof MapDataType) {
-                buildConfig((MapDataType) type, dataTypeBuilder, documentBuilder, built);
-            } else if (type instanceof AnnotationReferenceDataType) {
-                buildConfig((AnnotationReferenceDataType) type, dataTypeBuilder);
-            } else if (type instanceof TensorDataType) {
-                // The type of the tensor is not stored here but instead in each field as detailed type information
-                // to provide better compatibility. A tensor field can have its tensorType changed (in compatible ways)
-                // without changing the field type and thus requiring data refeed
-                return;
-            } else if (type instanceof NewDocumentReferenceDataType) {
-                var refType = (NewDocumentReferenceDataType) type;
-                if (refType.isTemporary()) {
-                    throw new IllegalArgumentException("Still temporary: " + refType);
-                }
-                buildConfig(refType, documentBuilder);
-                return;
-            } else {
-                return;
-            }
-            documentBuilder.datatype(dataTypeBuilder);
-        }
-    }
-
-    private void buildImportedFieldsConfig(Collection<String> fieldNames, DocumenttypesConfig.Documenttype.Builder  builder) {
-        for (String fieldName : fieldNames) {
-            var ib = new DocumenttypesConfig.Documenttype.Importedfield.Builder();
-            ib.name(fieldName);
-            builder.importedfield(ib);
-        }
-    }
-
-    private void buildConfig(StructDataType type,
-                             DocumenttypesConfig.Documenttype.Datatype.Builder dataTypeBuilder,
-                             DocumenttypesConfig.Documenttype.Builder documentBuilder,
-                             Set<Integer> built) {
-        dataTypeBuilder.type(DocumenttypesConfig.Documenttype.Datatype.Type.Enum.STRUCT);
-        DocumenttypesConfig.Documenttype.Datatype.Sstruct.Builder structBuilder = new DocumenttypesConfig.Documenttype.Datatype.Sstruct.Builder();
-        dataTypeBuilder.sstruct(structBuilder);
-        structBuilder.name(type.getName());
-        for (com.yahoo.document.Field field : type.getFields()) {
-            DocumenttypesConfig.Documenttype.Datatype.Sstruct.Field.Builder builder =
-                    new DocumenttypesConfig.Documenttype.Datatype.Sstruct.Field.Builder();
-            builder.name(field.getName()).
-                    id(field.getId()).
-                    datatype(field.getDataType().getId());
-            if (field.getDataType() instanceof TensorDataType) {
-                builder.detailedtype(((TensorDataType) field.getDataType()).getTensorType().toString());
-            }
-            structBuilder.field(builder);
-            buildConfig(field.getDataType(), documentBuilder, built);
-        }
-    }
-
-    private void buildConfig(ArrayDataType type,
-                             DocumenttypesConfig.Documenttype.Datatype.Builder dataTypeBuilder,
-                             DocumenttypesConfig.Documenttype.Builder documentBuilder,
-                             Set<Integer> built) {
-        dataTypeBuilder.
-                type(DocumenttypesConfig.Documenttype.Datatype.Type.Enum.ARRAY).
-                array(new DocumenttypesConfig.Documenttype.Datatype.Array.Builder().
-                        element(new DocumenttypesConfig.Documenttype.Datatype.Array.Element.Builder().id(type.getNestedType().getId())));
-        buildConfig(type.getNestedType(), documentBuilder, built);
-    }
-
-    private void buildConfig(WeightedSetDataType type,
-                             DocumenttypesConfig.Documenttype.Datatype.Builder dataTypeBuilder,
-                             DocumenttypesConfig.Documenttype.Builder documentBuilder,
-                             Set<Integer> built) {
-        dataTypeBuilder.type(DocumenttypesConfig.Documenttype.Datatype.Type.Enum.WSET).
-                wset(new DocumenttypesConfig.Documenttype.Datatype.Wset.Builder().
-                        key(new DocumenttypesConfig.Documenttype.Datatype.Wset.Key.Builder().
-                                id(type.getNestedType().getId())).
-                        createifnonexistent(type.createIfNonExistent()).
-                        removeifzero(type.removeIfZero()));
-        buildConfig(type.getNestedType(), documentBuilder, built);
-    }
-
-    private void buildConfig(MapDataType type,
-                             DocumenttypesConfig.Documenttype.Datatype.Builder dataTypeBuilder,
-                             DocumenttypesConfig.Documenttype.Builder documentBuilder,
-                             Set<Integer> built) {
-        dataTypeBuilder.
-                type(DocumenttypesConfig.Documenttype.Datatype.Type.Enum.MAP).
-                map(new DocumenttypesConfig.Documenttype.Datatype.Map.Builder().
-                        key(new DocumenttypesConfig.Documenttype.Datatype.Map.Key.Builder().
-                                id(type.getKeyType().getId())).
-                        value(new DocumenttypesConfig.Documenttype.Datatype.Map.Value.Builder().
-                                id(type.getValueType().getId())));
-        buildConfig(type.getKeyType(), documentBuilder, built);
-        buildConfig(type.getValueType(), documentBuilder, built);
-    }
-
-    private void buildConfig(AnnotationReferenceDataType type,
-                             DocumenttypesConfig.Documenttype.Datatype.Builder dataTypeBuilder) {
-        dataTypeBuilder.
-                type(DocumenttypesConfig.Documenttype.Datatype.Type.Enum.ANNOTATIONREF).
-                annotationref(new DocumenttypesConfig.Documenttype.Datatype.Annotationref.Builder().
-                        annotation(new DocumenttypesConfig.Documenttype.Datatype.Annotationref.Annotation.Builder().
-                                id(type.getAnnotationType().getId())));
-    }
-
-    private void buildConfig(NewDocumentReferenceDataType type,
-                             DocumenttypesConfig.Documenttype.Builder documentBuilder) {
-        NewDocumentReferenceDataType refType = type;
-        DocumenttypesConfig.Documenttype.Referencetype.Builder refBuilder =
-                new DocumenttypesConfig.Documenttype.Referencetype.Builder();
-        refBuilder.id(refType.getId());
-        refBuilder.target_type_id(type.getTargetTypeId());
-        documentBuilder.referencetype(refBuilder);
-    }
-
-    // Alternate (new) way to build config:
-
-    private DocumenttypesConfig.Builder produceDocTypes(DocumentModel model, DocumenttypesConfig.Builder builder) {
-        builder.usev8geopositions(this.useV8GeoPositions);
-        Map<NewDocumentType.Name, NewDocumentType> produced = new HashMap<>();
-        var indexMap = new IdxMap();
-        for (NewDocumentType documentType : model.getDocumentManager().getTypes()) {
-            docTypeInheritOrder(documentType, builder, produced, indexMap);
-        }
-        indexMap.verifyAllDone();
-        return builder;
-    }
-
     private void docTypeInheritOrder(NewDocumentType documentType,
                                      DocumenttypesConfig.Builder builder,
                                      Map<NewDocumentType.Name, NewDocumentType> produced,
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/clients/Clients.java b/config-model/src/main/java/com/yahoo/vespa/model/clients/Clients.java
index 47bcc64f663..8b8b9e5f40d 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/clients/Clients.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/clients/Clients.java
@@ -13,6 +13,7 @@ import com.yahoo.documentapi.messagebus.loadtypes.LoadTypeSet;
  *
  * @author Gunnar Gauslaa Bergem
  */
+@SuppressWarnings("removal") // TODO: Remove on Vespa 8
 public class Clients extends ConfigModel {
 
     private static final long serialVersionUID = 1L;
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/StorageGroup.java b/config-model/src/main/java/com/yahoo/vespa/model/content/StorageGroup.java
index 88739e92567..b41d805f2a7 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/content/StorageGroup.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/content/StorageGroup.java
@@ -212,8 +212,9 @@ public class StorageGroup {
 
             if (group.isPresent() && nodes.isPresent())
                 throw new IllegalArgumentException("Both <group> and <nodes> is specified: Only one of these tags can be used in the same configuration");
-            if (group.isPresent() && (group.get().stringAttribute("name") != null || group.get().integerAttribute("distribution-key") != null))
-                    deployState.getDeployLogger().logApplicationPackage(Level.INFO, "'distribution-key' attribute on a content cluster's root group is ignored");
+            if (group.isPresent() && (group.get().integerAttribute("distribution-key") != null)) {
+                deployState.getDeployLogger().logApplicationPackage(Level.INFO, "'distribution-key' attribute on a content cluster's root group is ignored");
+            }
 
             GroupBuilder groupBuilder = collectGroup(owner.isHosted(), group, nodes, null, null);
             StorageGroup storageGroup = owner.isHosted()
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java
index ce2bc351d2b..73bd064b967 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java
@@ -62,7 +62,6 @@ import java.util.Set;
 import java.util.TreeMap;
 import java.util.logging.Level;
 
-import static com.yahoo.config.provision.NodeResources.Architecture;
 import static com.yahoo.config.provision.NodeResources.DiskSpeed;
 import static com.yahoo.config.provision.NodeResources.StorageType;
 import static java.util.stream.Collectors.toList;
@@ -337,8 +336,7 @@ public class ContentCluster extends AbstractConfigProducer<AbstractConfigProduce
                                                                                 DeployState deployState,
                                                                                 String clusterName) {
             if (admin.getClusterControllers() == null) {
-                NodeResources nodeResources = clusterControllerResources
-                        .with(Architecture.valueOf(deployState.featureFlags().adminClusterNodeArchitecture()));
+                NodeResources nodeResources = clusterControllerResources.with(deployState.featureFlags().adminClusterArchitecture());
                 NodesSpecification spec = NodesSpecification.requiredFromSharedParents(deployState.zone().environment().isProduction() ? 3 : 1,
                                                                                        nodeResources,
                                                                                        contentElement,