update parser to handle more

* use LinkedHashMap

11 files changed, 76 insertions, 55 deletions

diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedAttribute.java b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedAttribute.java
index 42cd77729ba..b5dabcdb608 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedAttribute.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedAttribute.java
@@ -1,7 +1,7 @@
 // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.searchdefinition.parser;
 
-import java.util.HashMap;
+import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
@@ -21,7 +21,7 @@ class ParsedAttribute extends ParsedBlock {
     private boolean enableHuge = false;
     private boolean enableMutable = false;
     private boolean enablePaged = false;
-    private final Map<String, String> aliases = new HashMap<>();
+    private final Map<String, String> aliases = new LinkedHashMap<>();
     private ParsedSorting sortSettings = null;
     private String distanceMetric = null;
 
@@ -45,10 +45,17 @@ class ParsedAttribute extends ParsedBlock {
         verifyThat(! aliases.containsKey(to), "already has alias", to);
         aliases.put(to, from);
     }
+
     void setDistanceMetric(String value) {
         verifyThat(distanceMetric == null, "already has distance-metric", distanceMetric);
         this.distanceMetric = value;
     }
+
+    ParsedSorting sortInfo() {
+        if (sortSettings == null) sortSettings = new ParsedSorting(name(), "attribute.sorting");
+        return this.sortSettings;
+    }
+
     void setEnableBitVectors(boolean value) { this.enableBitVectors = value; }
     void setEnableOnlyBitVector(boolean value) { this.enableOnlyBitVector = value; }
     void setFastAccess(boolean value) { this.enableFastAccess = true; }
@@ -56,8 +63,4 @@ class ParsedAttribute extends ParsedBlock {
     void setHuge(boolean value) { this.enableHuge = true; }
     void setMutable(boolean value) { this.enableMutable = true; }
     void setPaged(boolean value) { this.enablePaged = true; }
-    ParsedSorting sortInfo() {
-        if (sortSettings == null) sortSettings = new ParsedSorting(name(), "attribute.sorting");
-        return this.sortSettings;
-    }
 }
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedDocument.java b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedDocument.java
index dd61124c3a7..065b66e22b1 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedDocument.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedDocument.java
@@ -4,7 +4,7 @@ package com.yahoo.searchdefinition.parser;
 
 import java.util.ArrayList;
 import java.util.Collection;
-import java.util.HashMap;
+import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
@@ -17,10 +17,10 @@ import java.util.Optional;
  **/
 public class ParsedDocument extends ParsedBlock {
     private final List<String> inherited = new ArrayList<>();
-    private final Map<String, ParsedDocument> resolvedInherits = new HashMap();
-    private final Map<String, ParsedField> docFields = new HashMap<>();
-    private final Map<String, ParsedStruct> docStructs = new HashMap<>();
-    private final Map<String, ParsedAnnotation> docAnnotations = new HashMap<>();
+    private final Map<String, ParsedDocument> resolvedInherits = new LinkedHashMap();
+    private final Map<String, ParsedField> docFields = new LinkedHashMap<>();
+    private final Map<String, ParsedStruct> docStructs = new LinkedHashMap<>();
+    private final Map<String, ParsedAnnotation> docAnnotations = new LinkedHashMap<>();
 
     public ParsedDocument(String name) {
         super(name, "document");
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedDocumentSummary.java b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedDocumentSummary.java
index 65cfa433dc0..08f4946a218 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedDocumentSummary.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedDocumentSummary.java
@@ -2,7 +2,7 @@
 package com.yahoo.searchdefinition.parser;
 
 import java.util.ArrayList;
-import java.util.HashMap;
+import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
 
@@ -17,7 +17,7 @@ class ParsedDocumentSummary extends ParsedBlock {
     private boolean omitSummaryFeatures;
     private boolean fromDisk;
     private final List<String> inherited = new ArrayList<>();
-    private final Map<String, ParsedSummaryField> fields = new HashMap<>();
+    private final Map<String, ParsedSummaryField> fields = new LinkedHashMap<>();
 
     ParsedDocumentSummary(String name) {
         super(name, "document-summary");
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedField.java b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedField.java
index c2998f77b49..5ee73abc28d 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedField.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedField.java
@@ -24,7 +24,7 @@ class ParsedField extends ParsedBlock {
     private boolean isLiteral = false;
     private boolean isNormal = false;
     private Integer weight;
-    private String normalizing;
+    private String normalizing = null;
     private final ParsedMatchSettings matchInfo = new ParsedMatchSettings();
     private Stemming stemming = null;
     private ParsedIndexingOp indexingOp = null;
@@ -59,6 +59,7 @@ class ParsedField extends ParsedBlock {
     String lookupAliasedFrom(String alias) { return aliases.get(alias); }
     ParsedMatchSettings matchSettings() { return this.matchInfo; }
     Optional<Stemming> getStemming() { return Optional.ofNullable(stemming); }
+    Optional<String> getNormalizing() { return Optional.ofNullable(normalizing); }
     Optional<ParsedIndexingOp> getIndexing() { return Optional.ofNullable(indexingOp); }
     Optional<ParsedSorting> getSorting() { return Optional.ofNullable(sortSettings); }
     Map<String, String> getRankTypes() { return Map.copyOf(rankTypes); }
@@ -149,6 +150,9 @@ class ParsedField extends ParsedBlock {
     void addSummaryField(ParsedSummaryField summaryField) {
         String fieldName = summaryField.name();
         verifyThat(! summaryFields.containsKey(fieldName), "already has summary field", fieldName);
+        if (summaryField.getType() == null) {
+            summaryField.setType(getType());
+        }
         summaryFields.put(fieldName, summaryField);
     }
 }
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedFieldSet.java b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedFieldSet.java
index 30597a6dc42..9103ed46631 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedFieldSet.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedFieldSet.java
@@ -2,6 +2,7 @@ package com.yahoo.searchdefinition.parser;
 
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Optional;
 
 /**
  * This class holds the extracted information after parsing a "fieldset"
@@ -13,15 +14,22 @@ class ParsedFieldSet extends ParsedBlock {
 
     private final List<String> fields = new ArrayList<>();
     private final List<String> queryCommands = new ArrayList<>();
-    private final ParsedMatchSettings matchInfo = new ParsedMatchSettings();
+    private ParsedMatchSettings matchInfo = null;
 
     ParsedFieldSet(String name) {
         super(name, "fieldset");
     }
 
-    ParsedMatchSettings matchSettings() { return this.matchInfo; }
+    ParsedMatchSettings matchSettings() {
+        if (matchInfo == null) matchInfo = new ParsedMatchSettings();
+        return this.matchInfo;
+    }
+
     List<String> getQueryCommands() { return List.copyOf(queryCommands); }
     List<String> getFieldNames() { return List.copyOf(fields); }
+    Optional<ParsedMatchSettings> getMatchSettings() {
+        return Optional.ofNullable(this.matchInfo);
+    }
 
     void addField(String field) { fields.add(field); }
     void addQueryCommand(String command) { queryCommands.add(command); }
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedIndex.java b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedIndex.java
index dfc9d911f57..a3504cd2bf7 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedIndex.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedIndex.java
@@ -6,6 +6,7 @@ import com.yahoo.searchdefinition.document.Stemming;
 
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Optional;
 
 /**
  * This class holds the extracted information after parsing an "index"
@@ -15,30 +16,30 @@ import java.util.List;
  **/
 class ParsedIndex extends ParsedBlock {
 
-    private boolean enableBm25 = false;
-    private boolean isPrefix = false;
-    private HnswIndexParams hnswParams;
+    private Boolean enableBm25 = null;
+    private Boolean isPrefix = null;
+    private HnswIndexParams hnswParams = null;
     private final List<String> aliases = new ArrayList<>();
     private Stemming stemming = null;
-    private Integer arity;
-    private Long lowerBound;
-    private Long upperBound;
-    private Double densePLT;
+    private Integer arity = null;
+    private Long lowerBound = null;
+    private Long upperBound = null;
+    private Double densePLT = null;
     
     ParsedIndex(String name) {
         super(name, "index");
     }
 
-    boolean getEnableBm25() { return this.enableBm25; }
-    boolean getPrefix() { return this.isPrefix; }
-    HnswIndexParams getHnswIndexParams() { return this.hnswParams; }
+    Optional<Boolean> getEnableBm25() { return Optional.ofNullable(this.enableBm25); }
+    Optional<Boolean> getPrefix() { return Optional.ofNullable(this.isPrefix); }
+    Optional<HnswIndexParams> getHnswIndexParams() { return Optional.ofNullable(this.hnswParams); }
     List<String> getAliases() { return List.copyOf(aliases); }
     boolean hasStemming() { return stemming != null; }
-    Stemming getStemming() { return stemming; }
-    Integer getArity() { return this.arity; }
-    Long getLowerBound() { return this.lowerBound; }
-    Long getUpperBound() { return this.upperBound; }
-    Double getDensePostingListThreshold() { return this.densePLT; }
+    Optional<Stemming> getStemming() { return Optional.ofNullable(stemming); }
+    Optional<Integer> getArity() { return Optional.ofNullable(this.arity); }
+    Optional<Long> getLowerBound() { return Optional.ofNullable(this.lowerBound); }
+    Optional<Long> getUpperBound() { return Optional.ofNullable(this.upperBound); }
+    Optional<Double> getDensePostingListThreshold() { return Optional.ofNullable(this.densePLT); }
 
     void addAlias(String alias) {
         aliases.add(alias);
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedRankProfile.java b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedRankProfile.java
index ec43d7f5d06..2c9ead4a446 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedRankProfile.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedRankProfile.java
@@ -9,7 +9,7 @@ import com.yahoo.searchlib.rankingexpression.evaluation.Value;
 import com.yahoo.searchlib.rankingexpression.rule.ReferenceNode;
 
 import java.util.ArrayList;
-import java.util.HashMap;
+import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
@@ -42,12 +42,12 @@ class ParsedRankProfile extends ParsedBlock {
     private Boolean strict = null;
     private final List<MutateOperation> mutateOperations = new ArrayList<>();
     private final List<String> inherited = new ArrayList<>();
-    private final Map<String, Boolean> fieldsRankFilter = new HashMap<>();
-    private final Map<String, Integer> fieldsRankWeight = new HashMap<>();
-    private final Map<String, ParsedRankFunction> functions = new HashMap<>();
-    private final Map<String, String> fieldsRankType = new HashMap<>();
-    private final Map<String, List<String>> rankProperties =  new HashMap<>();
-    private final Map<String, Value> constants = new HashMap<>();
+    private final Map<String, Boolean> fieldsRankFilter = new LinkedHashMap<>();
+    private final Map<String, Integer> fieldsRankWeight = new LinkedHashMap<>();
+    private final Map<String, ParsedRankFunction> functions = new LinkedHashMap<>();
+    private final Map<String, String> fieldsRankType = new LinkedHashMap<>();
+    private final Map<String, List<String>> rankProperties = new LinkedHashMap<>();
+    private final Map<String, Value> constants = new LinkedHashMap<>();
 
     ParsedRankProfile(String name) {
         super(name, "rank-profile");
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedSchema.java b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedSchema.java
index a0b238f1f43..bcbf14d9398 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedSchema.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedSchema.java
@@ -6,7 +6,7 @@ import com.yahoo.searchdefinition.RankingConstant;
 import com.yahoo.searchdefinition.document.Stemming;
 
 import java.util.ArrayList;
-import java.util.HashMap;
+import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
 
@@ -29,6 +29,7 @@ public class ParsedSchema extends ParsedBlock {
         }
     }
 
+    private boolean documentWithoutSchema = false;
     private boolean rawAsBase64 = false; // TODO Vespa 8 flip default
     private ParsedDocument myDocument = null;
     private Stemming defaultStemming = null;
@@ -37,20 +38,21 @@ public class ParsedSchema extends ParsedBlock {
     private final List<RankingConstant> rankingConstants = new ArrayList<>();
     private final List<String> inherited = new ArrayList<>();
     private final List<String> inheritedByDocument = new ArrayList<>();
-    private final Map<String, ParsedSchema> resolvedInherits = new HashMap();
-    private final Map<String, ParsedSchema> allResolvedInherits = new HashMap();
-    private final Map<String, ParsedAnnotation> extraAnnotations = new HashMap<>();
-    private final Map<String, ParsedDocumentSummary> docSums = new HashMap<>();
-    private final Map<String, ParsedField> extraFields = new HashMap<>();
-    private final Map<String, ParsedFieldSet> fieldSets = new HashMap<>();
-    private final Map<String, ParsedIndex> extraIndexes = new HashMap<>();
-    private final Map<String, ParsedRankProfile> rankProfiles = new HashMap<>();
-    private final Map<String, ParsedStruct> extraStructs = new HashMap<>();
+    private final Map<String, ParsedSchema> resolvedInherits = new LinkedHashMap();
+    private final Map<String, ParsedSchema> allResolvedInherits = new LinkedHashMap();
+    private final Map<String, ParsedAnnotation> extraAnnotations = new LinkedHashMap<>();
+    private final Map<String, ParsedDocumentSummary> docSums = new LinkedHashMap<>();
+    private final Map<String, ParsedField> extraFields = new LinkedHashMap<>();
+    private final Map<String, ParsedFieldSet> fieldSets = new LinkedHashMap<>();
+    private final Map<String, ParsedIndex> extraIndexes = new LinkedHashMap<>();
+    private final Map<String, ParsedRankProfile> rankProfiles = new LinkedHashMap<>();
+    private final Map<String, ParsedStruct> extraStructs = new LinkedHashMap<>();
 
     public ParsedSchema(String name) {
         super(name, "schema");
     }
 
+    boolean getDocumentWithoutSchema() { return documentWithoutSchema; }
     boolean getRawAsBase64() { return rawAsBase64; }
     boolean hasDocument() { return myDocument != null; }
     ParsedDocument getDocument() { return myDocument; }
@@ -67,7 +69,7 @@ public class ParsedSchema extends ParsedBlock {
     List<RankingConstant> getRankingConstants() { return List.copyOf(rankingConstants); }
     List<String> getInherited() { return List.copyOf(inherited); }
     List<String> getInheritedByDocument() { return List.copyOf(inheritedByDocument); }
-    Map<String, ParsedRankProfile> getRankProfiles() { return Map.copyOf(rankProfiles); }
+    List<ParsedRankProfile> getRankProfiles() { return List.copyOf(rankProfiles.values()); }
     List<ParsedSchema> getResolvedInherits() { return List.copyOf(resolvedInherits.values()); }
     List<ParsedSchema> getAllResolvedInherits() { return List.copyOf(allResolvedInherits.values()); }
 
@@ -85,6 +87,8 @@ public class ParsedSchema extends ParsedBlock {
         this.myDocument = document;
     }
 
+    void setDocumentWithoutSchema() { this.documentWithoutSchema = true; }
+
     void addDocumentSummary(ParsedDocumentSummary docsum) {
         String dsName = docsum.name();
         verifyThat(! docSums.containsKey(dsName), "already has document-summary", dsName);
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedSorting.java b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedSorting.java
index d4686c594da..32c822bf25b 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedSorting.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedSorting.java
@@ -1,6 +1,9 @@
 
 package com.yahoo.searchdefinition.parser;
 
+import com.yahoo.searchdefinition.document.Sorting.Function;
+import com.yahoo.searchdefinition.document.Sorting.Strength;
+
 import java.util.Optional;
 
 /**
@@ -11,10 +14,6 @@ import java.util.Optional;
  **/
 class ParsedSorting extends ParsedBlock {
 
-    enum Function { RAW, LOWERCASE, UCA }
-
-    enum Strength { PRIMARY, SECONDARY, TERTIARY, QUATERNARY, IDENTICAL }
-
     private boolean ascending = true;
     private Function sortFunction = null;
     private Strength sortStrength = null;
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedStruct.java b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedStruct.java
index cc3b2425726..17b20459c9c 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedStruct.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedStruct.java
@@ -2,7 +2,7 @@
 package com.yahoo.searchdefinition.parser;
 
 import java.util.ArrayList;
-import java.util.HashMap;
+import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
 
@@ -14,7 +14,7 @@ import java.util.Map;
  **/
 public class ParsedStruct extends ParsedBlock {
     private final List<String> inherited = new ArrayList<>();
-    private final Map<String, ParsedField> fields = new HashMap<>();
+    private final Map<String, ParsedField> fields = new LinkedHashMap<>();
     private String ownedBy = null;
 
     public ParsedStruct(String name) {
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedType.java b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedType.java
index d3e85bc1b11..3aed90a58e1 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedType.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedType.java
@@ -43,6 +43,8 @@ class ParsedType {
         case "double":    return Variant.BUILTIN;
         case "uri":       return Variant.BUILTIN;
         case "predicate": return Variant.BUILTIN;
+        case "raw":       return Variant.BUILTIN;
+        case "tag":       return Variant.BUILTIN;
         case "position":  return Variant.POSITION;
         }
         return Variant.UNKNOWN;