diff options
author | Jon Bratseth <bratseth@gmail.com> | 2022-05-10 14:53:34 +0200 |
---|---|---|
committer | Jon Bratseth <bratseth@gmail.com> | 2022-05-10 14:53:34 +0200 |
commit | 95339ca1e00f4ecb419a90a765229de65eef55ac (patch) | |
tree | 7e95c94a829c1b54013846661a5a8721d7e21759 /config-model/src/main/javacc/SDParser.jj | |
parent | 14e91114c5f821d16090ed42de51a88482c593ba (diff) |
Remove old parser: Not used by any apps
Diffstat (limited to 'config-model/src/main/javacc/SDParser.jj')
-rw-r--r-- | config-model/src/main/javacc/SDParser.jj | 2912 |
1 files changed, 0 insertions, 2912 deletions
diff --git a/config-model/src/main/javacc/SDParser.jj b/config-model/src/main/javacc/SDParser.jj deleted file mode 100644 index de2ce3d6938..00000000000 --- a/config-model/src/main/javacc/SDParser.jj +++ /dev/null @@ -1,2912 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -// Schema parser. -// -// NOTE: A full rewrite of parsing is in progress; avoid changing this -// file if possible, coordinate with IntermediateParser.jj if you must -// change something. -// -// NOTE: When this is changed, also change integration/intellij/src/main/bnf/ai/vespa/intellij/schema/parser/sd.bnf - -options { - UNICODE_INPUT = true; - CACHE_TOKENS = false; - DEBUG_PARSER = false; - ERROR_REPORTING = true; - FORCE_LA_CHECK = true; - USER_CHAR_STREAM = true; -} - -PARSER_BEGIN(SDParser) - -package com.yahoo.searchdefinition.parser; - -import com.yahoo.document.*; -import com.yahoo.documentmodel.*; -import com.yahoo.compress.Compressor; -import com.yahoo.compress.CompressionType; -import com.yahoo.config.application.api.ApplicationPackage; -import com.yahoo.searchdefinition.DistributableResource; -import com.yahoo.searchdefinition.document.*; -import com.yahoo.searchdefinition.document.annotation.SDAnnotationType; -import com.yahoo.searchdefinition.document.annotation.TemporaryAnnotationReferenceDataType; -import com.yahoo.searchdefinition.RankingConstant; -import com.yahoo.searchdefinition.OnnxModel; -import com.yahoo.searchdefinition.Index; -import com.yahoo.searchdefinition.RankProfile; -import com.yahoo.searchdefinition.DocumentsOnlyRankProfile; -import com.yahoo.searchdefinition.DefaultRankProfile; -import com.yahoo.searchdefinition.RankProfileRegistry; -import com.yahoo.searchdefinition.RankProfile.MatchPhaseSettings; -import com.yahoo.searchdefinition.RankProfile.DiversitySettings; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.DocumentOnlySchema; -import com.yahoo.searchdefinition.UnrankedRankProfile; -import com.yahoo.searchdefinition.fieldoperation.*; -import com.yahoo.searchlib.rankingexpression.FeatureList; -import com.yahoo.searchlib.rankingexpression.Reference; -import com.yahoo.searchlib.rankingexpression.evaluation.Value; -import com.yahoo.searchlib.rankingexpression.evaluation.TensorValue; -import com.yahoo.tensor.Tensor; -import com.yahoo.tensor.TensorType; -import com.yahoo.vespa.documentmodel.DocumentSummary; -import com.yahoo.vespa.documentmodel.SummaryField; -import com.yahoo.vespa.documentmodel.SummaryTransform; -import com.yahoo.config.model.test.MockApplicationPackage; -import com.yahoo.config.application.api.ApplicationPackage; -import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.config.application.api.FileRegistry; -import com.yahoo.config.model.api.ModelContext; -import com.yahoo.language.Linguistics; -import com.yahoo.language.process.Embedder; -import com.yahoo.language.simple.SimpleLinguistics; -import com.yahoo.search.query.ranking.Diversity; -import java.util.Optional; -import java.util.Map; -import java.util.List; -import java.util.ArrayList; -import java.util.LinkedHashMap; -import java.util.logging.Level; - -/** - * The schema parser - * - * @author bratseth - */ -@SuppressWarnings("deprecation") -public class SDParser { - - private DocumentTypeManager docMan = null; - private ApplicationPackage applicationPackage; - private FileRegistry fileRegistry; - private DeployLogger deployLogger; - private ModelContext.Properties properties; - private RankProfileRegistry rankProfileRegistry; - private boolean documentsOnly; - - /** - * Creates a parser - * - * @param documentsOnly true to only parse the document aspect of a schema (e.g skip rank profiles) - */ - public SDParser(SimpleCharStream stream, - ApplicationPackage applicationPackage, - FileRegistry fileRegistry, - DeployLogger deployLogger, - ModelContext.Properties properties, - RankProfileRegistry rankProfileRegistry, - boolean documentsOnly) { - this(stream); - this.applicationPackage = applicationPackage; - this.fileRegistry = fileRegistry; - this.deployLogger = deployLogger; - this.properties = properties; - this.rankProfileRegistry = rankProfileRegistry; - this.documentsOnly = documentsOnly; - } - - /** - * Consumes an indexing language script which will use the simple linguistics implementation - * for testing, by taking input from the current input stream. - * - * @param multiline Whether or not to allow multi-line expressions. - */ - @SuppressWarnings("deprecation") - private IndexingOperation newIndexingOperation(boolean multiline) throws ParseException { - return newIndexingOperation(multiline, new SimpleLinguistics(), Embedder.throwsOnUse.asMap()); - } - - /** - * Consumes an indexing language script from the current input stream. - * - * @param multiline Whether or not to allow multi-line expressions. - * @param linguistics What to use for tokenizing. - */ - private IndexingOperation newIndexingOperation(boolean multiline, Linguistics linguistics, Map<String, Embedder> embedders) throws ParseException { - SimpleCharStream input = (SimpleCharStream)token_source.input_stream; - if (token.next != null) { - input.backup(token.next.image.length()); - } - try { - return IndexingOperation.fromStream(input, multiline, linguistics, embedders); - } finally { - token.next = null; - jj_ntk = -1; - } - } - - /** - * Parses the given token image as a ranking expression feature list. - * - * @param image the token image to parse - * @return the consumed feature list - * @throws ParseException thrown if the image could not be parsed. - */ - private FeatureList getFeatureList(String image) throws ParseException { - try { - return new FeatureList(image); - } - catch (com.yahoo.searchlib.rankingexpression.parser.ParseException e) { - throw (ParseException) new ParseException("Could not parse feature list '" + image + "' at line " + - token_source.input_stream.getBeginLine() + ", column " + - token_source.input_stream.getBeginColumn() + ".").initCause(e); - } - } -} - -PARSER_END(SDParser) - - -// -------------------------------------------------------------------------------- -// -// Token declarations. -// -// -------------------------------------------------------------------------------- - -// Declare white space characters. These do not include newline because it has -// special meaning in several of the production rules. -SKIP : -{ - " " | "\t" | "\r" | "\f" -} - -// Declare all tokens to be recognized. When a word token is added it MUST be -// added to the identifier() production rule. -TOKEN : -{ - < NL: "\n" > -| < ANNOTATION: "annotation" > -| < ANNOTATIONREFERENCE: "annotationreference" > -| < SCHEMA: "schema" > -| < SEARCH: "search" > -| < DIVERSITY: "diversity" > -| < MIN_GROUPS: "min-groups" > -| < CUTOFF_FACTOR: "cutoff-factor" > -| < CUTOFF_STRATEGY: "cutoff-strategy" > -| < LOOSE: "loose" > -| < STRICT: "strict" > -| < DOCUMENT: "document" > -| < OPERATION: "operation" > -| < ON_MATCH: "on-match" > -| < ON_FIRST_PHASE: "on-first-phase" > -| < ON_SECOND_PHASE: "on-second-phase" > -| < ON_SUMMARY: "on-summary" > -| < STRUCT: "struct" > -| < INHERITS: "inherits" > -| < FIELD: "field" > -| < FIELDS: "fields" > -| < FIELDSET: "fieldset" > -| < STRUCTFIELD: "struct-field" > -| < IMPORT: "import" > -| < AS: "as" > -| < INDEXING: "indexing" > -| < SUMMARYTO: "summary-to" > -| < DOCUMENTSUMMARY: "document-summary" > -| <DOUBLE_KEYWORD: "double" > -| < RANKTYPE: "rank-type" > -| < WEIGHT: "weight" > -| < TYPE: "type" > -| < INDEX: "index" > -| < INPUTS: "inputs"> -| < MTOKEN: "token" > -| < TEXT: "text" > -| < WORD: "word" > -| < GRAM: "gram" > -| < GRAMSIZE: "gram-size" > -| < MAXLENGTH: "max-length" > -| < PREFIX: "prefix" > -| < SUBSTRING: "substring" > -| < SUFFIX: "suffix" > -| < CONSTANT: "constant"> -| < ONNXMODEL: "onnx-model"> -| < MODEL: "model" > -| < MUTATE: "mutate" > -| < QUERY: "query" > -| < RANKPROFILE: "rank-profile" > -| < RANKDEGRADATIONFREQ: "rank-degradation-frequency" > -| < RANKDEGRADATION: "rank-degradation" > -| < RAW_AS_BASE64_IN_SUMMARY: "raw-as-base64-in-summary" > -| < RPBINSIZE: "doc-frequency" > -| < RPBINLOW: "min-fullrank-docs"> -| < RPPOSBINSIZE: "occurrences-per-doc" > -| < SUMMARY: "summary" > -| < FULL: "full" > -| < STATIC: "static" > -| < DYNAMIC: "dynamic" > -| < MATCHEDELEMENTSONLY: "matched-elements-only" > -| < SSCONTEXTUAL: "contextual" > -| < SSOVERRIDE: "override" > -| < SSTITLE: "title" > -| < SSURL: "url" > -| < PROPERTIES: "properties" > -| < ATTRIBUTE: "attribute" > -| < SORTING: "sorting" > -| < DICTIONARY: "dictionary" > -| < ASCENDING: "ascending" > -| < DESCENDING: "descending" > -| < UCA: "uca" > -| < RAW: "raw" > -| < LOWERCASE: "lowercase" > -| < FUNCTION: "function" > -| < LOCALE: "locale" > -| < STRENGTH: "strength" > -| < PRIMARY: "primary" > -| < SECONDARY: "secondary" > -| < TERTIARY: "tertiary" > -| < QUATERNARY: "quaternary" > -| < IDENTICAL: "identical" > -| < STEMMING: "stemming" > -| < NORMALIZING: "normalizing" > -| < HASH: "hash" > -| < BTREE: "btree" > -| < CASED: "cased" > -| < UNCASED: "uncased" > -| < BOLDING: "bolding" > -| < BODY: "body" > -| < HEADER: "header" > -| < NONE: "none" > -| < ON: "on" > -| < OFF: "off" > -| < TRUE: "true" > -| < FALSE: "false" > -| < SYMMETRIC: "symmetric" > -| < QUERYCOMMAND: "query-command" > -| < ALIAS: "alias" > -| < MATCH: "match" > -| < RANK: "rank" > -| < LITERAL: "literal" > -| < EXACT: "exact" > -| < FILTER: "filter" > -| < NORMAL: "normal" > -| < EXACTTERMINATOR: "exact-terminator" > -| < INDEXINGREWRITE: "indexing-rewrite" > -| < IGNOREDEFAULTRANKFEATURES: "ignore-default-rank-features" > -| < ID: "id" > -| < SOURCE: "source" > -| < TO: "to" > -| < DIRECT: "direct" > -| < FROMDISK: "from-disk" > -| < OMITSUMMARYFEATURES: "omit-summary-features" > -| < ALWAYS: "always" > -| < ONDEMAND: "on-demand" > -| < NEVER: "never" > -| < ENABLEBITVECTORS: "enable-bit-vectors" > -| < ENABLEONLYBITVECTOR: "enable-only-bit-vector" > -| < FASTACCESS: "fast-access" > -| < MUTABLE: "mutable" > -| < PAGED: "paged" > -| < FASTSEARCH: "fast-search" > -| < HUGE: "huge" > -| < TENSOR_TYPE: "tensor" ("<" (~["<",">"])+ ">")? "(" (~["(",")"])* ")" > -| < TENSOR_VALUE_SL: "value" (" ")* ":" (" ")* ("{"<BRACE_SL_LEVEL_1>) ("\n")? > -| < TENSOR_VALUE_ML: "value" (<SEARCHLIB_SKIP>)? "{" (["\n"," "])* ("{"<BRACE_ML_LEVEL_1>) (["\n"," "])* "}" ("\n")? > -| < COMPRESSION: "compression" > -| < COMPRESSIONLEVEL: "level" > -| < COMPRESSIONTHRESHOLD: "threshold" > -| < LZ4: "lz4" > -| < USEDOCUMENT: "use-document" > -| < LBRACE: "{" > -| < RBRACE: "}" > -| < COLON: ":" > -| < DOT: "." > -| < COMMA: "," > -| < ARRAY: "array" > -| < WEIGHTEDSET: "weightedset" > -| < MAP: "map" > -| < REFERENCE: "reference" > -| < QUESTIONMARK: "?" > -| < CREATEIFNONEXISTENT: "create-if-nonexistent" > -| < REMOVEIFZERO: "remove-if-zero" > -| < MATCHPHASE: "match-phase" > -| < EVALUATION_POINT: "evaluation-point" > -| < PRE_POST_FILTER_TIPPING_POINT: "pre-post-filter-tipping-point" > -| < ORDER: "order" > -| < MAXFILTERCOVERAGE: "max-filter-coverage" > -| < MAXHITS: "max-hits" > -| < FIRSTPHASE: "first-phase" > -| < SECONDPHASE: "second-phase" > -| < MACRO: "macro" > -| < INLINE: "inline" > -| < ARITY: "arity" > -| < LOWERBOUND: "lower-bound" > -| < UPPERBOUND: "upper-bound" > -| < DENSEPOSTINGLISTTHRESHOLD: "dense-posting-list-threshold" > -| < ENABLE_BM25: "enable-bm25" > -| < HNSW: "hnsw" > -| < MAXLINKSPERNODE: "max-links-per-node" > -| < DISTANCEMETRIC: "distance-metric" > -| < NEIGHBORSTOEXPLOREATINSERT: "neighbors-to-explore-at-insert" > -| < MULTITHREADEDINDEXING: "multi-threaded-indexing" > -| < MATCHFEATURES_SL: "match-features" (" ")* ":" (~["}","\n"])* ("\n")? > -| < MATCHFEATURES_ML: "match-features" (<SEARCHLIB_SKIP>)? "{" (~["}"])* "}" > -| < MATCHFEATURES_ML_INHERITS: "match-features inherits " (<IDENTIFIER_WITH_DASH>) (<SEARCHLIB_SKIP>)? "{" (~["}"])* "}" > -| < SUMMARYFEATURES_SL: "summary-features" (" ")* ":" (~["}","\n"])* ("\n")? > -| < SUMMARYFEATURES_ML: "summary-features" (<SEARCHLIB_SKIP>)? "{" (~["}"])* "}" > -| < SUMMARYFEATURES_ML_INHERITS: "summary-features inherits " (<IDENTIFIER_WITH_DASH>) (<SEARCHLIB_SKIP>)? "{" (~["}"])* "}" > -| < RANKFEATURES_SL: "rank-features" (" ")* ":" (~["}","\n"])* ("\n")? > -| < RANKFEATURES_ML: "rank-features" (<SEARCHLIB_SKIP>)? "{" (~["}"])* "}" > -| < EXPRESSION_SL: "expression" (" ")* ":" (("{"<BRACE_SL_LEVEL_1>)|<BRACE_SL_CONTENT>)* ("\n")? > -| < EXPRESSION_ML: "expression" (<SEARCHLIB_SKIP>)? "{" (("{"<BRACE_ML_LEVEL_1>)|<BRACE_ML_CONTENT>)* "}" > -| < #BRACE_SL_LEVEL_1: (("{"<BRACE_SL_LEVEL_2>)|<BRACE_SL_CONTENT>)* "}" > -| < #BRACE_SL_LEVEL_2: (("{"<BRACE_SL_LEVEL_3>)|<BRACE_SL_CONTENT>)* "}" > -| < #BRACE_SL_LEVEL_3: <BRACE_SL_CONTENT> "}" > -| < #BRACE_SL_CONTENT: (~["{","}","\n"])* > -| < #BRACE_ML_LEVEL_1: (("{"<BRACE_ML_LEVEL_2>)|<BRACE_ML_CONTENT>)* "}" > -| < #BRACE_ML_LEVEL_2: (("{"<BRACE_ML_LEVEL_3>)|<BRACE_ML_CONTENT>)* "}" > -| < #BRACE_ML_LEVEL_3: <BRACE_ML_CONTENT> "}" > -| < #BRACE_ML_CONTENT: (~["{","}"])* > -| < #SEARCHLIB_SKIP: ([" ","\f","\n","\r","\t"])+ > -| < RANKPROPERTIES: "rank-properties" > -| < RERANKCOUNT: "rerank-count" > -| < NUMTHREADSPERSEARCH: "num-threads-per-search" > -| < MINHITSPERTHREAD: "min-hits-per-thread" > -| < NUMSEARCHPARTITIONS: "num-search-partitions" > -| < TERMWISELIMIT: "termwise-limit" > -| < POSTFILTERTHRESHOLD: "post-filter-threshold" > -| < APPROXIMATETHRESHOLD: "approximate-threshold" > -| < KEEPRANKCOUNT: "keep-rank-count" > -| < RANKSCOREDROPLIMIT: "rank-score-drop-limit" > -| < CONSTANTS: "constants" > -| < FILE: "file" > -| < URI: "uri" > -| < IDENTIFIER: ["a"-"z","A"-"Z", "_"] (["a"-"z","A"-"Z","0"-"9","_"])* > -| < IDENTIFIER_WITH_DASH: ["a"-"z","A"-"Z", "_"] (["a"-"z","A"-"Z","0"-"9","_","-"])* > -| < QUOTEDSTRING: "\"" ( ~["\""] )* "\"" > -| < CONTEXT: ["a"-"z","A"-"Z"] (["a"-"z", "A"-"Z", "0"-"9"])* > -| < DOUBLE: ("-")? (["0"-"9"])+ "." (["0"-"9"])+ > -| < INTEGER: ("-")? (["0"-"9"])+ > -| < LONG: ("-")? (["0"-"9"])+"L" > -| < STRING: (["a"-"z","A"-"Z","_","0"-"9","."])+ > -| < FILE_PATH: ["a"-"z","A"-"Z", "_"] (["a"-"z","A"-"Z","0"-"9","_","-", "/", "."])+ > -| < HTTP: ["h","H"] ["t","T"] ["t","T"] ["p","P"] (["s","S"])? > -| < URI_PATH: <HTTP> <COLON> ("//")? (["a"-"z","A"-"Z","0"-"9","_","-", "/", ".",":"])+ > -| < LESSTHAN: "<" > -| < GREATERTHAN: ">" > -| < VARIABLE: "$" <IDENTIFIER> > -| < ONNX_INPUT_SL: "input" (" ")* (<IDENTIFIER>|<QUOTEDSTRING>) (" ")* ":" (" ")* (~["\n"])* ("\n")? > -| < ONNX_OUTPUT_SL: "output" (" ")* (<IDENTIFIER>|<QUOTEDSTRING>) (" ")* ":" (" ")* (~["\n"])* ("\n")? > -} - -// Declare a special skip token for comments. -SPECIAL_TOKEN : -{ - <SINGLE_LINE_COMMENT: "#" (~["\n","\r"])* > -} - - -// -------------------------------------------------------------------------------- -// -// Production rules. -// -// -------------------------------------------------------------------------------- - -/** - * The rule consumes any schema and returns the corresponding object. This is the only production that should - * ever consume leading newlines. - * - * @return the schema object - */ -Schema schema(DocumentTypeManager docMan) : -{ - this.docMan = docMan; - Schema schema; -} -{ - (<NL>)* (schema = rootSchema() | schema = rootDocument()) - { return schema; } -} - -/** - * This rule consumes a proper schema block. This and rootDocument() are the only rules that should ever consume - * trailing newline tokens. - * - * @return the schema definition object. - */ -Schema rootSchema() : -{ - String name; - String inherited = null; - Schema schema; -} -{ - ( ( <SCHEMA> | <SEARCH> ) name = identifier() (<INHERITS> inherited = identifier() )? { - schema = new Schema(name, applicationPackage, Optional.ofNullable(inherited), fileRegistry, deployLogger, properties); - rankProfileRegistry.add(new DefaultRankProfile(schema, rankProfileRegistry, schema.rankingConstants())); - rankProfileRegistry.add(new UnrankedRankProfile(schema, rankProfileRegistry, schema.rankingConstants()));} - lbrace() (rootSchemaItem(schema) (<NL>)*)* <RBRACE> (<NL>)* <EOF>) - { return schema; } -} - -/** - * Consumes an element of a schema block. This and rootSearch() are the only rules that should ever consume - * trailing newline tokens. - * - * @param schema the schema object to modify. - */ -void rootSchemaItem(Schema schema) : { } -{ - ( document(schema) - | rawAsBase64(schema) - | documentSummary(schema) - | field(null, schema) - | index(schema, null) - | rankingConstant(schema) - | rankProfile(schema) - | searchStemming(schema) - | useDocument(schema) - | structOutside(schema) - | annotationOutside(schema) - | fieldSet(schema) - | importField(schema) - | onnxModel(schema) ) -} - -/** - * Consumes a schema definition that contains only documents to be used for inheritance, etc. - * - * @return the schema definition object. - */ -Schema rootDocument() : -{ - Schema schema = new DocumentOnlySchema(applicationPackage, fileRegistry, deployLogger, properties); -} -{ - ( (rootDocumentItem(schema) (<NL>)*)*<EOF> ) - { return schema; } -} - -/** - * Consumes a single item from within a root document node. - * - * @param schema the schema object to modify. - */ -void rootDocumentItem(Schema schema) : { } -{ - ( namedDocument(schema) ) -} - -/** - * Consumes a use-document statement. This currently does nothing. - * - * @param schema the schema object to modify. - */ -void useDocument(Schema schema) : { } -{ - <USEDOCUMENT> <COLON> identifier() - { deployLogger.logApplicationPackage(Level.WARNING, "Specifying 'use-document' is deprecated and has no effect."); } -} - -/** - * Consumes a document element. The name defaults to the schema's name, but may be set. - * - * @param schema the schema object to add content to. - */ -void document(Schema schema) : -{ - String name = schema.getName(); - SDDocumentType document; -} -{ - ( <DOCUMENT> (name = identifier())? (<NL>)* { document = new SDDocumentType(name, schema); } - [ inheritsDocument(document) (<NL>)* ] - <LBRACE> (<NL>)* (documentBody(document, schema) (<NL>)*)* <RBRACE> ) - { - schema.addDocument(document); - } -} - -/** - * Consumes a document element, explicitly named - * - * @param schema the schema object to add content to - */ -void namedDocument(Schema schema) : -{ - String name; - SDDocumentType document; -} -{ - ( <DOCUMENT> name = identifier() (<NL>)* { document = new SDDocumentType(name, schema); } - [ inheritsDocument(document) (<NL>)* ] - <LBRACE> (<NL>)* (documentBody(document, schema) (<NL>)*)* <RBRACE> ) - { - schema.addDocument(document); - } -} - -/** - * Consumes a document body block - * - * @param document the document type to modify. - * @param schema the schema object to add content to - */ -void documentBody(SDDocumentType document, Schema schema) : -{ -} -{ - ( annotation(schema, document) - | compression(document) - | headercfg(document) - | bodycfg(document) - | structInside(document, schema) - | field(document, schema) ) -} - -void rawAsBase64(Schema schema) : -{ - boolean enabled = false; -} -{ - <RAW_AS_BASE64_IN_SUMMARY> - { - enabled = true; - } - [ <COLON> ( <TRUE> | ( <FALSE> { enabled = false; } ) ) ] - { - schema.enableRawAsBase64(enabled); - } -} - -/** - * Consumes a document head block. - * - * @param document The document type to modify. - */ -void headercfg(SDDocumentType document) : { } -{ - <HEADER> lbrace() [compression(document) (<NL>)*] <RBRACE> -} - -/** - * Consumes a document body block. - * - * @param document The document type to modify. - */ -void bodycfg(SDDocumentType document) : { } -{ - <BODY> lbrace() [compression(document) (<NL>)*] <RBRACE> -} - -/** - * Consumes a compression block. This can be set in both document header and -body block. - * - * @param document The document type to modify. - */ -void compression(SDDocumentType document) : -{ - deployLogger.logApplicationPackage(Level.WARNING, "'compression' for a document is deprecated and ignored"); -} -{ - <COMPRESSION> lbrace() ( compressionItem() (<NL>)*)* <RBRACE> { } -} - -/** - * Consumes the body of a compression block. - * - */ -void compressionItem() : -{ } -{ - ( ( <TYPE> <COLON> <LZ4> ) - | (<COMPRESSIONTHRESHOLD> <COLON> <INTEGER>) - | (<COMPRESSIONLEVEL> <COLON> <INTEGER>) - ) { } -} - -/** - * Consumes a document inheritance statement. - * - * @param document The document type to modify. - */ -void inheritsDocument(SDDocumentType document) : -{ - String name; -} -{ - <INHERITS> name = identifier() { document.inherit(new DataTypeName(name)); } - ( <COMMA> name = identifier() { document.inherit(new DataTypeName(name)); } )* -} - -/** - * Consumes a field block from within a document element. - * - * @param document the document type to modify - * @param schema the schema object to add content to - */ -void field(SDDocumentType document, Schema schema) : -{ - String name; - SDField field; - DataType type; -} -{ - <FIELD> name = identifier() <TYPE> type = dataType() - { - if (name != null && Schema.isReservedName(name.toLowerCase())) { - throw new IllegalArgumentException("Reserved name '" + name + "' can not be used as a field name."); - } - field = new TemporarySDField(document == null ? schema.getDocument() : document, name, type, document); - } - lbrace() (fieldBody(field, schema, document) (<NL>)*)* <RBRACE> - { - if (document != null) { - document.addField(field); - } else { - schema.addExtraField(field); - } - } -} - -void fieldSet(Schema schema) : -{ - String setName; - String field; - String queryCommand; - List queryCommands = new ArrayList(); - FieldOperationContainer matchSetting; - List matchSettings = new ArrayList(); -} -{ - <FIELDSET> setName = identifier() lbrace() - (( - ( <FIELDS><COLON> field = identifier() { schema.fieldSets().addUserFieldSetItem(setName, field); } - ( <COMMA> field = identifier() { schema.fieldSets().addUserFieldSetItem(setName, field); } )* ) - | - ( <QUERYCOMMAND> <COLON> (queryCommand = identifierWithDash() | queryCommand = quotedString())) { queryCommands.add(queryCommand); } - | - ( matchSetting = match(new SDField(null, setName, DataType.STRING)) ) { matchSettings.add(matchSetting); } - )(<NL>)*)+ - <RBRACE> - { - // Apply settings after parsing since all user field items must be set first - - for (Object command : queryCommands) - schema.fieldSets().userFieldSets().get(setName).queryCommands().add((String)command); - - for (Object setting : matchSettings) { - ((SDField)setting).applyOperations(); - schema.fieldSets().userFieldSets().get(setName).setMatching(((SDField)setting).getMatching()); - } - } -} - -/** - * This rule consumes a annotation block from within either a document element or a schema element. - - * @param schema the schema object to add content to - */ -void annotationOutside(Schema schema) : -{ - String name; - SDAnnotationType type; -} -{ - <ANNOTATION> name = identifier() - { - type = new SDAnnotationType(name.trim()); - if (schema.getDocument() == null) { - throw new IllegalArgumentException("Can't add annotation '"+name+"' to a document type, define a document type first or declare the annotation inside of one."); - } - } - [ inheritsAnnotation(type) (<NL>)* ] - lbrace() (type = annotationBody(schema, schema.getDocument(), type)) <RBRACE> - { - schema.addAnnotation(type); - } -} - -/** - * Consumes a annotation block from within either a document element. - * - * @param document the document object to add content to - */ -void annotation(Schema schema, SDDocumentType document) : -{ - String name; - SDAnnotationType type; -} -{ - <ANNOTATION> name = identifier() - { - type = new SDAnnotationType(name.trim()); - } - [ inheritsAnnotation(type) (<NL>)* ] - lbrace() (type = annotationBody(schema, document, type)) <RBRACE> - { - document.addAnnotation(type); - } -} - - -/** - * Consumes a single element of an annotation body block. - * - * @param schema the schema object to add content to - * @param type the type being built - * @return a modified or new AnnotationType instance - */ -SDAnnotationType annotationBody(Schema schema, SDDocumentType repo, SDAnnotationType type) : -{ - SDDocumentType struct = new SDDocumentType("annotation." + type.getName(), schema); -} -{ - (structFieldDefinition(repo, struct) (<NL>)*)* - { - if (struct.getFieldCount() > 0) { // Must account for the temporary TemporarySDField. - type = new SDAnnotationType(type.getName(), struct, type.getInherits()); - struct.setStruct(null); - } - return type; - } -} - -void inheritsAnnotation(SDAnnotationType annotation) : -{ - String name; -} -{ - <INHERITS> name = identifier() { annotation.inherit(name); } -} - - -/** - * This rule consumes a struct block from within a document element. - * - * @param schema the schema object to add content to - */ -void structInside(SDDocumentType document, Schema schema) : -{ - SDDocumentType struct; -} -{ - ( - struct = structDefinition(schema, document) - ) - { - document.addType(struct); - } -} - -/** - * This rule consumes a struct block from within a document element. - * - * @param schema the schema object to add content to - */ -void structOutside(Schema schema) : -{ - SDDocumentType struct; -} -{ - ( - struct = structDefinition(schema, schema.getDocument()) - ) - { - schema.addType(struct); - } -} - -/** - * This rule consumes a struct block from within a document element. - * - * @param schema the schema object to add content to - */ -SDDocumentType structDefinition(Schema schema, SDDocumentType repo) : -{ - String name; - String inherited = null; - SDDocumentType struct; -} -{ - ( <STRUCT> name = identifier() (<NL>)* { - if (repo == null) - throw new IllegalArgumentException("Can't add struct '"+name+"' to a document type, define a document type first or declare the struct inside of one."); - struct = new SDDocumentType(name, schema); - } - [ inheritsDocument(struct) (<NL>)* ] - lbrace() (structFieldDefinition(repo, struct) (<NL>)*)* <RBRACE> ) - { - try { - docMan.getDataType(name); - throw new ParseException("Reserved name '" + name + "' can not be used to declare a struct."); - } catch (IllegalArgumentException e) { - // empty - } - SDDocumentType sdtype = repo.getOwnedType(struct.getDocumentName()); - if (sdtype != null) { - throw new ParseException("Struct '" + name + "' is already defined."); - } - DataType stype = new OwnedTemporaryType(name, repo.getName()); - struct.setStruct(stype); - return struct; - } -} - -/** - * This rule consumes a data type block from within a field element. - * - * @return the consumed data type - */ -DataType dataType() : -{ - String typeName = null; - boolean isArrayOldStyle = false; - DataType mapType = null; - DataType arrayType = null; - DataType wsetType = null; - TensorType tensorType; - String referencedDoc; -} -{ - ( LOOKAHEAD(<ARRAY> <LESSTHAN>) ( <ARRAY> <LESSTHAN> arrayType = dataType() <GREATERTHAN> { return DataType.getArray(arrayType); } ) - | LOOKAHEAD(<WEIGHTEDSET> <LESSTHAN>) ( <WEIGHTEDSET> <LESSTHAN> wsetType = dataType() <GREATERTHAN> { return DataType.getWeightedSet(wsetType); } ) - | LOOKAHEAD(<MAP> <LESSTHAN>) ( mapType = mapDataType() { return mapType; } ) - | LOOKAHEAD(<ANNOTATIONREFERENCE> <LESSTHAN>) ( mapType = annotationRefDataType() { return mapType; } ) - | LOOKAHEAD(<TENSOR_TYPE>) ( tensorType = tensorType("Field type") { return DataType.getTensor(tensorType); } ) - | LOOKAHEAD(<REFERENCE>) ( <REFERENCE> <LESSTHAN> referencedDoc = identifier() <GREATERTHAN> { return NewDocumentReferenceDataType.forDocumentName(referencedDoc); } ) - | ( typeName = identifier() ["[]" { isArrayOldStyle = true; }] ) - ) - { - DataType type = VespaDocumentType.INSTANCE.getDataType(typeName); - - if (type == null) { - // we are basically creating TemporaryUnknownType instances for ANYTHING here!! - // we must do this and clean them up later. - type = new TemporaryUnknownType(typeName); - } - - if (isArrayOldStyle) { - deployLogger.logApplicationPackage(Level.WARNING, "Data type syntax '" + typeName + "[]' is deprecated, use 'array<" + typeName + ">' instead."); - type = DataType.getArray(type); - } - if ("tag".equalsIgnoreCase(typeName) && type instanceof WeightedSetDataType) ((WeightedSetDataType)type).setTag(true); - return type; - } -} - -DataType annotationRefDataType() : -{ - DataType dataType; - String targetName; -} -{ - ( <ANNOTATIONREFERENCE> <LESSTHAN> targetName = identifier() <GREATERTHAN> ) - { - return new TemporaryAnnotationReferenceDataType(targetName); - } -} - -DataType mapDataType() : -{ - DataType keyType; - DataType valType; -} -{ - ( <MAP> <LESSTHAN> keyType = dataType() <COMMA> valType = dataType() <GREATERTHAN> ) - { - return DataType.getMap(keyType, valType); - } - -} - -/* Note: not currently used, remove when decided that map type will not support -polymorphism */ -DataType wildCardType() : -{ -} -{ -(<QUESTIONMARK>) { return DataType.NONE; } -} - -/** - * This rule consumes a field block of a struct body. - * - * @param struct The struct to modify. - */ -void structFieldDefinition(SDDocumentType document, SDDocumentType struct) : -{ - String name; - SDField field; - DataType type; -} -{ - <FIELD> name = identifier() <TYPE> type = dataType() { - if (name != null && Schema.isReservedName(name.toLowerCase())) { - throw new IllegalArgumentException("Reserved name '" + name + "' can not be used as a field name."); - } - field = new TemporarySDField(document, name, type, struct); - struct.addField(field); - } - lbrace() (id(field,struct) (<NL>)*)? (match(field) (<NL>)*)* <RBRACE> { - } -} - -/** - * This rule consumes a struct subfield from a document field body. This is not to be confused with a document - * struct's fields, but rather this is a subfield of a document field of type struct. - * - * @param field the field to modify - * @param schema the schema object to add content to - * @param document the document type to modify - */ -void structField(FieldOperationContainer field, Schema schema, SDDocumentType document) : -{ - String name; - SDField structField; -} -{ - <STRUCTFIELD> name = identifier() { - if (name != null && Schema.isReservedName(name.toLowerCase())) { - throw new IllegalArgumentException("Reserved name '" + name + "' can not be used as a field name."); - } - FieldOperationContainer structFieldOp = new StructFieldOperation(name); - field.addOperation((StructFieldOperation) structFieldOp); - } - lbrace() (structFieldBody(structFieldOp, schema, document) (<NL>)*)* <RBRACE> -} - - -/** - * This rule consumes a single element of a field body block. - * - * @param field the field being built - * @param schema the schema object to add content to - * @param document the owning document, or null if this is a search field - */ -void fieldBody(SDField field, Schema schema, SDDocumentType document) : { } -{ - ( alias(field) | - attribute(field) | - body(field) | - bolding(field) | - dictionary(field) | - fieldStemming(field) | - header(field) | - id(field, document) | - summaryInField(field) | - index(schema, field) | - indexing(field) | - indexingRewrite(field) | - match(field) | - normalizing(field) | - queryCommand(field) | - rank(field) | - rankType(field) | - sorting(field, field.getName()) | - structField(field, schema, document) | - summaryTo(field) | - weight(field) | - weightedset(field) ) -} - -/** - * This rule consumes a single element of a struct subfield body block. - * Only elements that are supported in streaming schema and indexed schema (with complex attributes) are allowed. - * - * @param field the field being built - * @param schema the schema object to add content to - * @param document the owning document, or null if this is a schema field - */ -void structFieldBody(FieldOperationContainer field, Schema schema, SDDocumentType document) : { } -{ - ( summaryInField(field) | - indexing(field) | - attribute(field) | - match(field) | - queryCommand(field) | - structField(field, schema, document) | - summaryTo(field) ) -} - -/** - * This rule consumes an indexing block of a field element. - * - * @param field The field to modify. - */ -void indexing(FieldOperationContainer field) : { } -{ - ( <INDEXING> ( (<COLON> indexingOperation(field, false)) | indexingOperation(field, true) ) ) -} - -/** - * This rule consumes an IL script block. This is expected to consume trailing newlines. - * - * @param field The field to modify. - */ -void indexingOperation(FieldOperationContainer field, boolean multiLine) : { } -{ - { field.addOperation(newIndexingOperation(multiLine)); } -} - -/** - * This rule consumes a summary-to statement of a field element. - * - * @param field The field to modify. - */ -void summaryTo(FieldOperationContainer field) : -{ - SummaryToOperation op = new SummaryToOperation(); - String destination; - String name = field.getName(); -} -{ - <SUMMARYTO> [name = identifier()] <COLON> destination = identifier() - { - op.setName(name); - op.addDestination(destination); - } - ( <COMMA> destination = identifier() {op.addDestination(destination); } )* - { - field.addOperation(op); - } -} - - -/** - * This rule consumes a weight statement of a field element. - * - * @param field The field to modify. - */ -void weight(FieldOperationContainer field) : -{ - int num; -} -{ - <WEIGHT> <COLON> num = integer() - { - WeightOperation op = new WeightOperation(); - op.setWeight(num); - field.addOperation(op); - } -} - -/** - * This rule consumes a weighted set statement of a field element. - * - * @param field The field to modify. - */ -void weightedset(FieldOperationContainer field) : -{ - WeightedSetOperation op = new WeightedSetOperation(); -} -{ - <WEIGHTEDSET> ( (<COLON> weightedsetBody(op)) - | (lbrace() (weightedsetBody(op) (<NL>)*)* <RBRACE>) ) - { - field.addOperation(op); - } -} - -/** - * This rule consumes one body item of a weighted set block. - * - * @param field The field to modify. - */ -void weightedsetBody(WeightedSetOperation field) : { } -{ - ( <CREATEIFNONEXISTENT> { field.setCreateIfNonExistent(true); } - | <REMOVEIFZERO> { field.setRemoveIfZero(true); } ) -} - -/** - * This rule consumes a rank-type statement of a field element. - * - * @param field The field to modify. - */ -void rankType(FieldOperationContainer field) : -{ - String typeName; - String indexName = null; -} -{ - <RANKTYPE> [indexName = identifier()] <COLON> typeName = identifier() - { - RankTypeOperation op = new RankTypeOperation(); - op.setType(RankType.fromString(typeName)); - op.setIndexName(indexName); - field.addOperation(op); - } -} - -/** - * This rule consumes an attribute statement of a field element. - * - * @param field The field to modify. - */ -void attribute(FieldOperationContainer field) : -{ - String name = field.getName(); -} -{ - <ATTRIBUTE> [name = identifier()] - { - AttributeOperation op = new AttributeOperation(name); - } - ( (<COLON> attributeSetting(field, op, name)) - | (lbrace() (attributeSetting(field, op, name) (<NL>)*)* <RBRACE>) ) - { - field.addOperation(op); - } -} - -void sorting(FieldOperationContainer field, String name) : -{ - SortingOperation op = new SortingOperation(name); -} -{ - <SORTING> - ( (<COLON> sortingSetting(op, name)) - | (lbrace() (sortingSetting(op, name) (<NL>)*)* <RBRACE>) ) - { - field.addOperation(op); - } -} - -void sortingSetting(SortingOperation sorting, String attributeName) : -{ - String locale; -} -{ - ( - <ASCENDING> { sorting.setAscending(); } - | <DESCENDING> { sorting.setDescending(); } - | <FUNCTION> <COLON> ( - <UCA> { sorting.setFunction(Sorting.Function.UCA); } - | <RAW> { sorting.setFunction(Sorting.Function.RAW); } - | <LOWERCASE> { sorting.setFunction(Sorting.Function.LOWERCASE); } - ) - | <STRENGTH> <COLON> ( - <PRIMARY> { sorting.setStrength(Sorting.Strength.PRIMARY); } - | <SECONDARY> { sorting.setStrength(Sorting.Strength.SECONDARY); } - | <TERTIARY> { sorting.setStrength(Sorting.Strength.TERTIARY); } - | <QUATERNARY> { sorting.setStrength(Sorting.Strength.QUATERNARY); } - | <IDENTICAL> { sorting.setStrength(Sorting.Strength.IDENTICAL); } - ) - | <LOCALE> <COLON> locale = identifierWithDash() { sorting.setLocale(locale); } - ) -} - -/** - * This rule consumes a single attribute setting statement of an attribute element. - * - * @param field The field to modify. - * @param attributeName The name of the attribute to change. - */ -void attributeSetting(FieldOperationContainer field, AttributeOperation attribute, String attributeName) : -{ - String str; -} -{ - ( - <HUGE> { attribute.setHuge(true); } - | <FASTSEARCH> { attribute.setFastSearch(true); } - | <FASTACCESS> { attribute.setFastAccess(true); } - | <MUTABLE> { attribute.setMutable(true); } - | <PAGED> { attribute.setPaged(true); } - | <ENABLEBITVECTORS> { attribute.setEnableBitVectors(true); } - | <ENABLEONLYBITVECTOR> { attribute.setEnableOnlyBitVector(true); } - | sorting(field, attributeName) - | <ALIAS> { String alias; String aliasedName=attributeName; } [aliasedName = identifier()] <COLON> alias = identifierWithDash() { - attribute.setDoAlias(true); - attribute.setAlias(alias); - attribute.setAliasedName(aliasedName); - } - | attributeTensorType(attribute) - | <DISTANCEMETRIC> <COLON> str = identifierWithDash() { attribute.setDistanceMetric(str); } - ) -} - -/** - * This rule consumes a tensor type statement for an attribute element. - * - * @param attribute The attribute to modify. - */ -void attributeTensorType(AttributeOperation attribute) : -{ - TensorType tensorType; -} -{ - tensorType = tensorType("For attribute field '" + attribute.getName() + "'") - { - // TODO: Remove on Vespa 8 - deployLogger.logApplicationPackage(Level.WARNING, "In field '" + attribute.getName() + "': Specifying tensor type on the attribute is deprecated and has no effect."); - } -} - -/** - * This rule consumes a summary statement defined inside a document-summary block. - * - * @param document The document summary to modify. - */ -void summaryInDocument(DocumentSummary document) : -{ - String name; - DataType type; - SummaryField summary; - -} -{ - <SUMMARY> name = identifierWithDash() { } - <TYPE> type = dataType() { - summary = new SummaryField(name, type); - summary.setVsmCommand(SummaryField.VsmCommand.FLATTENSPACE); - - SummaryInFieldLongOperation op = new SummaryInFieldLongOperation(); - } - lbrace() (summaryItem(op) (<NL>)*)* <RBRACE> - { - if (op.destinationIterator().hasNext()) { - throw new ParseException("Summaries defined in a document-summary section " + - "can not have a 'to' line."); - } - op.applyToSummary(summary); - document.add(summary); - } -} - -/** - * The rule consumes a summary statement defined inside a field. - * - * @param field The field to modify. - */ -void summaryInField(FieldOperationContainer field) : -{ - SummaryInFieldOperation summary; -} -{ - ( <SUMMARY> ( LOOKAHEAD(2) summary = summaryInFieldShort(field) - | summary = summaryInFieldLong(field)) ) - { - field.addOperation(summary); - } -} - -/** - * This rule consumes a single-line summary field. - * - * @param field The field to modify. - * @return The consumed summary field. - */ -SummaryInFieldOperation summaryInFieldShort(FieldOperationContainer field) : -{ - String name = field.getName(); - SummaryField ret; -} -{ - [ name = identifier() ] - { - SummaryInFieldShortOperation op = new SummaryInFieldShortOperation(name); - } - <COLON> ( <DYNAMIC> { op.setTransform(SummaryTransform.DYNAMICTEASER); - op.addSource(name); - } - | <MATCHEDELEMENTSONLY> { op.setTransform(SummaryTransform.MATCHED_ELEMENTS_FILTER); } - | (<FULL> | <STATIC>) { op.setTransform(SummaryTransform.NONE); } ) - { return op; } -} - -/** - * This rule consumes a multi-line summary field. - * - * @return The consumed summary field. - */ -SummaryInFieldOperation summaryInFieldLong(FieldOperationContainer field) : -{ - String name = field.getName(); - DataType type = null; -} -{ - ( [ name = identifier() [ <TYPE> type = dataType() ] ] - lbrace() - { - SummaryInFieldLongOperation op = new SummaryInFieldLongOperation(name); - op.setType(type); - } - (summaryItem(op) (<NL>)*)* <RBRACE> ) - { return op; } -} - -/** - * This rule consumes an item of a summary field block. - * - * @param field The field to modify. - */ -void summaryItem(SummaryInFieldLongOperation field) : { } -{ - ( summaryTransform(field) - | summaryBolding(field) - | summarySourceList(field) - | summaryDestinationList(field) - | summaryProperties(field) ) -} - -/** - * This rule consumes a transform statement for a summary field element. - * - * @param field The field to modify. - */ -void summaryTransform(SummaryInFieldOperation field) : { } -{ - ( <DYNAMIC> { field.setTransform(SummaryTransform.DYNAMICTEASER); } - | <MATCHEDELEMENTSONLY> { field.setTransform(SummaryTransform.MATCHED_ELEMENTS_FILTER); } - | (<FULL> | <STATIC>) { field.setTransform(SummaryTransform.NONE); } ) -} - -/** - * This rule consumes a bolding statement for a summary field element. - * - * @param field The summary field to modify. - */ -void summaryBolding(SummaryInFieldLongOperation field) : -{ - boolean bold; -} -{ - <BOLDING> <COLON> bold = bool() - { field.setBold(bold); } -} - -/** - * This rule consumes a source-list statement for a summary field element. - * - * @param field The summary field to modify. - */ -void summarySourceList(SummaryInFieldOperation field) : -{ - String str; -} -{ - ( <SOURCE> <COLON> str = identifier() { field.addSource(str); } - ( <COMMA> str = identifier() { field.addSource(str); } )* ) + -} - -/** - * This rule consumes a destination-list statement for a summary field element. - * - * @param field The summary field to modify. - */ -void summaryDestinationList(SummaryInFieldLongOperation field) : -{ - String str; -} -{ - <TO> <COLON> str = identifier() { field.addDestination(str); } - ( <COMMA> str = identifier() { field.addDestination(str); } )* -} - -/** - * This rule consumes properties for a summary field element. - * - * @param field The summary field to modify. - */ -void summaryProperties(SummaryInFieldLongOperation field) : { } -{ - <PROPERTIES> lbrace() (summaryProperty(field) <NL>)+ <RBRACE> -} - -/** - * This rule consumes a single summary property pair for a summary field element. - * - * @param field The summary field to modify. - */ -void summaryProperty(SummaryInFieldLongOperation field) : -{ - String name, value; -} -{ - name = identifierWithDash() <COLON> (value = identifierWithDash() | value = quotedString()) - { deployLogger.logApplicationPackage(Level.WARNING, "Specifying summary properties is deprecated and has no effect."); } -} - -/** - * This rule consumes a stemming block of a field element. - * - * @param field The field to modify. - */ -void fieldStemming(FieldOperationContainer field) : -{ - String setting; - StemmingOperation op = new StemmingOperation(); -} -{ - <STEMMING> <COLON> setting = identifierWithDash() - { - op.setSetting(setting); - field.addOperation(op); - } -} - -/** - * This rule consumes a stemming statement for a schema element. - * - * @param schema the schema to modify - */ -void searchStemming(Schema schema) : -{ - String setting; -} -{ - <STEMMING> <COLON> setting = identifierWithDash() - { schema.setStemming(Stemming.get(setting)); } -} - -/** - * This rule consumes a normalizing statement of a field element. At the moment, this can only be used to turn off - * normalizing. - * - * @param field The field to modify. - */ -void normalizing(FieldOperationContainer field) : -{ - String setting; -} -{ - <NORMALIZING> <COLON> setting = identifierWithDash() - { - field.addOperation(new NormalizingOperation(setting)); - } -} - -/** - * This rule consumes a bolding statement of a field element. - * - * @param field The field to modify. - */ -void bolding(FieldOperationContainer field) : -{ - boolean bold; -} -{ - <BOLDING> <COLON> bold = bool() - { - field.addOperation(new BoldingOperation(bold)); - } -} - -/** - * This rule consumes a dictionary statement of a field element. - * - * @param field The field to modify. - */ -void dictionary(FieldOperationContainer field) : -{ -} -{ - <DICTIONARY> - ( (<COLON> dictionarySetting(field)) - | (lbrace() (dictionarySetting(field) (<NL>)*)* <RBRACE>)) - { - } -} - -void dictionarySetting(FieldOperationContainer field) : -{ - Dictionary.Type type; -} -{ - ( <HASH> { field.addOperation(new DictionaryOperation(DictionaryOperation.Operation.HASH)); } - | <BTREE> { field.addOperation(new DictionaryOperation(DictionaryOperation.Operation.BTREE)); } - | <CASED> { field.addOperation(new DictionaryOperation(DictionaryOperation.Operation.CASED)); } - | <UNCASED> { field.addOperation(new DictionaryOperation(DictionaryOperation.Operation.UNCASED)); }) - { - } -} - -/** - * This rule consumes a body statement of a field element. - * - * @param field The field to modify. - */ -void body(SDField field) : { } -{ - <BODY> - { - deployLogger.logApplicationPackage(Level.WARNING, field + ": 'header/body' is deprecated and has no effect."); - } -} - -/** - * This rule consumes a header statement of a field element. - * - * @param field The field to modify. - */ -void header(SDField field) : { } -{ - <HEADER> - { - deployLogger.logApplicationPackage(Level.WARNING, field + ": 'header/body' is deprecated and has no effect."); - } -} - -void queryCommand(FieldOperationContainer container) : -{ - String command; - QueryCommandOperation field = new QueryCommandOperation(); -} -{ - <QUERYCOMMAND> <COLON> ( command = identifierWithDash() | command = quotedString() ) - { - field.addQueryCommand(command); - container.addOperation(field); - } -} - -void alias(FieldOperationContainer container) : -{ - String aliasedName = null; - String alias; -} -{ - <ALIAS> [aliasedName = identifier()] <COLON> alias = identifierWithDash() - { - AliasOperation op = new AliasOperation(aliasedName, alias); - container.addOperation(op); - } -} - -FieldOperationContainer match(FieldOperationContainer field) : { } -{ - <MATCH> ( (<COLON> matchType(field)) - | (lbrace() (matchItem(field) (<NL>)*)* <RBRACE>) ) - { return field; } -} - -/** - * This rule consumes a single match item for a match block. - * - * @param field The field to modify. - */ -void matchItem(FieldOperationContainer field) : { } -{ - ( matchType(field) | exactTerminator(field) | gramSize(field) | matchSize(field) ) -} - -void matchType(FieldOperationContainer container) : -{ - MatchOperation matchOp = new MatchOperation(); -} -{ - ( <MTOKEN> { matchOp.setMatchingType(MatchType.TEXT); } // Deprecated synonym to TEXT - | <TEXT> { matchOp.setMatchingType(MatchType.TEXT); } - | <WORD> { matchOp.setMatchingType(MatchType.WORD); } - | <EXACT> { matchOp.setMatchingType(MatchType.EXACT); } - | <GRAM> { matchOp.setMatchingType(MatchType.GRAM); } - | <CASED> { matchOp.setCase(Case.CASED); } - | <UNCASED> { matchOp.setCase(Case.UNCASED); } - | <PREFIX> { matchOp.setMatchingAlgorithm(MatchAlgorithm.PREFIX); } - | <SUBSTRING> { matchOp.setMatchingAlgorithm(MatchAlgorithm.SUBSTRING); } - | <SUFFIX> { matchOp.setMatchingAlgorithm(MatchAlgorithm.SUFFIX); } ) - { - container.addOperation(matchOp); - } -} - -void exactTerminator(FieldOperationContainer container) : -{ - String terminator; - MatchOperation field = new MatchOperation(); -} -{ - <EXACTTERMINATOR> <COLON> terminator = quotedString() - { - field.setExactMatchTerminator(terminator); - container.addOperation(field); - } -} - -void gramSize(FieldOperationContainer container) : -{ - int gramSize; - MatchOperation field = new MatchOperation(); -} -{ - <GRAMSIZE> <COLON> gramSize = integer() - { - field.setGramSize(gramSize); - container.addOperation(field); - } -} - -void matchSize(FieldOperationContainer container) : -{ - int matchSize; - MatchOperation field = new MatchOperation(); -} -{ - <MAXLENGTH> <COLON> matchSize = integer() - { - field.setMaxLength(matchSize); - container.addOperation(field); - } -} -/** - * Consumes a rank statement of a field element. - * - * @param field The field to modify. - */ -void rank(FieldOperationContainer field) : -{ - RankOperation op = new RankOperation(); -} -{ - <RANK> ( (<COLON> rankSetting(op)) - | (lbrace() (rankSetting(op) (<NL>)*)* <RBRACE>) ) - { - field.addOperation(op); - } -} - -/** - * Consumes a single rank setting of a rank statement. - * - * @param field The field to modify. - */ -void rankSetting(RankOperation field) : { } -{ - ( <LITERAL> { field.setLiteral(true); } - | <NORMAL> { field.setNormal(true); } - | <FILTER> { field.setFilter(true); } ) -} - -/** - * Consumes an id statement of a field body block. - * - * @param field The field to modify. - * @param document The document type to modify. - */ -void id(FieldOperationContainer field, SDDocumentType document) : -{ - int fieldId; - IdOperation op = new IdOperation(); -} -{ - <ID> <COLON> fieldId = integer() - { - op.setDocument(document); - op.setFieldId(fieldId); - field.addOperation(op); - } -} - -/** - * Consumes an indexing-rewrite statement of a field body block. - * - * @param field The field to modify. - */ -void indexingRewrite(FieldOperationContainer field) : { } -{ - <INDEXINGREWRITE> <COLON> <NONE> - { deployLogger.logApplicationPackage(Level.WARNING, "Specifying 'indexing-rewrite' is deprecated and has no effect."); } -} - -/** - * Consumes a document-summary block from within a schema block. - * - * @param schema the schema object to add content to - */ -void documentSummary(Schema schema) : -{ - String name; - DocumentSummary summary; -} -{ - ( <DOCUMENTSUMMARY> - name = identifierWithDash() { schema.addSummary(summary = new DocumentSummary(name, schema)); } - [inheritsDocumentSummary(summary, schema)] - lbrace() - ( - <FROMDISK> { summary.setFromDisk(true); } | - <OMITSUMMARYFEATURES> { summary.setOmitSummaryFeatures(true); } | - documentSummaryItem(summary) | - <NL> - )* - <RBRACE> - ) -} - -/** - * This rule consumes an inherits statement of a document summary. - * - * @param documentSummary the document summary to modify - * @param schema the schema object documentSummary is being added to - */ -void inheritsDocumentSummary(DocumentSummary documentSummary, Schema schema) : -{ - String name; -} -{ - <INHERITS> name = identifierWithDash() - { - documentSummary.setInherited(name); - } -} - -/** - * Consumes a single document-summary item. - * - * @param summary The document summary to modify. - */ -void documentSummaryItem(DocumentSummary summary) : { } -{ - summaryInDocument(summary) -} - -/** - * Consumes an index block for a field element. - * - * @param schema the schema object to add content to - * @param field the field to modify - */ -void index(Schema schema, FieldOperationContainer field) : -{ - IndexOperation op = new IndexOperation(); - String indexName = (field != null) ? field.getName() : null; -} -{ - <INDEX> [indexName = identifier()] - { - if (indexName == null) { - throw new ParseException("Index statements outside fields must have an explicit name."); - } - op.setIndexName(indexName); - } - ( (<COLON> indexBody(op) (<COMMA> indexBody(op))*) | - (lbrace() (indexBody(op) (<NL>)*)* <RBRACE>) ) - { - if (field == null) { - - Index index = new Index(indexName); - op.applyToIndex(index); - schema.addIndex(index); - } else { - field.addOperation(op); - } - } -} - -/** - * Consumes a single index statement for an index block. - * - * @param index The index to modify. - */ -void indexBody(IndexOperation index) : -{ - String str; - int arity; - long num; - double threshold; -} -{ - ( <PREFIX> { index.setPrefix(true); } - | <ALIAS> <COLON> str = identifierWithDash() { index.addAlias(str); } - | <STEMMING> <COLON> str = identifierWithDash() { index.setStemming(str); } - | <ARITY> <COLON> arity = integer() { index.setArity(arity); } - | <LOWERBOUND> <COLON> num = consumeLong() { index.setLowerBound(num); } - | <UPPERBOUND> <COLON> num = consumeLong() { index.setUpperBound(num); } - | <DENSEPOSTINGLISTTHRESHOLD> <COLON> threshold = consumeFloat() { index.setDensePostingListThreshold(threshold); } - | <ENABLE_BM25> { index.setEnableBm25(true); } - | hnswIndex(index) { } - ) -} - -void hnswIndex(IndexOperation index) : -{ - HnswIndexParams.Builder params = new HnswIndexParams.Builder(); -} -{ - ( LOOKAHEAD(<HNSW> lbrace()) - <HNSW> ( (lbrace() (hnswIndexBody(params) (<NL>)*)* <RBRACE>) ) | - <HNSW> ) - { - index.setHnswIndexParams(params); - } -} - -void hnswIndexBody(HnswIndexParams.Builder params) : -{ - int num; - boolean bool; -} -{ - ( <MAXLINKSPERNODE> <COLON> num = integer() { params.setMaxLinksPerNode(num); } - | <NEIGHBORSTOEXPLOREATINSERT> <COLON> num = integer() { params.setNeighborsToExploreAtInsert(num); } - | <MULTITHREADEDINDEXING> <COLON> bool = bool() { params.setMultiThreadedIndexing(bool); } ) -} - -/** - * Consumes a onnx-model block of a schema element. - * - * @param schema the schema object to add content to. - */ -void onnxModel(Schema schema) : -{ - String name; - OnnxModel onnxModel; -} -{ - ( <ONNXMODEL> name = identifier() - { - onnxModel = new OnnxModel(name); - } - lbrace() (onnxModelItem(onnxModel) (<NL>)*)+ <RBRACE> ) - { - if (documentsOnly) return; - schema.onnxModels().add(onnxModel); - } -} - -/** - * This rule consumes an onnx-model block. - * - * @param onnxModel The onnxModel to modify. - */ -void onnxModelItem(OnnxModel onnxModel) : -{ - String path = null; -} -{ - ( - (path = fileItem()) { onnxModel.setFileName(path); } | - (path = uriItem()) { onnxModel.setUri(path); } | - (<ONNX_INPUT_SL>) { - String name = token.image.substring(5, token.image.lastIndexOf(":")).trim(); - if (name.startsWith("\"")) { name = name.substring(1, name.length() - 1); } - String source = token.image.substring(token.image.lastIndexOf(":") + 1).trim(); - onnxModel.addInputNameMapping(name, source); - } | - (<ONNX_OUTPUT_SL>) { - String name = token.image.substring(6, token.image.lastIndexOf(":")).trim(); - if (name.startsWith("\"")) { name = name.substring(1, name.length() - 1); } - String as = token.image.substring(token.image.lastIndexOf(":") + 1).trim(); - onnxModel.addOutputNameMapping(name, as); - } - ) -} - -/** - * Consumes a constant block of a schema element. - * - * @param schema the schema object to add content to - */ -void rankingConstant(Schema schema) : -{ - String name; - String path = null; - DistributableResource.PathType pathType = DistributableResource.PathType.FILE; - TensorType type = null; -} -{ - ( <CONSTANT> name = identifier() lbrace() - (path = fileItem() { pathType = DistributableResource.PathType.FILE; } - | path = uriItem() { pathType = DistributableResource.PathType.URI; } - | type = tensorTypeWithPrefix(rankingConstantErrorMessage(name)) (<NL>)* - )+ - <RBRACE> - ) - { - if (documentsOnly) return; - schema.rankingConstants().add(new RankingConstant(name, type, path, pathType)); - } -} - -String fileItem() : -{ - String path; -} -{ - (<FILE> <COLON> ( <FILE_PATH> | <STRING> | <IDENTIFIER>) { path = com.yahoo.path.Path.fromString(token.image).getRelative(); } { } (<NL>)*) { return path; } -} -String uriItem() : -{ - String path; -} -{ - (<URI> <COLON> ( <URI_PATH> ) { path = token.image; } (<NL>)*) { return path; } -} - -String rankingConstantErrorMessage(String name) : {} -{ - { return "For ranking constant ' " + name + "'"; } -} - -/** - * Consumes a rank-profile block of a schema element. - * - * @param schema the schema object to add content to - */ -void rankProfile(Schema schema) : -{ - String name; - RankProfile profile; -} -{ - ( ( <MODEL> | <RANKPROFILE> ) name = identifierWithDash() - { - if (documentsOnly) { - profile = new DocumentsOnlyRankProfile(name, schema, rankProfileRegistry, schema.rankingConstants()); - } - else if ("default".equals(name)) { - profile = rankProfileRegistry.get(schema, "default"); - } else { - profile = new RankProfile(name, schema, rankProfileRegistry, schema.rankingConstants()); - } - } - [inheritsRankProfile(profile)] - lbrace() (rankProfileItem(profile) (<NL>)*)* <RBRACE> ) - { - if (documentsOnly) return; - rankProfileRegistry.add(profile); - } -} - -/** - * This rule consumes a single statement for a rank-profile block. - * - * @param profile The rank profile to modify. - */ -void rankProfileItem(RankProfile profile) : { } -{ - ( fieldRankType(profile) - | fieldWeight(profile) - | fieldRankFilter(profile) - | firstPhase(profile) - | matchPhase(profile) - | function(profile) - | mutate(profile) - | ignoreRankFeatures(profile) - | numThreadsPerSearch(profile) - | minHitsPerThread(profile) - | numSearchPartitions(profile) - | termwiseLimit(profile) - | postFilterThreshold(profile) - | approximateThreshold(profile) - | rankFeatures(profile) - | rankProperties(profile) - | secondPhase(profile) - | rankDegradation(profile) - | inputs(profile) - | constants(profile) - | matchFeatures(profile) - | summaryFeatures(profile) - | strict(profile) ) -} - -/** - * This rule consumes an inherits statement of a rank-profile. - * - * @param profile the profile to modify - */ -void inheritsRankProfile(RankProfile profile) : -{ - String name; -} -{ - <INHERITS> name = identifierWithDash() { profile.inherit(name); } - ( <COMMA> name = identifierWithDash() { profile.inherit(name);; } )* -} - -/** - * This rule consumes an mutate statement of a rank-profile. - * - * @param profile The profile to modify. - */ -void mutate(RankProfile profile) : -{ -} -{ - <MUTATE> lbrace() (mutate_operation(profile) <NL>)+ <RBRACE> - { } -} - -void mutate_operation(RankProfile profile) : -{ - String attribute, operation; - RankProfile.MutateOperation.Phase phase; -} -{ - ( <ON_MATCH> { phase = RankProfile.MutateOperation.Phase.on_match; } - | <ON_FIRST_PHASE> { phase = RankProfile.MutateOperation.Phase.on_first_phase; } - | <ON_SECOND_PHASE> { phase = RankProfile.MutateOperation.Phase.on_second_phase; } - | <ON_SUMMARY> { phase = RankProfile.MutateOperation.Phase.on_summary; } - ) - lbrace() attribute = identifier() operation = mutate_expr() (<NL>)* <RBRACE> - { profile.addMutateOperation(phase, attribute, operation); } -} - -String mutate_expr() : -{ - String op; - Number constant = null; -} -{ - (("+=" | "-=" | "=") { op = token.image; } constant = consumeNumber()) - { return constant != null ? (op + constant) : op; } -} - -/** - * This rule consumes a function statement of a rank-profile. - * - * @param profile The profile to modify. - */ -void function(RankProfile profile) : -{ - String name, expression, parameter; - List parameters = new ArrayList(); - boolean inline = false; -} -{ - ( ( <FUNCTION> | <MACRO> ) inline = inline() name = identifier() [ "$" { name = name + token.image; } ] - "(" - [ parameter = identifier() { parameters.add(parameter); } - ( <COMMA> parameter = identifier() { parameters.add(parameter); } )* ] - ")" - lbrace() expression = expression() (<NL>)* <RBRACE> ) - { profile.addFunction(name, parameters, expression, inline); } -} - -boolean inline() : -{ -} -{ - ( <INLINE> { return true; } ) ? - { return false; } -} - -/** - * This rule consumes a match-phase block of a rank profile. - * - * @param profile The rank profile to modify. - */ -void matchPhase(RankProfile profile) : -{ - MatchPhaseSettings settings = new MatchPhaseSettings(); -} -{ - <MATCHPHASE> lbrace() (matchPhaseItem(settings) (<NL>)*)* <RBRACE> - { - settings.checkValid(); - profile.setMatchPhaseSettings(settings); - } -} - -void matchPhaseItem(MatchPhaseSettings settings) : -{ - String str; - int num; - double multiplier; - double coverage; -} -{ - ( <ATTRIBUTE> <COLON> str = identifier() { settings.setAttribute(str); } - | diversity(settings) - | <ORDER> <COLON> ( <ASCENDING> { settings.setAscending(true); } - | <DESCENDING> { settings.setAscending(false); } ) - | <MAXHITS> <COLON> num = integer() { settings.setMaxHits(num); } - | <MAXFILTERCOVERAGE> <COLON> coverage = consumeFloat() { settings.setMaxFilterCoverage(coverage); } - | <EVALUATION_POINT> <COLON> multiplier = consumeFloat() { settings.setEvaluationPoint(multiplier); } - | <PRE_POST_FILTER_TIPPING_POINT> <COLON> multiplier = consumeFloat() { settings.setPrePostFilterTippingPoint(multiplier); } - ) - { return; } -} - -/** - * This rule consumes a diversity block of a rank profile. - * - * @param profile The rank profile to modify. - */ -void diversity(MatchPhaseSettings profile) : -{ - DiversitySettings settings = new DiversitySettings(); -} -{ - <DIVERSITY> lbrace() (diversityItem(settings) (<NL>)*)* <RBRACE> - { - profile.setDiversity(settings); - } -} - -void diversityItem(DiversitySettings settings) : -{ - String str; - int num; - double multiplier; -} -{ - ( <ATTRIBUTE> <COLON> str = identifier() { settings.setAttribute(str); } - | <MIN_GROUPS> <COLON> num = integer() { settings.setMinGroups(num); } - | <CUTOFF_FACTOR> <COLON> multiplier = consumeFloat() { settings.setCutoffFactor(multiplier); } - | <CUTOFF_STRATEGY> <COLON> - ( <STRICT> { settings.setCutoffStrategy(Diversity.CutoffStrategy.strict); } - | <LOOSE> { settings.setCutoffStrategy(Diversity.CutoffStrategy.loose); } - ) - ) - { return; } -} - - - -/** - * Consumes the first-phase block of a rank profile. - * - * @param profile The rank profile to modify. - */ -void firstPhase(RankProfile profile) : -{ - String exp; -} -{ - <FIRSTPHASE> lbrace() (firstPhaseItem(profile) (<NL>)*)* <RBRACE> -} - -void firstPhaseItem(RankProfile profile) : -{ - String expression; - int rerankCount; - double dropLimit; -} -{ - ( expression = expression() { profile.setFirstPhaseRanking(expression); } - | (<KEEPRANKCOUNT> <COLON> rerankCount = integer()) { profile.setKeepRankCount(rerankCount); } - | (<RANKSCOREDROPLIMIT> <COLON> dropLimit = consumeFloat()) { profile.setRankScoreDropLimit(dropLimit); } - ) -} - -/** - * Consumes the second-phase block of a rank profile. - * - * @param profile The rank profile to modify. - */ -void secondPhase(RankProfile profile) : { } -{ - <SECONDPHASE> lbrace() (secondPhaseItem(profile) (<NL>)*)* <RBRACE> -} - -/** - * Consumes a statement for a second-phase block. - * - * @param profile The rank profile to modify. - */ -void secondPhaseItem(RankProfile profile) : -{ - String expression; - int rerankCount; -} -{ - ( expression = expression() { profile.setSecondPhaseRanking(expression); } - | (<RERANKCOUNT> <COLON> rerankCount = integer()) { profile.setRerankCount(rerankCount); } - ) -} - -/** Consumes an inputs block of a rank profile. */ -void inputs(RankProfile profile) : {} -{ - <INPUTS> <LBRACE> (<NL>)* - ( input(profile) (<NL>)*) * - <RBRACE> -} - -void input(RankProfile profile) : -{ - Reference reference; - TensorType type; -} -{ - reference = queryFeature() type = inputType(reference) - { - profile.addInput(reference, new RankProfile.Input(reference, type, Optional.empty())); - } -} - -TensorType inputType(Reference reference) : -{ - TensorType type; -} -{ - ( - ( type = tensorType("Type of " + reference) ) - | - ( <DOUBLE_KEYWORD> { type = TensorType.empty; } ) - ) - { return type; } -} - -Reference queryFeature() : -{ - String argument; -} -{ - <QUERY> "(" argument = identifier() ")" - { return Reference.simple("query", argument); } -} - -/** Consumes a summary-features block of a rank profile. */ -void summaryFeatures(RankProfile profile) : -{ - String features; - String inherited = null; -} -{ - ( <SUMMARYFEATURES_SL> { features = token.image.substring(token.image.indexOf(":") + 1).trim(); } | - <SUMMARYFEATURES_ML> { features = token.image.substring(token.image.indexOf("{") + 1, - token.image.lastIndexOf("}")).trim(); } | - <SUMMARYFEATURES_ML_INHERITS> { - int inheritsIndex = token.image.indexOf("inherits "); - String rest = token.image.substring(inheritsIndex + "inherits ".length()); - profile.setInheritedSummaryFeatures(rest.substring(0, rest.indexOf(" ")).trim()); - features = token.image.substring(token.image.indexOf("{") + 1, token.image.lastIndexOf("}")).trim(); - } - ) - { - profile.addSummaryFeatures(getFeatureList(features)); - } -} - -void strict(RankProfile profile) : -{} -{ - <STRICT> <COLON> ( - ( <TRUE> { profile.setStrict(true); } ) | - ( <FALSE> { profile.setStrict(false); } ) - ) -} - -/** - * This rule consumes a match-features block of a rank profile. - * - * @param profile The rank profile to modify. - */ -void matchFeatures(RankProfile profile) : -{ - String features; -} -{ - ( <MATCHFEATURES_SL> { features = token.image.substring(token.image.indexOf(":") + 1).trim(); } | - <MATCHFEATURES_ML> { features = token.image.substring(token.image.indexOf("{") + 1, - token.image.lastIndexOf("}")).trim(); } | - <MATCHFEATURES_ML_INHERITS> { - int inheritsIndex = token.image.indexOf("inherits "); - String rest = token.image.substring(inheritsIndex + "inherits ".length()); - profile.setInheritedMatchFeatures(rest.substring(0, rest.indexOf(" ")).trim()); - features = token.image.substring(token.image.indexOf("{") + 1, token.image.lastIndexOf("}")).trim(); - } - ) - { - profile.addMatchFeatures(getFeatureList(features)); - } -} - -/** Consumes a rank-features block of a rank profile */ -void rankFeatures(RankProfile profile) : -{ - String features; -} -{ - ( <RANKFEATURES_SL> { features = token.image.substring(token.image.indexOf(":") + 1).trim(); } | - <RANKFEATURES_ML> { features = token.image.substring(token.image.indexOf("{") + 1, - token.image.lastIndexOf("}")).trim(); } ) - { - profile.addRankFeatures(getFeatureList(features)); - } -} - -/** - * This rule consumes a ignore-default-rank-features statement for a rank profile. - * - * @param profile The rank profile to modify. - */ -void ignoreRankFeatures(RankProfile profile) : { } -{ - <IGNOREDEFAULTRANKFEATURES> { profile.setIgnoreDefaultRankFeatures(true); } -} - -/** - * This rule consumes a num-threads-per-search statement for a rank profile. - * - * @param profile The rank profile to modify. - */ -void numThreadsPerSearch(RankProfile profile) : -{ - int num; -} -{ - (<NUMTHREADSPERSEARCH> <COLON> num = integer()) { profile.setNumThreadsPerSearch(num); } -} - -/** - * This rule consumes a min-hits-per-thread statement for a rank profile. - * - * @param profile The rank profile to modify. - */ -void minHitsPerThread(RankProfile profile) : -{ - int num; -} -{ - (<MINHITSPERTHREAD> <COLON> num = integer()) { profile.setMinHitsPerThread(num); } -} - -/** - * This rule consumes a num-search-partitions statement for a rank profile. - * - * @param profile the rank profile to modify - */ -void numSearchPartitions(RankProfile profile) : -{ - int num; -} -{ - (<NUMSEARCHPARTITIONS> <COLON> num = integer()) { profile.setNumSearchPartitions(num); } -} - -/** - * This rule consumes a termwise-limit statement for a rank profile. - * - * @param profile the rank profile to modify - */ -void termwiseLimit(RankProfile profile) : -{ - double num; -} -{ - (<TERMWISELIMIT> <COLON> num = consumeFloat()) { profile.setTermwiseLimit(num); } -} - -/** - * This rule consumes a post-filter-threshold statement for a rank profile. - * - * @param profile the rank profile to modify - */ -void postFilterThreshold(RankProfile profile) : -{ - double threshold; -} -{ - (<POSTFILTERTHRESHOLD> <COLON> threshold = consumeFloat()) { profile.setPostFilterThreshold(threshold); } -} - -/** - * This rule consumes an approximate-threshold statement for a rank profile. - * - * @param profile the rank profile to modify - */ -void approximateThreshold(RankProfile profile) : -{ - double threshold; -} -{ - (<APPROXIMATETHRESHOLD> <COLON> threshold = consumeFloat()) { profile.setApproximateThreshold(threshold); } -} - -/** - * This rule consumes a rank-properties block of a rank profile. There is a little trick within this rule to allow the - * final rank property to skip the terminating newline token. - * - * @param profile the rank profile to modify - */ -void rankProperties(RankProfile profile) : { } -{ - <RANKPROPERTIES> lbrace() (LOOKAHEAD(rankPropertyItem() <COLON> rankPropertyItem() <NL>) - rankProperty(profile) (<NL>)+)* [rankProperty(profile)] <RBRACE> -} - -/** - * This rule consumes a single rank property pair for a rank profile. - * - * @param profile the rank profile to modify - */ -void rankProperty(RankProfile profile) : -{ - String key, val; -} -{ - key = rankPropertyItem() <COLON> val = rankPropertyItem() - { profile.addRankProperty(key, val); } -} - - -/** - * This rule consumes a single rank property for a rank-properties block. - * - * @return The token image of the consumed item. - */ -String rankPropertyItem() : -{ - String image; - String ret = ""; - Token dToken; -} -{ - ( ( image = identifierWithDash() { ret += image; } - | dToken = <DOUBLE> { ret += dToken.image; } - | image = quotedString() { ret += image; } - | ( "(" | ")" | <DOT> | <COMMA> ) { ret += token.image; } )+ ) - { return ret; } -} - -/** - * This rule consumes a field-weight statement of a rank profile. - * - * @param profile The rank profile to modify. - */ -void fieldWeight(RankProfile profile) : -{ - Integer num; - String name; -} -{ - <WEIGHT> name = identifier() <COLON> num = integer() - { profile.addRankSetting(name, RankProfile.RankSetting.Type.WEIGHT, num); } -} - -/** - * This rule consumes a rank-type statement of a rank profile. - * - * @param profile The rank profile to modify. - */ -void fieldRankType(RankProfile profile) : -{ - String name; - String type; -} -{ - <RANKTYPE> name = identifier() <COLON> type = identifier() - { profile.addRankSetting(name, RankProfile.RankSetting.Type.RANKTYPE, RankType.fromString(type)); } -} - -/** - * This rule consumes a rank filter statement of a rank profile. - * - * @param profile The rank profile to modify. - */ -void fieldRankFilter(RankProfile profile) : -{ - String name; -} -{ - <RANK> name = identifier() <COLON> <FILTER> - { profile.addRankSetting(name, RankProfile.RankSetting.Type.PREFERBITVECTOR, Boolean.TRUE); } -} - -/** - * This rule consumes part of a rank-degradation statement of a rank profile. - */ -void rankDegradationBinSize() : -{ - double freq; -} -{ - <RPBINSIZE> <COLON> freq = consumeFloat() - { deployLogger.logApplicationPackage(Level.WARNING, "Specifying 'doc-frequency' in 'rank-degradation' is deprecated and has no effect."); } -} - - -/** - * This rule consumes part of a rank-degradation statement of a rank profile. - */ -void rankDegradationBinLow() : -{ - int n; -} -{ - <RPBINLOW> <COLON> n = integer() - { deployLogger.logApplicationPackage(Level.WARNING, "Specifying 'min-fullrank-docs' in 'rank-degradation' is deprecated and has no effect."); } -} - - -/** - * This rule consumes part of a rank-degradation statement of a rank profile. - */ -void rankDegradationPosbinSize() : -{ - double avgOcc; -} -{ - <RPPOSBINSIZE> <COLON> avgOcc = consumeFloat() - { deployLogger.logApplicationPackage(Level.WARNING, "Specifying 'occurrences-per-doc' in 'rank-degradation' is deprecated and has no effect."); } -} - - -/** - * This rule consumes part of a rank-degradation statement of a rank profile. - */ -void rankDegradationItem() : -{ -} -{ - ( rankDegradationBinSize() - | rankDegradationBinLow() - | rankDegradationPosbinSize() ) -} - -/** - * This rule consumes a rank-degradation statement of a rank profile. - * - * @param profile The rank profile to modify. - */ -void rankDegradation(RankProfile profile) : -{ - double freq; -} -{ - ( <RANKDEGRADATIONFREQ> <COLON> freq = consumeFloat() - { deployLogger.logApplicationPackage(Level.WARNING, "Specifying 'rank-degradation-frequency' in 'rank-profile' is deprecated and has no effect."); } - | <RANKDEGRADATION> lbrace() ( rankDegradationItem() (<NL>)*)+ <RBRACE> - ) -} - -/** Consumes a set of constants available in ranking expressions in the enclosing profile. */ -void constants(RankProfile profile) : -{ - String name; -} -{ - <CONSTANTS> <LBRACE> (<NL>)* - ( name = identifier() ( constantValue(profile, name) | - constantTensor(profile, name) ) (<NL>)* )* - <RBRACE> -} - -void constantValue(RankProfile profile, String name) : -{ - Token value; -} -{ - <COLON> ( value = <DOUBLE> | value = <INTEGER> | value = <IDENTIFIER> ) { profile.addConstant(name, Value.parse(value.image)); } -} - -void constantTensor(RankProfile profile, String name) : -{ - String tensorString = ""; - TensorType tensorType = null; -} -{ - <LBRACE> (<NL>)* - (( tensorString = tensorValue() | - tensorType = tensorTypeWithPrefix(constantTensorErrorMessage(profile.name(), name)) ) (<NL>)* )* <RBRACE> - { - if (tensorType != null) { - profile.addConstantTensor(name, new TensorValue(Tensor.from(tensorType, tensorString))); - } else { - profile.addConstantTensor(name, new TensorValue(Tensor.from(tensorString))); - } - } -} - -String constantTensorErrorMessage(String rankProfileName, String constantTensorName) : {} -{ - { return "For constant tensor '" + constantTensorName + "' in rank profile '" + rankProfileName + "'"; } -} - -String tensorValue() : -{ - String tensor; -} -{ - ( <TENSOR_VALUE_SL> { tensor = token.image.substring(token.image.indexOf(":") + 1); } | - <TENSOR_VALUE_ML> { tensor = token.image.substring(token.image.indexOf("{") + 1, - token.image.lastIndexOf("}")); } ) - { - return tensor; - } -} - -TensorType tensorTypeWithPrefix(String errorMessage) : -{ TensorType type; } -{ - <TYPE> <COLON> type= tensorType(errorMessage) - { return type; } -} - -TensorType tensorType(String errorMessage) : -{ - String tensorTypeString; -} -{ - <TENSOR_TYPE> { tensorTypeString = token.image; } - { - TensorType tensorType; - try { - tensorType = TensorType.fromSpec(tensorTypeString); - } catch (IllegalArgumentException e) { - throw new IllegalArgumentException(errorMessage + ": Illegal tensor type spec: " + e.getMessage()); - } - return tensorType; - } -} - -void importField(Schema schema) : -{ - String fieldRefSpec; - String aliasFieldName; -} -{ - <IMPORT> <FIELD> fieldRefSpec = identifier() <AS> aliasFieldName = identifier() lbrace() - <RBRACE> - { - long nDots = Utils.count(fieldRefSpec, '.'); - if (nDots != 1) { - throw new IllegalArgumentException("Illegal field reference spec '" + fieldRefSpec + "': Does not include a single '.'"); - } - int indexOfDot = fieldRefSpec.indexOf('.'); - String documentReferenceFieldName = fieldRefSpec.substring(0, indexOfDot); - String foreignFieldName = fieldRefSpec.substring(indexOfDot + 1); - TemporaryImportedFields importedFields = schema.temporaryImportedFields().get(); - if (importedFields.hasField(aliasFieldName)) { - throw new IllegalArgumentException("For schema '" + schema.getName() + "', import field as '" + aliasFieldName + "': Field already imported"); - } - importedFields.add(new TemporaryImportedField(aliasFieldName, documentReferenceFieldName, foreignFieldName)); - } -} - - -/** - * This rule consumes an expression token and returns its image. - * - * @return The consumed token image. - */ -String expression() : -{ - String exp; -} -{ - ( <EXPRESSION_SL> { exp = token.image.substring(token.image.indexOf(":") + 1); } | - <EXPRESSION_ML> { exp = token.image.substring(token.image.indexOf("{") + 1, - token.image.lastIndexOf("}")); } ) - { return exp; } -} - -String identifierWithDash() : -{ - String identifier; -} -{ - ( identifier = identifier() { return identifier; } ) - | - ( <IDENTIFIER_WITH_DASH> { return token.image; } ) -} - -/** - * Consumes an identifier. This must be kept in sync with all word tokens that should be parseable as - * identifiers. - * - * @return the identifier string - */ -String identifier() : { } -{ - ( <ALIAS> - | <ALWAYS> - | <ANNOTATION> - | <ANNOTATIONREFERENCE> - | <ARITY> - | <ARRAY> - | <AS> - | <ASCENDING> - | <ATTRIBUTE> - | <BODY> - | <BOLDING> - | <BTREE> - | <CASED> - | <COMPRESSION> - | <COMPRESSIONLEVEL> - | <COMPRESSIONTHRESHOLD> - | <CONSTANT> - | <CONSTANTS> - | <CONTEXT> - | <CREATEIFNONEXISTENT> - | <DENSEPOSTINGLISTTHRESHOLD> - | <DESCENDING> - | <DICTIONARY> - | <DIRECT> - | <DOCUMENT> - | <DOCUMENTSUMMARY> - | <DOUBLE_KEYWORD> - | <DYNAMIC> - | <ENABLEBITVECTORS> - | <ENABLEONLYBITVECTOR> - | <EXACT> - | <EXACTTERMINATOR> - | <FALSE> - | <FASTACCESS> - | <FASTSEARCH> - | <FIELD> - | <FIELDS> - | <FIELDSET> - | <FILE> - | <FILTER> - | <FIRSTPHASE> - | <FULL> - | <FUNCTION> - | <GRAM> - | <HASH> - | <HEADER> - | <HUGE> - | <ID> - | <IDENTICAL> - | <IDENTIFIER> - | <IGNOREDEFAULTRANKFEATURES> - | <IMPORT> - | <INDEX> - | <INDEXING> - | <INDEXINGREWRITE> - | <INHERITS> - | <INLINE> - | <INPUTS> - | <INTEGER> - | <KEEPRANKCOUNT> - | <LITERAL> - | <LOCALE> - | <LONG> - | <LOOSE> - | <LOWERBOUND> - | <LOWERCASE> - | <MACRO> - | <MAP> - | <MATCH> - | <MATCHPHASE> - | <MAXFILTERCOVERAGE> - | <MAXHITS> - | <MTOKEN> - | <MUTABLE> - | <NEVER> - | <NONE> - | <NORMAL> - | <NORMALIZING> - | <OFF> - | <ON> - | <ONDEMAND> - | <ORDER> - | <PREFIX> - | <PRIMARY> - | <PROPERTIES> - | <QUATERNARY> - | <QUERY> - | <QUERYCOMMAND> - | <RANK> - | <MODEL> - | <RANKPROFILE> - | <RANKPROPERTIES> - | <RANKSCOREDROPLIMIT> - | <RANKTYPE> - | <RAW> - | <REFERENCE> - | <REMOVEIFZERO> - | <RERANKCOUNT> - | <SCHEMA> - | <SEARCH> - | <SECONDARY> - | <SECONDPHASE> - | <SORTING> - | <SOURCE> - | <PAGED> - | <SSCONTEXTUAL> - | <SSOVERRIDE> - | <SSTITLE> - | <SSURL> - | <STATIC> - | <STEMMING> - | <STRENGTH> - | <STRICT> - | <STRING> - | <STRUCT> - | <SUBSTRING> - | <SUFFIX> - | <SUMMARY> - | <SUMMARYTO> - | <SYMMETRIC> - | <TERTIARY> - | <TEXT> - | <TO> - | <TRUE> - | <TYPE> - | <UCA> - | <UNCASED> - | <URI> - | <UPPERBOUND> - | <USEDOCUMENT> - | <VARIABLE> - | <WEIGHT> - | <WEIGHTEDSET> - | <WORD> - ) - { return token.image; } -} - -/** - * Consumes a string token and returns the token image. - * - * @return The consumed token image. - */ -String string() : { } -{ - <STRING> { return token.image; } -} - -/** - * Consumes a quoted string token and returns the token image minus the quotes. This does not perform - * unescaping of the content, it simply removes the first and last character of the image. However, the token itself can - * contain anything but a double quote. - * - * @return The unquoted token image. - */ -String quotedString() : { } -{ - <QUOTEDSTRING> { return token.image.substring(1, token.image.length() - 1); } -} - -/** - * This rule consumes a boolean value. - * - * @return The consumed boolean value. - */ -Boolean bool() : { } -{ - ( ( <ON> | <TRUE> ) { return true; } | - ( <OFF> | <FALSE> ) { return false; } ) -} - -/** - * This rule consumes an integer token and returns its numeric value. - * - * @return The consumed integer value. - */ -int integer() : { } -{ - <INTEGER> { return Integer.parseInt(token.image); } -} - -/** - * This rule consumes a long or integer token and returns its numeric value. - * - * @return The consumed long value. - */ -long consumeLong() : { } -{ - ( <INTEGER> { return Long.parseLong(token.image); } | - <LONG> { return Long.parseLong(token.image.substring(0, token.image.length()-1)); } - ) -} - -/** - * This rule consumes a floating-point token and returns its numeric value. - * - * @return The consumed value. - */ -double consumeFloat() : { } -{ - <DOUBLE> { return Double.valueOf(token.image); } -} - -Number consumeNumber() : -{ - Number num; -} -{ - (num = consumeFloat() | num = consumeLong()) { return num; } -} - -/** - * This rule consumes an opening brace with leading and trailing newline tokens. - */ -void lbrace() : { } -{ - (<NL>)* <LBRACE> (<NL>)* -} |