aboutsummaryrefslogtreecommitdiffstats
path: root/config-model/src/main/javacc/SDParser.jj
diff options
context:
space:
mode:
authorJon Bratseth <bratseth@gmail.com>2022-05-10 14:53:34 +0200
committerJon Bratseth <bratseth@gmail.com>2022-05-10 14:53:34 +0200
commit95339ca1e00f4ecb419a90a765229de65eef55ac (patch)
tree7e95c94a829c1b54013846661a5a8721d7e21759 /config-model/src/main/javacc/SDParser.jj
parent14e91114c5f821d16090ed42de51a88482c593ba (diff)
Remove old parser: Not used by any apps
Diffstat (limited to 'config-model/src/main/javacc/SDParser.jj')
-rw-r--r--config-model/src/main/javacc/SDParser.jj2912
1 files changed, 0 insertions, 2912 deletions
diff --git a/config-model/src/main/javacc/SDParser.jj b/config-model/src/main/javacc/SDParser.jj
deleted file mode 100644
index de2ce3d6938..00000000000
--- a/config-model/src/main/javacc/SDParser.jj
+++ /dev/null
@@ -1,2912 +0,0 @@
-// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-
-// Schema parser.
-//
-// NOTE: A full rewrite of parsing is in progress; avoid changing this
-// file if possible, coordinate with IntermediateParser.jj if you must
-// change something.
-//
-// NOTE: When this is changed, also change integration/intellij/src/main/bnf/ai/vespa/intellij/schema/parser/sd.bnf
-
-options {
- UNICODE_INPUT = true;
- CACHE_TOKENS = false;
- DEBUG_PARSER = false;
- ERROR_REPORTING = true;
- FORCE_LA_CHECK = true;
- USER_CHAR_STREAM = true;
-}
-
-PARSER_BEGIN(SDParser)
-
-package com.yahoo.searchdefinition.parser;
-
-import com.yahoo.document.*;
-import com.yahoo.documentmodel.*;
-import com.yahoo.compress.Compressor;
-import com.yahoo.compress.CompressionType;
-import com.yahoo.config.application.api.ApplicationPackage;
-import com.yahoo.searchdefinition.DistributableResource;
-import com.yahoo.searchdefinition.document.*;
-import com.yahoo.searchdefinition.document.annotation.SDAnnotationType;
-import com.yahoo.searchdefinition.document.annotation.TemporaryAnnotationReferenceDataType;
-import com.yahoo.searchdefinition.RankingConstant;
-import com.yahoo.searchdefinition.OnnxModel;
-import com.yahoo.searchdefinition.Index;
-import com.yahoo.searchdefinition.RankProfile;
-import com.yahoo.searchdefinition.DocumentsOnlyRankProfile;
-import com.yahoo.searchdefinition.DefaultRankProfile;
-import com.yahoo.searchdefinition.RankProfileRegistry;
-import com.yahoo.searchdefinition.RankProfile.MatchPhaseSettings;
-import com.yahoo.searchdefinition.RankProfile.DiversitySettings;
-import com.yahoo.searchdefinition.Schema;
-import com.yahoo.searchdefinition.DocumentOnlySchema;
-import com.yahoo.searchdefinition.UnrankedRankProfile;
-import com.yahoo.searchdefinition.fieldoperation.*;
-import com.yahoo.searchlib.rankingexpression.FeatureList;
-import com.yahoo.searchlib.rankingexpression.Reference;
-import com.yahoo.searchlib.rankingexpression.evaluation.Value;
-import com.yahoo.searchlib.rankingexpression.evaluation.TensorValue;
-import com.yahoo.tensor.Tensor;
-import com.yahoo.tensor.TensorType;
-import com.yahoo.vespa.documentmodel.DocumentSummary;
-import com.yahoo.vespa.documentmodel.SummaryField;
-import com.yahoo.vespa.documentmodel.SummaryTransform;
-import com.yahoo.config.model.test.MockApplicationPackage;
-import com.yahoo.config.application.api.ApplicationPackage;
-import com.yahoo.config.application.api.DeployLogger;
-import com.yahoo.config.application.api.FileRegistry;
-import com.yahoo.config.model.api.ModelContext;
-import com.yahoo.language.Linguistics;
-import com.yahoo.language.process.Embedder;
-import com.yahoo.language.simple.SimpleLinguistics;
-import com.yahoo.search.query.ranking.Diversity;
-import java.util.Optional;
-import java.util.Map;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.LinkedHashMap;
-import java.util.logging.Level;
-
-/**
- * The schema parser
- *
- * @author bratseth
- */
-@SuppressWarnings("deprecation")
-public class SDParser {
-
- private DocumentTypeManager docMan = null;
- private ApplicationPackage applicationPackage;
- private FileRegistry fileRegistry;
- private DeployLogger deployLogger;
- private ModelContext.Properties properties;
- private RankProfileRegistry rankProfileRegistry;
- private boolean documentsOnly;
-
- /**
- * Creates a parser
- *
- * @param documentsOnly true to only parse the document aspect of a schema (e.g skip rank profiles)
- */
- public SDParser(SimpleCharStream stream,
- ApplicationPackage applicationPackage,
- FileRegistry fileRegistry,
- DeployLogger deployLogger,
- ModelContext.Properties properties,
- RankProfileRegistry rankProfileRegistry,
- boolean documentsOnly) {
- this(stream);
- this.applicationPackage = applicationPackage;
- this.fileRegistry = fileRegistry;
- this.deployLogger = deployLogger;
- this.properties = properties;
- this.rankProfileRegistry = rankProfileRegistry;
- this.documentsOnly = documentsOnly;
- }
-
- /**
- * Consumes an indexing language script which will use the simple linguistics implementation
- * for testing, by taking input from the current input stream.
- *
- * @param multiline Whether or not to allow multi-line expressions.
- */
- @SuppressWarnings("deprecation")
- private IndexingOperation newIndexingOperation(boolean multiline) throws ParseException {
- return newIndexingOperation(multiline, new SimpleLinguistics(), Embedder.throwsOnUse.asMap());
- }
-
- /**
- * Consumes an indexing language script from the current input stream.
- *
- * @param multiline Whether or not to allow multi-line expressions.
- * @param linguistics What to use for tokenizing.
- */
- private IndexingOperation newIndexingOperation(boolean multiline, Linguistics linguistics, Map<String, Embedder> embedders) throws ParseException {
- SimpleCharStream input = (SimpleCharStream)token_source.input_stream;
- if (token.next != null) {
- input.backup(token.next.image.length());
- }
- try {
- return IndexingOperation.fromStream(input, multiline, linguistics, embedders);
- } finally {
- token.next = null;
- jj_ntk = -1;
- }
- }
-
- /**
- * Parses the given token image as a ranking expression feature list.
- *
- * @param image the token image to parse
- * @return the consumed feature list
- * @throws ParseException thrown if the image could not be parsed.
- */
- private FeatureList getFeatureList(String image) throws ParseException {
- try {
- return new FeatureList(image);
- }
- catch (com.yahoo.searchlib.rankingexpression.parser.ParseException e) {
- throw (ParseException) new ParseException("Could not parse feature list '" + image + "' at line " +
- token_source.input_stream.getBeginLine() + ", column " +
- token_source.input_stream.getBeginColumn() + ".").initCause(e);
- }
- }
-}
-
-PARSER_END(SDParser)
-
-
-// --------------------------------------------------------------------------------
-//
-// Token declarations.
-//
-// --------------------------------------------------------------------------------
-
-// Declare white space characters. These do not include newline because it has
-// special meaning in several of the production rules.
-SKIP :
-{
- " " | "\t" | "\r" | "\f"
-}
-
-// Declare all tokens to be recognized. When a word token is added it MUST be
-// added to the identifier() production rule.
-TOKEN :
-{
- < NL: "\n" >
-| < ANNOTATION: "annotation" >
-| < ANNOTATIONREFERENCE: "annotationreference" >
-| < SCHEMA: "schema" >
-| < SEARCH: "search" >
-| < DIVERSITY: "diversity" >
-| < MIN_GROUPS: "min-groups" >
-| < CUTOFF_FACTOR: "cutoff-factor" >
-| < CUTOFF_STRATEGY: "cutoff-strategy" >
-| < LOOSE: "loose" >
-| < STRICT: "strict" >
-| < DOCUMENT: "document" >
-| < OPERATION: "operation" >
-| < ON_MATCH: "on-match" >
-| < ON_FIRST_PHASE: "on-first-phase" >
-| < ON_SECOND_PHASE: "on-second-phase" >
-| < ON_SUMMARY: "on-summary" >
-| < STRUCT: "struct" >
-| < INHERITS: "inherits" >
-| < FIELD: "field" >
-| < FIELDS: "fields" >
-| < FIELDSET: "fieldset" >
-| < STRUCTFIELD: "struct-field" >
-| < IMPORT: "import" >
-| < AS: "as" >
-| < INDEXING: "indexing" >
-| < SUMMARYTO: "summary-to" >
-| < DOCUMENTSUMMARY: "document-summary" >
-| <DOUBLE_KEYWORD: "double" >
-| < RANKTYPE: "rank-type" >
-| < WEIGHT: "weight" >
-| < TYPE: "type" >
-| < INDEX: "index" >
-| < INPUTS: "inputs">
-| < MTOKEN: "token" >
-| < TEXT: "text" >
-| < WORD: "word" >
-| < GRAM: "gram" >
-| < GRAMSIZE: "gram-size" >
-| < MAXLENGTH: "max-length" >
-| < PREFIX: "prefix" >
-| < SUBSTRING: "substring" >
-| < SUFFIX: "suffix" >
-| < CONSTANT: "constant">
-| < ONNXMODEL: "onnx-model">
-| < MODEL: "model" >
-| < MUTATE: "mutate" >
-| < QUERY: "query" >
-| < RANKPROFILE: "rank-profile" >
-| < RANKDEGRADATIONFREQ: "rank-degradation-frequency" >
-| < RANKDEGRADATION: "rank-degradation" >
-| < RAW_AS_BASE64_IN_SUMMARY: "raw-as-base64-in-summary" >
-| < RPBINSIZE: "doc-frequency" >
-| < RPBINLOW: "min-fullrank-docs">
-| < RPPOSBINSIZE: "occurrences-per-doc" >
-| < SUMMARY: "summary" >
-| < FULL: "full" >
-| < STATIC: "static" >
-| < DYNAMIC: "dynamic" >
-| < MATCHEDELEMENTSONLY: "matched-elements-only" >
-| < SSCONTEXTUAL: "contextual" >
-| < SSOVERRIDE: "override" >
-| < SSTITLE: "title" >
-| < SSURL: "url" >
-| < PROPERTIES: "properties" >
-| < ATTRIBUTE: "attribute" >
-| < SORTING: "sorting" >
-| < DICTIONARY: "dictionary" >
-| < ASCENDING: "ascending" >
-| < DESCENDING: "descending" >
-| < UCA: "uca" >
-| < RAW: "raw" >
-| < LOWERCASE: "lowercase" >
-| < FUNCTION: "function" >
-| < LOCALE: "locale" >
-| < STRENGTH: "strength" >
-| < PRIMARY: "primary" >
-| < SECONDARY: "secondary" >
-| < TERTIARY: "tertiary" >
-| < QUATERNARY: "quaternary" >
-| < IDENTICAL: "identical" >
-| < STEMMING: "stemming" >
-| < NORMALIZING: "normalizing" >
-| < HASH: "hash" >
-| < BTREE: "btree" >
-| < CASED: "cased" >
-| < UNCASED: "uncased" >
-| < BOLDING: "bolding" >
-| < BODY: "body" >
-| < HEADER: "header" >
-| < NONE: "none" >
-| < ON: "on" >
-| < OFF: "off" >
-| < TRUE: "true" >
-| < FALSE: "false" >
-| < SYMMETRIC: "symmetric" >
-| < QUERYCOMMAND: "query-command" >
-| < ALIAS: "alias" >
-| < MATCH: "match" >
-| < RANK: "rank" >
-| < LITERAL: "literal" >
-| < EXACT: "exact" >
-| < FILTER: "filter" >
-| < NORMAL: "normal" >
-| < EXACTTERMINATOR: "exact-terminator" >
-| < INDEXINGREWRITE: "indexing-rewrite" >
-| < IGNOREDEFAULTRANKFEATURES: "ignore-default-rank-features" >
-| < ID: "id" >
-| < SOURCE: "source" >
-| < TO: "to" >
-| < DIRECT: "direct" >
-| < FROMDISK: "from-disk" >
-| < OMITSUMMARYFEATURES: "omit-summary-features" >
-| < ALWAYS: "always" >
-| < ONDEMAND: "on-demand" >
-| < NEVER: "never" >
-| < ENABLEBITVECTORS: "enable-bit-vectors" >
-| < ENABLEONLYBITVECTOR: "enable-only-bit-vector" >
-| < FASTACCESS: "fast-access" >
-| < MUTABLE: "mutable" >
-| < PAGED: "paged" >
-| < FASTSEARCH: "fast-search" >
-| < HUGE: "huge" >
-| < TENSOR_TYPE: "tensor" ("<" (~["<",">"])+ ">")? "(" (~["(",")"])* ")" >
-| < TENSOR_VALUE_SL: "value" (" ")* ":" (" ")* ("{"<BRACE_SL_LEVEL_1>) ("\n")? >
-| < TENSOR_VALUE_ML: "value" (<SEARCHLIB_SKIP>)? "{" (["\n"," "])* ("{"<BRACE_ML_LEVEL_1>) (["\n"," "])* "}" ("\n")? >
-| < COMPRESSION: "compression" >
-| < COMPRESSIONLEVEL: "level" >
-| < COMPRESSIONTHRESHOLD: "threshold" >
-| < LZ4: "lz4" >
-| < USEDOCUMENT: "use-document" >
-| < LBRACE: "{" >
-| < RBRACE: "}" >
-| < COLON: ":" >
-| < DOT: "." >
-| < COMMA: "," >
-| < ARRAY: "array" >
-| < WEIGHTEDSET: "weightedset" >
-| < MAP: "map" >
-| < REFERENCE: "reference" >
-| < QUESTIONMARK: "?" >
-| < CREATEIFNONEXISTENT: "create-if-nonexistent" >
-| < REMOVEIFZERO: "remove-if-zero" >
-| < MATCHPHASE: "match-phase" >
-| < EVALUATION_POINT: "evaluation-point" >
-| < PRE_POST_FILTER_TIPPING_POINT: "pre-post-filter-tipping-point" >
-| < ORDER: "order" >
-| < MAXFILTERCOVERAGE: "max-filter-coverage" >
-| < MAXHITS: "max-hits" >
-| < FIRSTPHASE: "first-phase" >
-| < SECONDPHASE: "second-phase" >
-| < MACRO: "macro" >
-| < INLINE: "inline" >
-| < ARITY: "arity" >
-| < LOWERBOUND: "lower-bound" >
-| < UPPERBOUND: "upper-bound" >
-| < DENSEPOSTINGLISTTHRESHOLD: "dense-posting-list-threshold" >
-| < ENABLE_BM25: "enable-bm25" >
-| < HNSW: "hnsw" >
-| < MAXLINKSPERNODE: "max-links-per-node" >
-| < DISTANCEMETRIC: "distance-metric" >
-| < NEIGHBORSTOEXPLOREATINSERT: "neighbors-to-explore-at-insert" >
-| < MULTITHREADEDINDEXING: "multi-threaded-indexing" >
-| < MATCHFEATURES_SL: "match-features" (" ")* ":" (~["}","\n"])* ("\n")? >
-| < MATCHFEATURES_ML: "match-features" (<SEARCHLIB_SKIP>)? "{" (~["}"])* "}" >
-| < MATCHFEATURES_ML_INHERITS: "match-features inherits " (<IDENTIFIER_WITH_DASH>) (<SEARCHLIB_SKIP>)? "{" (~["}"])* "}" >
-| < SUMMARYFEATURES_SL: "summary-features" (" ")* ":" (~["}","\n"])* ("\n")? >
-| < SUMMARYFEATURES_ML: "summary-features" (<SEARCHLIB_SKIP>)? "{" (~["}"])* "}" >
-| < SUMMARYFEATURES_ML_INHERITS: "summary-features inherits " (<IDENTIFIER_WITH_DASH>) (<SEARCHLIB_SKIP>)? "{" (~["}"])* "}" >
-| < RANKFEATURES_SL: "rank-features" (" ")* ":" (~["}","\n"])* ("\n")? >
-| < RANKFEATURES_ML: "rank-features" (<SEARCHLIB_SKIP>)? "{" (~["}"])* "}" >
-| < EXPRESSION_SL: "expression" (" ")* ":" (("{"<BRACE_SL_LEVEL_1>)|<BRACE_SL_CONTENT>)* ("\n")? >
-| < EXPRESSION_ML: "expression" (<SEARCHLIB_SKIP>)? "{" (("{"<BRACE_ML_LEVEL_1>)|<BRACE_ML_CONTENT>)* "}" >
-| < #BRACE_SL_LEVEL_1: (("{"<BRACE_SL_LEVEL_2>)|<BRACE_SL_CONTENT>)* "}" >
-| < #BRACE_SL_LEVEL_2: (("{"<BRACE_SL_LEVEL_3>)|<BRACE_SL_CONTENT>)* "}" >
-| < #BRACE_SL_LEVEL_3: <BRACE_SL_CONTENT> "}" >
-| < #BRACE_SL_CONTENT: (~["{","}","\n"])* >
-| < #BRACE_ML_LEVEL_1: (("{"<BRACE_ML_LEVEL_2>)|<BRACE_ML_CONTENT>)* "}" >
-| < #BRACE_ML_LEVEL_2: (("{"<BRACE_ML_LEVEL_3>)|<BRACE_ML_CONTENT>)* "}" >
-| < #BRACE_ML_LEVEL_3: <BRACE_ML_CONTENT> "}" >
-| < #BRACE_ML_CONTENT: (~["{","}"])* >
-| < #SEARCHLIB_SKIP: ([" ","\f","\n","\r","\t"])+ >
-| < RANKPROPERTIES: "rank-properties" >
-| < RERANKCOUNT: "rerank-count" >
-| < NUMTHREADSPERSEARCH: "num-threads-per-search" >
-| < MINHITSPERTHREAD: "min-hits-per-thread" >
-| < NUMSEARCHPARTITIONS: "num-search-partitions" >
-| < TERMWISELIMIT: "termwise-limit" >
-| < POSTFILTERTHRESHOLD: "post-filter-threshold" >
-| < APPROXIMATETHRESHOLD: "approximate-threshold" >
-| < KEEPRANKCOUNT: "keep-rank-count" >
-| < RANKSCOREDROPLIMIT: "rank-score-drop-limit" >
-| < CONSTANTS: "constants" >
-| < FILE: "file" >
-| < URI: "uri" >
-| < IDENTIFIER: ["a"-"z","A"-"Z", "_"] (["a"-"z","A"-"Z","0"-"9","_"])* >
-| < IDENTIFIER_WITH_DASH: ["a"-"z","A"-"Z", "_"] (["a"-"z","A"-"Z","0"-"9","_","-"])* >
-| < QUOTEDSTRING: "\"" ( ~["\""] )* "\"" >
-| < CONTEXT: ["a"-"z","A"-"Z"] (["a"-"z", "A"-"Z", "0"-"9"])* >
-| < DOUBLE: ("-")? (["0"-"9"])+ "." (["0"-"9"])+ >
-| < INTEGER: ("-")? (["0"-"9"])+ >
-| < LONG: ("-")? (["0"-"9"])+"L" >
-| < STRING: (["a"-"z","A"-"Z","_","0"-"9","."])+ >
-| < FILE_PATH: ["a"-"z","A"-"Z", "_"] (["a"-"z","A"-"Z","0"-"9","_","-", "/", "."])+ >
-| < HTTP: ["h","H"] ["t","T"] ["t","T"] ["p","P"] (["s","S"])? >
-| < URI_PATH: <HTTP> <COLON> ("//")? (["a"-"z","A"-"Z","0"-"9","_","-", "/", ".",":"])+ >
-| < LESSTHAN: "<" >
-| < GREATERTHAN: ">" >
-| < VARIABLE: "$" <IDENTIFIER> >
-| < ONNX_INPUT_SL: "input" (" ")* (<IDENTIFIER>|<QUOTEDSTRING>) (" ")* ":" (" ")* (~["\n"])* ("\n")? >
-| < ONNX_OUTPUT_SL: "output" (" ")* (<IDENTIFIER>|<QUOTEDSTRING>) (" ")* ":" (" ")* (~["\n"])* ("\n")? >
-}
-
-// Declare a special skip token for comments.
-SPECIAL_TOKEN :
-{
- <SINGLE_LINE_COMMENT: "#" (~["\n","\r"])* >
-}
-
-
-// --------------------------------------------------------------------------------
-//
-// Production rules.
-//
-// --------------------------------------------------------------------------------
-
-/**
- * The rule consumes any schema and returns the corresponding object. This is the only production that should
- * ever consume leading newlines.
- *
- * @return the schema object
- */
-Schema schema(DocumentTypeManager docMan) :
-{
- this.docMan = docMan;
- Schema schema;
-}
-{
- (<NL>)* (schema = rootSchema() | schema = rootDocument())
- { return schema; }
-}
-
-/**
- * This rule consumes a proper schema block. This and rootDocument() are the only rules that should ever consume
- * trailing newline tokens.
- *
- * @return the schema definition object.
- */
-Schema rootSchema() :
-{
- String name;
- String inherited = null;
- Schema schema;
-}
-{
- ( ( <SCHEMA> | <SEARCH> ) name = identifier() (<INHERITS> inherited = identifier() )? {
- schema = new Schema(name, applicationPackage, Optional.ofNullable(inherited), fileRegistry, deployLogger, properties);
- rankProfileRegistry.add(new DefaultRankProfile(schema, rankProfileRegistry, schema.rankingConstants()));
- rankProfileRegistry.add(new UnrankedRankProfile(schema, rankProfileRegistry, schema.rankingConstants()));}
- lbrace() (rootSchemaItem(schema) (<NL>)*)* <RBRACE> (<NL>)* <EOF>)
- { return schema; }
-}
-
-/**
- * Consumes an element of a schema block. This and rootSearch() are the only rules that should ever consume
- * trailing newline tokens.
- *
- * @param schema the schema object to modify.
- */
-void rootSchemaItem(Schema schema) : { }
-{
- ( document(schema)
- | rawAsBase64(schema)
- | documentSummary(schema)
- | field(null, schema)
- | index(schema, null)
- | rankingConstant(schema)
- | rankProfile(schema)
- | searchStemming(schema)
- | useDocument(schema)
- | structOutside(schema)
- | annotationOutside(schema)
- | fieldSet(schema)
- | importField(schema)
- | onnxModel(schema) )
-}
-
-/**
- * Consumes a schema definition that contains only documents to be used for inheritance, etc.
- *
- * @return the schema definition object.
- */
-Schema rootDocument() :
-{
- Schema schema = new DocumentOnlySchema(applicationPackage, fileRegistry, deployLogger, properties);
-}
-{
- ( (rootDocumentItem(schema) (<NL>)*)*<EOF> )
- { return schema; }
-}
-
-/**
- * Consumes a single item from within a root document node.
- *
- * @param schema the schema object to modify.
- */
-void rootDocumentItem(Schema schema) : { }
-{
- ( namedDocument(schema) )
-}
-
-/**
- * Consumes a use-document statement. This currently does nothing.
- *
- * @param schema the schema object to modify.
- */
-void useDocument(Schema schema) : { }
-{
- <USEDOCUMENT> <COLON> identifier()
- { deployLogger.logApplicationPackage(Level.WARNING, "Specifying 'use-document' is deprecated and has no effect."); }
-}
-
-/**
- * Consumes a document element. The name defaults to the schema's name, but may be set.
- *
- * @param schema the schema object to add content to.
- */
-void document(Schema schema) :
-{
- String name = schema.getName();
- SDDocumentType document;
-}
-{
- ( <DOCUMENT> (name = identifier())? (<NL>)* { document = new SDDocumentType(name, schema); }
- [ inheritsDocument(document) (<NL>)* ]
- <LBRACE> (<NL>)* (documentBody(document, schema) (<NL>)*)* <RBRACE> )
- {
- schema.addDocument(document);
- }
-}
-
-/**
- * Consumes a document element, explicitly named
- *
- * @param schema the schema object to add content to
- */
-void namedDocument(Schema schema) :
-{
- String name;
- SDDocumentType document;
-}
-{
- ( <DOCUMENT> name = identifier() (<NL>)* { document = new SDDocumentType(name, schema); }
- [ inheritsDocument(document) (<NL>)* ]
- <LBRACE> (<NL>)* (documentBody(document, schema) (<NL>)*)* <RBRACE> )
- {
- schema.addDocument(document);
- }
-}
-
-/**
- * Consumes a document body block
- *
- * @param document the document type to modify.
- * @param schema the schema object to add content to
- */
-void documentBody(SDDocumentType document, Schema schema) :
-{
-}
-{
- ( annotation(schema, document)
- | compression(document)
- | headercfg(document)
- | bodycfg(document)
- | structInside(document, schema)
- | field(document, schema) )
-}
-
-void rawAsBase64(Schema schema) :
-{
- boolean enabled = false;
-}
-{
- <RAW_AS_BASE64_IN_SUMMARY>
- {
- enabled = true;
- }
- [ <COLON> ( <TRUE> | ( <FALSE> { enabled = false; } ) ) ]
- {
- schema.enableRawAsBase64(enabled);
- }
-}
-
-/**
- * Consumes a document head block.
- *
- * @param document The document type to modify.
- */
-void headercfg(SDDocumentType document) : { }
-{
- <HEADER> lbrace() [compression(document) (<NL>)*] <RBRACE>
-}
-
-/**
- * Consumes a document body block.
- *
- * @param document The document type to modify.
- */
-void bodycfg(SDDocumentType document) : { }
-{
- <BODY> lbrace() [compression(document) (<NL>)*] <RBRACE>
-}
-
-/**
- * Consumes a compression block. This can be set in both document header and -body block.
- *
- * @param document The document type to modify.
- */
-void compression(SDDocumentType document) :
-{
- deployLogger.logApplicationPackage(Level.WARNING, "'compression' for a document is deprecated and ignored");
-}
-{
- <COMPRESSION> lbrace() ( compressionItem() (<NL>)*)* <RBRACE> { }
-}
-
-/**
- * Consumes the body of a compression block.
- *
- */
-void compressionItem() :
-{ }
-{
- ( ( <TYPE> <COLON> <LZ4> )
- | (<COMPRESSIONTHRESHOLD> <COLON> <INTEGER>)
- | (<COMPRESSIONLEVEL> <COLON> <INTEGER>)
- ) { }
-}
-
-/**
- * Consumes a document inheritance statement.
- *
- * @param document The document type to modify.
- */
-void inheritsDocument(SDDocumentType document) :
-{
- String name;
-}
-{
- <INHERITS> name = identifier() { document.inherit(new DataTypeName(name)); }
- ( <COMMA> name = identifier() { document.inherit(new DataTypeName(name)); } )*
-}
-
-/**
- * Consumes a field block from within a document element.
- *
- * @param document the document type to modify
- * @param schema the schema object to add content to
- */
-void field(SDDocumentType document, Schema schema) :
-{
- String name;
- SDField field;
- DataType type;
-}
-{
- <FIELD> name = identifier() <TYPE> type = dataType()
- {
- if (name != null && Schema.isReservedName(name.toLowerCase())) {
- throw new IllegalArgumentException("Reserved name '" + name + "' can not be used as a field name.");
- }
- field = new TemporarySDField(document == null ? schema.getDocument() : document, name, type, document);
- }
- lbrace() (fieldBody(field, schema, document) (<NL>)*)* <RBRACE>
- {
- if (document != null) {
- document.addField(field);
- } else {
- schema.addExtraField(field);
- }
- }
-}
-
-void fieldSet(Schema schema) :
-{
- String setName;
- String field;
- String queryCommand;
- List queryCommands = new ArrayList();
- FieldOperationContainer matchSetting;
- List matchSettings = new ArrayList();
-}
-{
- <FIELDSET> setName = identifier() lbrace()
- ((
- ( <FIELDS><COLON> field = identifier() { schema.fieldSets().addUserFieldSetItem(setName, field); }
- ( <COMMA> field = identifier() { schema.fieldSets().addUserFieldSetItem(setName, field); } )* )
- |
- ( <QUERYCOMMAND> <COLON> (queryCommand = identifierWithDash() | queryCommand = quotedString())) { queryCommands.add(queryCommand); }
- |
- ( matchSetting = match(new SDField(null, setName, DataType.STRING)) ) { matchSettings.add(matchSetting); }
- )(<NL>)*)+
- <RBRACE>
- {
- // Apply settings after parsing since all user field items must be set first
-
- for (Object command : queryCommands)
- schema.fieldSets().userFieldSets().get(setName).queryCommands().add((String)command);
-
- for (Object setting : matchSettings) {
- ((SDField)setting).applyOperations();
- schema.fieldSets().userFieldSets().get(setName).setMatching(((SDField)setting).getMatching());
- }
- }
-}
-
-/**
- * This rule consumes a annotation block from within either a document element or a schema element.
-
- * @param schema the schema object to add content to
- */
-void annotationOutside(Schema schema) :
-{
- String name;
- SDAnnotationType type;
-}
-{
- <ANNOTATION> name = identifier()
- {
- type = new SDAnnotationType(name.trim());
- if (schema.getDocument() == null) {
- throw new IllegalArgumentException("Can't add annotation '"+name+"' to a document type, define a document type first or declare the annotation inside of one.");
- }
- }
- [ inheritsAnnotation(type) (<NL>)* ]
- lbrace() (type = annotationBody(schema, schema.getDocument(), type)) <RBRACE>
- {
- schema.addAnnotation(type);
- }
-}
-
-/**
- * Consumes a annotation block from within either a document element.
- *
- * @param document the document object to add content to
- */
-void annotation(Schema schema, SDDocumentType document) :
-{
- String name;
- SDAnnotationType type;
-}
-{
- <ANNOTATION> name = identifier()
- {
- type = new SDAnnotationType(name.trim());
- }
- [ inheritsAnnotation(type) (<NL>)* ]
- lbrace() (type = annotationBody(schema, document, type)) <RBRACE>
- {
- document.addAnnotation(type);
- }
-}
-
-
-/**
- * Consumes a single element of an annotation body block.
- *
- * @param schema the schema object to add content to
- * @param type the type being built
- * @return a modified or new AnnotationType instance
- */
-SDAnnotationType annotationBody(Schema schema, SDDocumentType repo, SDAnnotationType type) :
-{
- SDDocumentType struct = new SDDocumentType("annotation." + type.getName(), schema);
-}
-{
- (structFieldDefinition(repo, struct) (<NL>)*)*
- {
- if (struct.getFieldCount() > 0) { // Must account for the temporary TemporarySDField.
- type = new SDAnnotationType(type.getName(), struct, type.getInherits());
- struct.setStruct(null);
- }
- return type;
- }
-}
-
-void inheritsAnnotation(SDAnnotationType annotation) :
-{
- String name;
-}
-{
- <INHERITS> name = identifier() { annotation.inherit(name); }
-}
-
-
-/**
- * This rule consumes a struct block from within a document element.
- *
- * @param schema the schema object to add content to
- */
-void structInside(SDDocumentType document, Schema schema) :
-{
- SDDocumentType struct;
-}
-{
- (
- struct = structDefinition(schema, document)
- )
- {
- document.addType(struct);
- }
-}
-
-/**
- * This rule consumes a struct block from within a document element.
- *
- * @param schema the schema object to add content to
- */
-void structOutside(Schema schema) :
-{
- SDDocumentType struct;
-}
-{
- (
- struct = structDefinition(schema, schema.getDocument())
- )
- {
- schema.addType(struct);
- }
-}
-
-/**
- * This rule consumes a struct block from within a document element.
- *
- * @param schema the schema object to add content to
- */
-SDDocumentType structDefinition(Schema schema, SDDocumentType repo) :
-{
- String name;
- String inherited = null;
- SDDocumentType struct;
-}
-{
- ( <STRUCT> name = identifier() (<NL>)* {
- if (repo == null)
- throw new IllegalArgumentException("Can't add struct '"+name+"' to a document type, define a document type first or declare the struct inside of one.");
- struct = new SDDocumentType(name, schema);
- }
- [ inheritsDocument(struct) (<NL>)* ]
- lbrace() (structFieldDefinition(repo, struct) (<NL>)*)* <RBRACE> )
- {
- try {
- docMan.getDataType(name);
- throw new ParseException("Reserved name '" + name + "' can not be used to declare a struct.");
- } catch (IllegalArgumentException e) {
- // empty
- }
- SDDocumentType sdtype = repo.getOwnedType(struct.getDocumentName());
- if (sdtype != null) {
- throw new ParseException("Struct '" + name + "' is already defined.");
- }
- DataType stype = new OwnedTemporaryType(name, repo.getName());
- struct.setStruct(stype);
- return struct;
- }
-}
-
-/**
- * This rule consumes a data type block from within a field element.
- *
- * @return the consumed data type
- */
-DataType dataType() :
-{
- String typeName = null;
- boolean isArrayOldStyle = false;
- DataType mapType = null;
- DataType arrayType = null;
- DataType wsetType = null;
- TensorType tensorType;
- String referencedDoc;
-}
-{
- ( LOOKAHEAD(<ARRAY> <LESSTHAN>) ( <ARRAY> <LESSTHAN> arrayType = dataType() <GREATERTHAN> { return DataType.getArray(arrayType); } )
- | LOOKAHEAD(<WEIGHTEDSET> <LESSTHAN>) ( <WEIGHTEDSET> <LESSTHAN> wsetType = dataType() <GREATERTHAN> { return DataType.getWeightedSet(wsetType); } )
- | LOOKAHEAD(<MAP> <LESSTHAN>) ( mapType = mapDataType() { return mapType; } )
- | LOOKAHEAD(<ANNOTATIONREFERENCE> <LESSTHAN>) ( mapType = annotationRefDataType() { return mapType; } )
- | LOOKAHEAD(<TENSOR_TYPE>) ( tensorType = tensorType("Field type") { return DataType.getTensor(tensorType); } )
- | LOOKAHEAD(<REFERENCE>) ( <REFERENCE> <LESSTHAN> referencedDoc = identifier() <GREATERTHAN> { return NewDocumentReferenceDataType.forDocumentName(referencedDoc); } )
- | ( typeName = identifier() ["[]" { isArrayOldStyle = true; }] )
- )
- {
- DataType type = VespaDocumentType.INSTANCE.getDataType(typeName);
-
- if (type == null) {
- // we are basically creating TemporaryUnknownType instances for ANYTHING here!!
- // we must do this and clean them up later.
- type = new TemporaryUnknownType(typeName);
- }
-
- if (isArrayOldStyle) {
- deployLogger.logApplicationPackage(Level.WARNING, "Data type syntax '" + typeName + "[]' is deprecated, use 'array<" + typeName + ">' instead.");
- type = DataType.getArray(type);
- }
- if ("tag".equalsIgnoreCase(typeName) && type instanceof WeightedSetDataType) ((WeightedSetDataType)type).setTag(true);
- return type;
- }
-}
-
-DataType annotationRefDataType() :
-{
- DataType dataType;
- String targetName;
-}
-{
- ( <ANNOTATIONREFERENCE> <LESSTHAN> targetName = identifier() <GREATERTHAN> )
- {
- return new TemporaryAnnotationReferenceDataType(targetName);
- }
-}
-
-DataType mapDataType() :
-{
- DataType keyType;
- DataType valType;
-}
-{
- ( <MAP> <LESSTHAN> keyType = dataType() <COMMA> valType = dataType() <GREATERTHAN> )
- {
- return DataType.getMap(keyType, valType);
- }
-
-}
-
-/* Note: not currently used, remove when decided that map type will not support
-polymorphism */
-DataType wildCardType() :
-{
-}
-{
-(<QUESTIONMARK>) { return DataType.NONE; }
-}
-
-/**
- * This rule consumes a field block of a struct body.
- *
- * @param struct The struct to modify.
- */
-void structFieldDefinition(SDDocumentType document, SDDocumentType struct) :
-{
- String name;
- SDField field;
- DataType type;
-}
-{
- <FIELD> name = identifier() <TYPE> type = dataType() {
- if (name != null && Schema.isReservedName(name.toLowerCase())) {
- throw new IllegalArgumentException("Reserved name '" + name + "' can not be used as a field name.");
- }
- field = new TemporarySDField(document, name, type, struct);
- struct.addField(field);
- }
- lbrace() (id(field,struct) (<NL>)*)? (match(field) (<NL>)*)* <RBRACE> {
- }
-}
-
-/**
- * This rule consumes a struct subfield from a document field body. This is not to be confused with a document
- * struct's fields, but rather this is a subfield of a document field of type struct.
- *
- * @param field the field to modify
- * @param schema the schema object to add content to
- * @param document the document type to modify
- */
-void structField(FieldOperationContainer field, Schema schema, SDDocumentType document) :
-{
- String name;
- SDField structField;
-}
-{
- <STRUCTFIELD> name = identifier() {
- if (name != null && Schema.isReservedName(name.toLowerCase())) {
- throw new IllegalArgumentException("Reserved name '" + name + "' can not be used as a field name.");
- }
- FieldOperationContainer structFieldOp = new StructFieldOperation(name);
- field.addOperation((StructFieldOperation) structFieldOp);
- }
- lbrace() (structFieldBody(structFieldOp, schema, document) (<NL>)*)* <RBRACE>
-}
-
-
-/**
- * This rule consumes a single element of a field body block.
- *
- * @param field the field being built
- * @param schema the schema object to add content to
- * @param document the owning document, or null if this is a search field
- */
-void fieldBody(SDField field, Schema schema, SDDocumentType document) : { }
-{
- ( alias(field) |
- attribute(field) |
- body(field) |
- bolding(field) |
- dictionary(field) |
- fieldStemming(field) |
- header(field) |
- id(field, document) |
- summaryInField(field) |
- index(schema, field) |
- indexing(field) |
- indexingRewrite(field) |
- match(field) |
- normalizing(field) |
- queryCommand(field) |
- rank(field) |
- rankType(field) |
- sorting(field, field.getName()) |
- structField(field, schema, document) |
- summaryTo(field) |
- weight(field) |
- weightedset(field) )
-}
-
-/**
- * This rule consumes a single element of a struct subfield body block.
- * Only elements that are supported in streaming schema and indexed schema (with complex attributes) are allowed.
- *
- * @param field the field being built
- * @param schema the schema object to add content to
- * @param document the owning document, or null if this is a schema field
- */
-void structFieldBody(FieldOperationContainer field, Schema schema, SDDocumentType document) : { }
-{
- ( summaryInField(field) |
- indexing(field) |
- attribute(field) |
- match(field) |
- queryCommand(field) |
- structField(field, schema, document) |
- summaryTo(field) )
-}
-
-/**
- * This rule consumes an indexing block of a field element.
- *
- * @param field The field to modify.
- */
-void indexing(FieldOperationContainer field) : { }
-{
- ( <INDEXING> ( (<COLON> indexingOperation(field, false)) | indexingOperation(field, true) ) )
-}
-
-/**
- * This rule consumes an IL script block. This is expected to consume trailing newlines.
- *
- * @param field The field to modify.
- */
-void indexingOperation(FieldOperationContainer field, boolean multiLine) : { }
-{
- { field.addOperation(newIndexingOperation(multiLine)); }
-}
-
-/**
- * This rule consumes a summary-to statement of a field element.
- *
- * @param field The field to modify.
- */
-void summaryTo(FieldOperationContainer field) :
-{
- SummaryToOperation op = new SummaryToOperation();
- String destination;
- String name = field.getName();
-}
-{
- <SUMMARYTO> [name = identifier()] <COLON> destination = identifier()
- {
- op.setName(name);
- op.addDestination(destination);
- }
- ( <COMMA> destination = identifier() {op.addDestination(destination); } )*
- {
- field.addOperation(op);
- }
-}
-
-
-/**
- * This rule consumes a weight statement of a field element.
- *
- * @param field The field to modify.
- */
-void weight(FieldOperationContainer field) :
-{
- int num;
-}
-{
- <WEIGHT> <COLON> num = integer()
- {
- WeightOperation op = new WeightOperation();
- op.setWeight(num);
- field.addOperation(op);
- }
-}
-
-/**
- * This rule consumes a weighted set statement of a field element.
- *
- * @param field The field to modify.
- */
-void weightedset(FieldOperationContainer field) :
-{
- WeightedSetOperation op = new WeightedSetOperation();
-}
-{
- <WEIGHTEDSET> ( (<COLON> weightedsetBody(op))
- | (lbrace() (weightedsetBody(op) (<NL>)*)* <RBRACE>) )
- {
- field.addOperation(op);
- }
-}
-
-/**
- * This rule consumes one body item of a weighted set block.
- *
- * @param field The field to modify.
- */
-void weightedsetBody(WeightedSetOperation field) : { }
-{
- ( <CREATEIFNONEXISTENT> { field.setCreateIfNonExistent(true); }
- | <REMOVEIFZERO> { field.setRemoveIfZero(true); } )
-}
-
-/**
- * This rule consumes a rank-type statement of a field element.
- *
- * @param field The field to modify.
- */
-void rankType(FieldOperationContainer field) :
-{
- String typeName;
- String indexName = null;
-}
-{
- <RANKTYPE> [indexName = identifier()] <COLON> typeName = identifier()
- {
- RankTypeOperation op = new RankTypeOperation();
- op.setType(RankType.fromString(typeName));
- op.setIndexName(indexName);
- field.addOperation(op);
- }
-}
-
-/**
- * This rule consumes an attribute statement of a field element.
- *
- * @param field The field to modify.
- */
-void attribute(FieldOperationContainer field) :
-{
- String name = field.getName();
-}
-{
- <ATTRIBUTE> [name = identifier()]
- {
- AttributeOperation op = new AttributeOperation(name);
- }
- ( (<COLON> attributeSetting(field, op, name))
- | (lbrace() (attributeSetting(field, op, name) (<NL>)*)* <RBRACE>) )
- {
- field.addOperation(op);
- }
-}
-
-void sorting(FieldOperationContainer field, String name) :
-{
- SortingOperation op = new SortingOperation(name);
-}
-{
- <SORTING>
- ( (<COLON> sortingSetting(op, name))
- | (lbrace() (sortingSetting(op, name) (<NL>)*)* <RBRACE>) )
- {
- field.addOperation(op);
- }
-}
-
-void sortingSetting(SortingOperation sorting, String attributeName) :
-{
- String locale;
-}
-{
- (
- <ASCENDING> { sorting.setAscending(); }
- | <DESCENDING> { sorting.setDescending(); }
- | <FUNCTION> <COLON> (
- <UCA> { sorting.setFunction(Sorting.Function.UCA); }
- | <RAW> { sorting.setFunction(Sorting.Function.RAW); }
- | <LOWERCASE> { sorting.setFunction(Sorting.Function.LOWERCASE); }
- )
- | <STRENGTH> <COLON> (
- <PRIMARY> { sorting.setStrength(Sorting.Strength.PRIMARY); }
- | <SECONDARY> { sorting.setStrength(Sorting.Strength.SECONDARY); }
- | <TERTIARY> { sorting.setStrength(Sorting.Strength.TERTIARY); }
- | <QUATERNARY> { sorting.setStrength(Sorting.Strength.QUATERNARY); }
- | <IDENTICAL> { sorting.setStrength(Sorting.Strength.IDENTICAL); }
- )
- | <LOCALE> <COLON> locale = identifierWithDash() { sorting.setLocale(locale); }
- )
-}
-
-/**
- * This rule consumes a single attribute setting statement of an attribute element.
- *
- * @param field The field to modify.
- * @param attributeName The name of the attribute to change.
- */
-void attributeSetting(FieldOperationContainer field, AttributeOperation attribute, String attributeName) :
-{
- String str;
-}
-{
- (
- <HUGE> { attribute.setHuge(true); }
- | <FASTSEARCH> { attribute.setFastSearch(true); }
- | <FASTACCESS> { attribute.setFastAccess(true); }
- | <MUTABLE> { attribute.setMutable(true); }
- | <PAGED> { attribute.setPaged(true); }
- | <ENABLEBITVECTORS> { attribute.setEnableBitVectors(true); }
- | <ENABLEONLYBITVECTOR> { attribute.setEnableOnlyBitVector(true); }
- | sorting(field, attributeName)
- | <ALIAS> { String alias; String aliasedName=attributeName; } [aliasedName = identifier()] <COLON> alias = identifierWithDash() {
- attribute.setDoAlias(true);
- attribute.setAlias(alias);
- attribute.setAliasedName(aliasedName);
- }
- | attributeTensorType(attribute)
- | <DISTANCEMETRIC> <COLON> str = identifierWithDash() { attribute.setDistanceMetric(str); }
- )
-}
-
-/**
- * This rule consumes a tensor type statement for an attribute element.
- *
- * @param attribute The attribute to modify.
- */
-void attributeTensorType(AttributeOperation attribute) :
-{
- TensorType tensorType;
-}
-{
- tensorType = tensorType("For attribute field '" + attribute.getName() + "'")
- {
- // TODO: Remove on Vespa 8
- deployLogger.logApplicationPackage(Level.WARNING, "In field '" + attribute.getName() + "': Specifying tensor type on the attribute is deprecated and has no effect.");
- }
-}
-
-/**
- * This rule consumes a summary statement defined inside a document-summary block.
- *
- * @param document The document summary to modify.
- */
-void summaryInDocument(DocumentSummary document) :
-{
- String name;
- DataType type;
- SummaryField summary;
-
-}
-{
- <SUMMARY> name = identifierWithDash() { }
- <TYPE> type = dataType() {
- summary = new SummaryField(name, type);
- summary.setVsmCommand(SummaryField.VsmCommand.FLATTENSPACE);
-
- SummaryInFieldLongOperation op = new SummaryInFieldLongOperation();
- }
- lbrace() (summaryItem(op) (<NL>)*)* <RBRACE>
- {
- if (op.destinationIterator().hasNext()) {
- throw new ParseException("Summaries defined in a document-summary section " +
- "can not have a 'to' line.");
- }
- op.applyToSummary(summary);
- document.add(summary);
- }
-}
-
-/**
- * The rule consumes a summary statement defined inside a field.
- *
- * @param field The field to modify.
- */
-void summaryInField(FieldOperationContainer field) :
-{
- SummaryInFieldOperation summary;
-}
-{
- ( <SUMMARY> ( LOOKAHEAD(2) summary = summaryInFieldShort(field)
- | summary = summaryInFieldLong(field)) )
- {
- field.addOperation(summary);
- }
-}
-
-/**
- * This rule consumes a single-line summary field.
- *
- * @param field The field to modify.
- * @return The consumed summary field.
- */
-SummaryInFieldOperation summaryInFieldShort(FieldOperationContainer field) :
-{
- String name = field.getName();
- SummaryField ret;
-}
-{
- [ name = identifier() ]
- {
- SummaryInFieldShortOperation op = new SummaryInFieldShortOperation(name);
- }
- <COLON> ( <DYNAMIC> { op.setTransform(SummaryTransform.DYNAMICTEASER);
- op.addSource(name);
- }
- | <MATCHEDELEMENTSONLY> { op.setTransform(SummaryTransform.MATCHED_ELEMENTS_FILTER); }
- | (<FULL> | <STATIC>) { op.setTransform(SummaryTransform.NONE); } )
- { return op; }
-}
-
-/**
- * This rule consumes a multi-line summary field.
- *
- * @return The consumed summary field.
- */
-SummaryInFieldOperation summaryInFieldLong(FieldOperationContainer field) :
-{
- String name = field.getName();
- DataType type = null;
-}
-{
- ( [ name = identifier() [ <TYPE> type = dataType() ] ]
- lbrace()
- {
- SummaryInFieldLongOperation op = new SummaryInFieldLongOperation(name);
- op.setType(type);
- }
- (summaryItem(op) (<NL>)*)* <RBRACE> )
- { return op; }
-}
-
-/**
- * This rule consumes an item of a summary field block.
- *
- * @param field The field to modify.
- */
-void summaryItem(SummaryInFieldLongOperation field) : { }
-{
- ( summaryTransform(field)
- | summaryBolding(field)
- | summarySourceList(field)
- | summaryDestinationList(field)
- | summaryProperties(field) )
-}
-
-/**
- * This rule consumes a transform statement for a summary field element.
- *
- * @param field The field to modify.
- */
-void summaryTransform(SummaryInFieldOperation field) : { }
-{
- ( <DYNAMIC> { field.setTransform(SummaryTransform.DYNAMICTEASER); }
- | <MATCHEDELEMENTSONLY> { field.setTransform(SummaryTransform.MATCHED_ELEMENTS_FILTER); }
- | (<FULL> | <STATIC>) { field.setTransform(SummaryTransform.NONE); } )
-}
-
-/**
- * This rule consumes a bolding statement for a summary field element.
- *
- * @param field The summary field to modify.
- */
-void summaryBolding(SummaryInFieldLongOperation field) :
-{
- boolean bold;
-}
-{
- <BOLDING> <COLON> bold = bool()
- { field.setBold(bold); }
-}
-
-/**
- * This rule consumes a source-list statement for a summary field element.
- *
- * @param field The summary field to modify.
- */
-void summarySourceList(SummaryInFieldOperation field) :
-{
- String str;
-}
-{
- ( <SOURCE> <COLON> str = identifier() { field.addSource(str); }
- ( <COMMA> str = identifier() { field.addSource(str); } )* ) +
-}
-
-/**
- * This rule consumes a destination-list statement for a summary field element.
- *
- * @param field The summary field to modify.
- */
-void summaryDestinationList(SummaryInFieldLongOperation field) :
-{
- String str;
-}
-{
- <TO> <COLON> str = identifier() { field.addDestination(str); }
- ( <COMMA> str = identifier() { field.addDestination(str); } )*
-}
-
-/**
- * This rule consumes properties for a summary field element.
- *
- * @param field The summary field to modify.
- */
-void summaryProperties(SummaryInFieldLongOperation field) : { }
-{
- <PROPERTIES> lbrace() (summaryProperty(field) <NL>)+ <RBRACE>
-}
-
-/**
- * This rule consumes a single summary property pair for a summary field element.
- *
- * @param field The summary field to modify.
- */
-void summaryProperty(SummaryInFieldLongOperation field) :
-{
- String name, value;
-}
-{
- name = identifierWithDash() <COLON> (value = identifierWithDash() | value = quotedString())
- { deployLogger.logApplicationPackage(Level.WARNING, "Specifying summary properties is deprecated and has no effect."); }
-}
-
-/**
- * This rule consumes a stemming block of a field element.
- *
- * @param field The field to modify.
- */
-void fieldStemming(FieldOperationContainer field) :
-{
- String setting;
- StemmingOperation op = new StemmingOperation();
-}
-{
- <STEMMING> <COLON> setting = identifierWithDash()
- {
- op.setSetting(setting);
- field.addOperation(op);
- }
-}
-
-/**
- * This rule consumes a stemming statement for a schema element.
- *
- * @param schema the schema to modify
- */
-void searchStemming(Schema schema) :
-{
- String setting;
-}
-{
- <STEMMING> <COLON> setting = identifierWithDash()
- { schema.setStemming(Stemming.get(setting)); }
-}
-
-/**
- * This rule consumes a normalizing statement of a field element. At the moment, this can only be used to turn off
- * normalizing.
- *
- * @param field The field to modify.
- */
-void normalizing(FieldOperationContainer field) :
-{
- String setting;
-}
-{
- <NORMALIZING> <COLON> setting = identifierWithDash()
- {
- field.addOperation(new NormalizingOperation(setting));
- }
-}
-
-/**
- * This rule consumes a bolding statement of a field element.
- *
- * @param field The field to modify.
- */
-void bolding(FieldOperationContainer field) :
-{
- boolean bold;
-}
-{
- <BOLDING> <COLON> bold = bool()
- {
- field.addOperation(new BoldingOperation(bold));
- }
-}
-
-/**
- * This rule consumes a dictionary statement of a field element.
- *
- * @param field The field to modify.
- */
-void dictionary(FieldOperationContainer field) :
-{
-}
-{
- <DICTIONARY>
- ( (<COLON> dictionarySetting(field))
- | (lbrace() (dictionarySetting(field) (<NL>)*)* <RBRACE>))
- {
- }
-}
-
-void dictionarySetting(FieldOperationContainer field) :
-{
- Dictionary.Type type;
-}
-{
- ( <HASH> { field.addOperation(new DictionaryOperation(DictionaryOperation.Operation.HASH)); }
- | <BTREE> { field.addOperation(new DictionaryOperation(DictionaryOperation.Operation.BTREE)); }
- | <CASED> { field.addOperation(new DictionaryOperation(DictionaryOperation.Operation.CASED)); }
- | <UNCASED> { field.addOperation(new DictionaryOperation(DictionaryOperation.Operation.UNCASED)); })
- {
- }
-}
-
-/**
- * This rule consumes a body statement of a field element.
- *
- * @param field The field to modify.
- */
-void body(SDField field) : { }
-{
- <BODY>
- {
- deployLogger.logApplicationPackage(Level.WARNING, field + ": 'header/body' is deprecated and has no effect.");
- }
-}
-
-/**
- * This rule consumes a header statement of a field element.
- *
- * @param field The field to modify.
- */
-void header(SDField field) : { }
-{
- <HEADER>
- {
- deployLogger.logApplicationPackage(Level.WARNING, field + ": 'header/body' is deprecated and has no effect.");
- }
-}
-
-void queryCommand(FieldOperationContainer container) :
-{
- String command;
- QueryCommandOperation field = new QueryCommandOperation();
-}
-{
- <QUERYCOMMAND> <COLON> ( command = identifierWithDash() | command = quotedString() )
- {
- field.addQueryCommand(command);
- container.addOperation(field);
- }
-}
-
-void alias(FieldOperationContainer container) :
-{
- String aliasedName = null;
- String alias;
-}
-{
- <ALIAS> [aliasedName = identifier()] <COLON> alias = identifierWithDash()
- {
- AliasOperation op = new AliasOperation(aliasedName, alias);
- container.addOperation(op);
- }
-}
-
-FieldOperationContainer match(FieldOperationContainer field) : { }
-{
- <MATCH> ( (<COLON> matchType(field))
- | (lbrace() (matchItem(field) (<NL>)*)* <RBRACE>) )
- { return field; }
-}
-
-/**
- * This rule consumes a single match item for a match block.
- *
- * @param field The field to modify.
- */
-void matchItem(FieldOperationContainer field) : { }
-{
- ( matchType(field) | exactTerminator(field) | gramSize(field) | matchSize(field) )
-}
-
-void matchType(FieldOperationContainer container) :
-{
- MatchOperation matchOp = new MatchOperation();
-}
-{
- ( <MTOKEN> { matchOp.setMatchingType(MatchType.TEXT); } // Deprecated synonym to TEXT
- | <TEXT> { matchOp.setMatchingType(MatchType.TEXT); }
- | <WORD> { matchOp.setMatchingType(MatchType.WORD); }
- | <EXACT> { matchOp.setMatchingType(MatchType.EXACT); }
- | <GRAM> { matchOp.setMatchingType(MatchType.GRAM); }
- | <CASED> { matchOp.setCase(Case.CASED); }
- | <UNCASED> { matchOp.setCase(Case.UNCASED); }
- | <PREFIX> { matchOp.setMatchingAlgorithm(MatchAlgorithm.PREFIX); }
- | <SUBSTRING> { matchOp.setMatchingAlgorithm(MatchAlgorithm.SUBSTRING); }
- | <SUFFIX> { matchOp.setMatchingAlgorithm(MatchAlgorithm.SUFFIX); } )
- {
- container.addOperation(matchOp);
- }
-}
-
-void exactTerminator(FieldOperationContainer container) :
-{
- String terminator;
- MatchOperation field = new MatchOperation();
-}
-{
- <EXACTTERMINATOR> <COLON> terminator = quotedString()
- {
- field.setExactMatchTerminator(terminator);
- container.addOperation(field);
- }
-}
-
-void gramSize(FieldOperationContainer container) :
-{
- int gramSize;
- MatchOperation field = new MatchOperation();
-}
-{
- <GRAMSIZE> <COLON> gramSize = integer()
- {
- field.setGramSize(gramSize);
- container.addOperation(field);
- }
-}
-
-void matchSize(FieldOperationContainer container) :
-{
- int matchSize;
- MatchOperation field = new MatchOperation();
-}
-{
- <MAXLENGTH> <COLON> matchSize = integer()
- {
- field.setMaxLength(matchSize);
- container.addOperation(field);
- }
-}
-/**
- * Consumes a rank statement of a field element.
- *
- * @param field The field to modify.
- */
-void rank(FieldOperationContainer field) :
-{
- RankOperation op = new RankOperation();
-}
-{
- <RANK> ( (<COLON> rankSetting(op))
- | (lbrace() (rankSetting(op) (<NL>)*)* <RBRACE>) )
- {
- field.addOperation(op);
- }
-}
-
-/**
- * Consumes a single rank setting of a rank statement.
- *
- * @param field The field to modify.
- */
-void rankSetting(RankOperation field) : { }
-{
- ( <LITERAL> { field.setLiteral(true); }
- | <NORMAL> { field.setNormal(true); }
- | <FILTER> { field.setFilter(true); } )
-}
-
-/**
- * Consumes an id statement of a field body block.
- *
- * @param field The field to modify.
- * @param document The document type to modify.
- */
-void id(FieldOperationContainer field, SDDocumentType document) :
-{
- int fieldId;
- IdOperation op = new IdOperation();
-}
-{
- <ID> <COLON> fieldId = integer()
- {
- op.setDocument(document);
- op.setFieldId(fieldId);
- field.addOperation(op);
- }
-}
-
-/**
- * Consumes an indexing-rewrite statement of a field body block.
- *
- * @param field The field to modify.
- */
-void indexingRewrite(FieldOperationContainer field) : { }
-{
- <INDEXINGREWRITE> <COLON> <NONE>
- { deployLogger.logApplicationPackage(Level.WARNING, "Specifying 'indexing-rewrite' is deprecated and has no effect."); }
-}
-
-/**
- * Consumes a document-summary block from within a schema block.
- *
- * @param schema the schema object to add content to
- */
-void documentSummary(Schema schema) :
-{
- String name;
- DocumentSummary summary;
-}
-{
- ( <DOCUMENTSUMMARY>
- name = identifierWithDash() { schema.addSummary(summary = new DocumentSummary(name, schema)); }
- [inheritsDocumentSummary(summary, schema)]
- lbrace()
- (
- <FROMDISK> { summary.setFromDisk(true); } |
- <OMITSUMMARYFEATURES> { summary.setOmitSummaryFeatures(true); } |
- documentSummaryItem(summary) |
- <NL>
- )*
- <RBRACE>
- )
-}
-
-/**
- * This rule consumes an inherits statement of a document summary.
- *
- * @param documentSummary the document summary to modify
- * @param schema the schema object documentSummary is being added to
- */
-void inheritsDocumentSummary(DocumentSummary documentSummary, Schema schema) :
-{
- String name;
-}
-{
- <INHERITS> name = identifierWithDash()
- {
- documentSummary.setInherited(name);
- }
-}
-
-/**
- * Consumes a single document-summary item.
- *
- * @param summary The document summary to modify.
- */
-void documentSummaryItem(DocumentSummary summary) : { }
-{
- summaryInDocument(summary)
-}
-
-/**
- * Consumes an index block for a field element.
- *
- * @param schema the schema object to add content to
- * @param field the field to modify
- */
-void index(Schema schema, FieldOperationContainer field) :
-{
- IndexOperation op = new IndexOperation();
- String indexName = (field != null) ? field.getName() : null;
-}
-{
- <INDEX> [indexName = identifier()]
- {
- if (indexName == null) {
- throw new ParseException("Index statements outside fields must have an explicit name.");
- }
- op.setIndexName(indexName);
- }
- ( (<COLON> indexBody(op) (<COMMA> indexBody(op))*) |
- (lbrace() (indexBody(op) (<NL>)*)* <RBRACE>) )
- {
- if (field == null) {
-
- Index index = new Index(indexName);
- op.applyToIndex(index);
- schema.addIndex(index);
- } else {
- field.addOperation(op);
- }
- }
-}
-
-/**
- * Consumes a single index statement for an index block.
- *
- * @param index The index to modify.
- */
-void indexBody(IndexOperation index) :
-{
- String str;
- int arity;
- long num;
- double threshold;
-}
-{
- ( <PREFIX> { index.setPrefix(true); }
- | <ALIAS> <COLON> str = identifierWithDash() { index.addAlias(str); }
- | <STEMMING> <COLON> str = identifierWithDash() { index.setStemming(str); }
- | <ARITY> <COLON> arity = integer() { index.setArity(arity); }
- | <LOWERBOUND> <COLON> num = consumeLong() { index.setLowerBound(num); }
- | <UPPERBOUND> <COLON> num = consumeLong() { index.setUpperBound(num); }
- | <DENSEPOSTINGLISTTHRESHOLD> <COLON> threshold = consumeFloat() { index.setDensePostingListThreshold(threshold); }
- | <ENABLE_BM25> { index.setEnableBm25(true); }
- | hnswIndex(index) { }
- )
-}
-
-void hnswIndex(IndexOperation index) :
-{
- HnswIndexParams.Builder params = new HnswIndexParams.Builder();
-}
-{
- ( LOOKAHEAD(<HNSW> lbrace())
- <HNSW> ( (lbrace() (hnswIndexBody(params) (<NL>)*)* <RBRACE>) ) |
- <HNSW> )
- {
- index.setHnswIndexParams(params);
- }
-}
-
-void hnswIndexBody(HnswIndexParams.Builder params) :
-{
- int num;
- boolean bool;
-}
-{
- ( <MAXLINKSPERNODE> <COLON> num = integer() { params.setMaxLinksPerNode(num); }
- | <NEIGHBORSTOEXPLOREATINSERT> <COLON> num = integer() { params.setNeighborsToExploreAtInsert(num); }
- | <MULTITHREADEDINDEXING> <COLON> bool = bool() { params.setMultiThreadedIndexing(bool); } )
-}
-
-/**
- * Consumes a onnx-model block of a schema element.
- *
- * @param schema the schema object to add content to.
- */
-void onnxModel(Schema schema) :
-{
- String name;
- OnnxModel onnxModel;
-}
-{
- ( <ONNXMODEL> name = identifier()
- {
- onnxModel = new OnnxModel(name);
- }
- lbrace() (onnxModelItem(onnxModel) (<NL>)*)+ <RBRACE> )
- {
- if (documentsOnly) return;
- schema.onnxModels().add(onnxModel);
- }
-}
-
-/**
- * This rule consumes an onnx-model block.
- *
- * @param onnxModel The onnxModel to modify.
- */
-void onnxModelItem(OnnxModel onnxModel) :
-{
- String path = null;
-}
-{
- (
- (path = fileItem()) { onnxModel.setFileName(path); } |
- (path = uriItem()) { onnxModel.setUri(path); } |
- (<ONNX_INPUT_SL>) {
- String name = token.image.substring(5, token.image.lastIndexOf(":")).trim();
- if (name.startsWith("\"")) { name = name.substring(1, name.length() - 1); }
- String source = token.image.substring(token.image.lastIndexOf(":") + 1).trim();
- onnxModel.addInputNameMapping(name, source);
- } |
- (<ONNX_OUTPUT_SL>) {
- String name = token.image.substring(6, token.image.lastIndexOf(":")).trim();
- if (name.startsWith("\"")) { name = name.substring(1, name.length() - 1); }
- String as = token.image.substring(token.image.lastIndexOf(":") + 1).trim();
- onnxModel.addOutputNameMapping(name, as);
- }
- )
-}
-
-/**
- * Consumes a constant block of a schema element.
- *
- * @param schema the schema object to add content to
- */
-void rankingConstant(Schema schema) :
-{
- String name;
- String path = null;
- DistributableResource.PathType pathType = DistributableResource.PathType.FILE;
- TensorType type = null;
-}
-{
- ( <CONSTANT> name = identifier() lbrace()
- (path = fileItem() { pathType = DistributableResource.PathType.FILE; }
- | path = uriItem() { pathType = DistributableResource.PathType.URI; }
- | type = tensorTypeWithPrefix(rankingConstantErrorMessage(name)) (<NL>)*
- )+
- <RBRACE>
- )
- {
- if (documentsOnly) return;
- schema.rankingConstants().add(new RankingConstant(name, type, path, pathType));
- }
-}
-
-String fileItem() :
-{
- String path;
-}
-{
- (<FILE> <COLON> ( <FILE_PATH> | <STRING> | <IDENTIFIER>) { path = com.yahoo.path.Path.fromString(token.image).getRelative(); } { } (<NL>)*) { return path; }
-}
-String uriItem() :
-{
- String path;
-}
-{
- (<URI> <COLON> ( <URI_PATH> ) { path = token.image; } (<NL>)*) { return path; }
-}
-
-String rankingConstantErrorMessage(String name) : {}
-{
- { return "For ranking constant ' " + name + "'"; }
-}
-
-/**
- * Consumes a rank-profile block of a schema element.
- *
- * @param schema the schema object to add content to
- */
-void rankProfile(Schema schema) :
-{
- String name;
- RankProfile profile;
-}
-{
- ( ( <MODEL> | <RANKPROFILE> ) name = identifierWithDash()
- {
- if (documentsOnly) {
- profile = new DocumentsOnlyRankProfile(name, schema, rankProfileRegistry, schema.rankingConstants());
- }
- else if ("default".equals(name)) {
- profile = rankProfileRegistry.get(schema, "default");
- } else {
- profile = new RankProfile(name, schema, rankProfileRegistry, schema.rankingConstants());
- }
- }
- [inheritsRankProfile(profile)]
- lbrace() (rankProfileItem(profile) (<NL>)*)* <RBRACE> )
- {
- if (documentsOnly) return;
- rankProfileRegistry.add(profile);
- }
-}
-
-/**
- * This rule consumes a single statement for a rank-profile block.
- *
- * @param profile The rank profile to modify.
- */
-void rankProfileItem(RankProfile profile) : { }
-{
- ( fieldRankType(profile)
- | fieldWeight(profile)
- | fieldRankFilter(profile)
- | firstPhase(profile)
- | matchPhase(profile)
- | function(profile)
- | mutate(profile)
- | ignoreRankFeatures(profile)
- | numThreadsPerSearch(profile)
- | minHitsPerThread(profile)
- | numSearchPartitions(profile)
- | termwiseLimit(profile)
- | postFilterThreshold(profile)
- | approximateThreshold(profile)
- | rankFeatures(profile)
- | rankProperties(profile)
- | secondPhase(profile)
- | rankDegradation(profile)
- | inputs(profile)
- | constants(profile)
- | matchFeatures(profile)
- | summaryFeatures(profile)
- | strict(profile) )
-}
-
-/**
- * This rule consumes an inherits statement of a rank-profile.
- *
- * @param profile the profile to modify
- */
-void inheritsRankProfile(RankProfile profile) :
-{
- String name;
-}
-{
- <INHERITS> name = identifierWithDash() { profile.inherit(name); }
- ( <COMMA> name = identifierWithDash() { profile.inherit(name);; } )*
-}
-
-/**
- * This rule consumes an mutate statement of a rank-profile.
- *
- * @param profile The profile to modify.
- */
-void mutate(RankProfile profile) :
-{
-}
-{
- <MUTATE> lbrace() (mutate_operation(profile) <NL>)+ <RBRACE>
- { }
-}
-
-void mutate_operation(RankProfile profile) :
-{
- String attribute, operation;
- RankProfile.MutateOperation.Phase phase;
-}
-{
- ( <ON_MATCH> { phase = RankProfile.MutateOperation.Phase.on_match; }
- | <ON_FIRST_PHASE> { phase = RankProfile.MutateOperation.Phase.on_first_phase; }
- | <ON_SECOND_PHASE> { phase = RankProfile.MutateOperation.Phase.on_second_phase; }
- | <ON_SUMMARY> { phase = RankProfile.MutateOperation.Phase.on_summary; }
- )
- lbrace() attribute = identifier() operation = mutate_expr() (<NL>)* <RBRACE>
- { profile.addMutateOperation(phase, attribute, operation); }
-}
-
-String mutate_expr() :
-{
- String op;
- Number constant = null;
-}
-{
- (("+=" | "-=" | "=") { op = token.image; } constant = consumeNumber())
- { return constant != null ? (op + constant) : op; }
-}
-
-/**
- * This rule consumes a function statement of a rank-profile.
- *
- * @param profile The profile to modify.
- */
-void function(RankProfile profile) :
-{
- String name, expression, parameter;
- List parameters = new ArrayList();
- boolean inline = false;
-}
-{
- ( ( <FUNCTION> | <MACRO> ) inline = inline() name = identifier() [ "$" { name = name + token.image; } ]
- "("
- [ parameter = identifier() { parameters.add(parameter); }
- ( <COMMA> parameter = identifier() { parameters.add(parameter); } )* ]
- ")"
- lbrace() expression = expression() (<NL>)* <RBRACE> )
- { profile.addFunction(name, parameters, expression, inline); }
-}
-
-boolean inline() :
-{
-}
-{
- ( <INLINE> { return true; } ) ?
- { return false; }
-}
-
-/**
- * This rule consumes a match-phase block of a rank profile.
- *
- * @param profile The rank profile to modify.
- */
-void matchPhase(RankProfile profile) :
-{
- MatchPhaseSettings settings = new MatchPhaseSettings();
-}
-{
- <MATCHPHASE> lbrace() (matchPhaseItem(settings) (<NL>)*)* <RBRACE>
- {
- settings.checkValid();
- profile.setMatchPhaseSettings(settings);
- }
-}
-
-void matchPhaseItem(MatchPhaseSettings settings) :
-{
- String str;
- int num;
- double multiplier;
- double coverage;
-}
-{
- ( <ATTRIBUTE> <COLON> str = identifier() { settings.setAttribute(str); }
- | diversity(settings)
- | <ORDER> <COLON> ( <ASCENDING> { settings.setAscending(true); }
- | <DESCENDING> { settings.setAscending(false); } )
- | <MAXHITS> <COLON> num = integer() { settings.setMaxHits(num); }
- | <MAXFILTERCOVERAGE> <COLON> coverage = consumeFloat() { settings.setMaxFilterCoverage(coverage); }
- | <EVALUATION_POINT> <COLON> multiplier = consumeFloat() { settings.setEvaluationPoint(multiplier); }
- | <PRE_POST_FILTER_TIPPING_POINT> <COLON> multiplier = consumeFloat() { settings.setPrePostFilterTippingPoint(multiplier); }
- )
- { return; }
-}
-
-/**
- * This rule consumes a diversity block of a rank profile.
- *
- * @param profile The rank profile to modify.
- */
-void diversity(MatchPhaseSettings profile) :
-{
- DiversitySettings settings = new DiversitySettings();
-}
-{
- <DIVERSITY> lbrace() (diversityItem(settings) (<NL>)*)* <RBRACE>
- {
- profile.setDiversity(settings);
- }
-}
-
-void diversityItem(DiversitySettings settings) :
-{
- String str;
- int num;
- double multiplier;
-}
-{
- ( <ATTRIBUTE> <COLON> str = identifier() { settings.setAttribute(str); }
- | <MIN_GROUPS> <COLON> num = integer() { settings.setMinGroups(num); }
- | <CUTOFF_FACTOR> <COLON> multiplier = consumeFloat() { settings.setCutoffFactor(multiplier); }
- | <CUTOFF_STRATEGY> <COLON>
- ( <STRICT> { settings.setCutoffStrategy(Diversity.CutoffStrategy.strict); }
- | <LOOSE> { settings.setCutoffStrategy(Diversity.CutoffStrategy.loose); }
- )
- )
- { return; }
-}
-
-
-
-/**
- * Consumes the first-phase block of a rank profile.
- *
- * @param profile The rank profile to modify.
- */
-void firstPhase(RankProfile profile) :
-{
- String exp;
-}
-{
- <FIRSTPHASE> lbrace() (firstPhaseItem(profile) (<NL>)*)* <RBRACE>
-}
-
-void firstPhaseItem(RankProfile profile) :
-{
- String expression;
- int rerankCount;
- double dropLimit;
-}
-{
- ( expression = expression() { profile.setFirstPhaseRanking(expression); }
- | (<KEEPRANKCOUNT> <COLON> rerankCount = integer()) { profile.setKeepRankCount(rerankCount); }
- | (<RANKSCOREDROPLIMIT> <COLON> dropLimit = consumeFloat()) { profile.setRankScoreDropLimit(dropLimit); }
- )
-}
-
-/**
- * Consumes the second-phase block of a rank profile.
- *
- * @param profile The rank profile to modify.
- */
-void secondPhase(RankProfile profile) : { }
-{
- <SECONDPHASE> lbrace() (secondPhaseItem(profile) (<NL>)*)* <RBRACE>
-}
-
-/**
- * Consumes a statement for a second-phase block.
- *
- * @param profile The rank profile to modify.
- */
-void secondPhaseItem(RankProfile profile) :
-{
- String expression;
- int rerankCount;
-}
-{
- ( expression = expression() { profile.setSecondPhaseRanking(expression); }
- | (<RERANKCOUNT> <COLON> rerankCount = integer()) { profile.setRerankCount(rerankCount); }
- )
-}
-
-/** Consumes an inputs block of a rank profile. */
-void inputs(RankProfile profile) : {}
-{
- <INPUTS> <LBRACE> (<NL>)*
- ( input(profile) (<NL>)*) *
- <RBRACE>
-}
-
-void input(RankProfile profile) :
-{
- Reference reference;
- TensorType type;
-}
-{
- reference = queryFeature() type = inputType(reference)
- {
- profile.addInput(reference, new RankProfile.Input(reference, type, Optional.empty()));
- }
-}
-
-TensorType inputType(Reference reference) :
-{
- TensorType type;
-}
-{
- (
- ( type = tensorType("Type of " + reference) )
- |
- ( <DOUBLE_KEYWORD> { type = TensorType.empty; } )
- )
- { return type; }
-}
-
-Reference queryFeature() :
-{
- String argument;
-}
-{
- <QUERY> "(" argument = identifier() ")"
- { return Reference.simple("query", argument); }
-}
-
-/** Consumes a summary-features block of a rank profile. */
-void summaryFeatures(RankProfile profile) :
-{
- String features;
- String inherited = null;
-}
-{
- ( <SUMMARYFEATURES_SL> { features = token.image.substring(token.image.indexOf(":") + 1).trim(); } |
- <SUMMARYFEATURES_ML> { features = token.image.substring(token.image.indexOf("{") + 1,
- token.image.lastIndexOf("}")).trim(); } |
- <SUMMARYFEATURES_ML_INHERITS> {
- int inheritsIndex = token.image.indexOf("inherits ");
- String rest = token.image.substring(inheritsIndex + "inherits ".length());
- profile.setInheritedSummaryFeatures(rest.substring(0, rest.indexOf(" ")).trim());
- features = token.image.substring(token.image.indexOf("{") + 1, token.image.lastIndexOf("}")).trim();
- }
- )
- {
- profile.addSummaryFeatures(getFeatureList(features));
- }
-}
-
-void strict(RankProfile profile) :
-{}
-{
- <STRICT> <COLON> (
- ( <TRUE> { profile.setStrict(true); } ) |
- ( <FALSE> { profile.setStrict(false); } )
- )
-}
-
-/**
- * This rule consumes a match-features block of a rank profile.
- *
- * @param profile The rank profile to modify.
- */
-void matchFeatures(RankProfile profile) :
-{
- String features;
-}
-{
- ( <MATCHFEATURES_SL> { features = token.image.substring(token.image.indexOf(":") + 1).trim(); } |
- <MATCHFEATURES_ML> { features = token.image.substring(token.image.indexOf("{") + 1,
- token.image.lastIndexOf("}")).trim(); } |
- <MATCHFEATURES_ML_INHERITS> {
- int inheritsIndex = token.image.indexOf("inherits ");
- String rest = token.image.substring(inheritsIndex + "inherits ".length());
- profile.setInheritedMatchFeatures(rest.substring(0, rest.indexOf(" ")).trim());
- features = token.image.substring(token.image.indexOf("{") + 1, token.image.lastIndexOf("}")).trim();
- }
- )
- {
- profile.addMatchFeatures(getFeatureList(features));
- }
-}
-
-/** Consumes a rank-features block of a rank profile */
-void rankFeatures(RankProfile profile) :
-{
- String features;
-}
-{
- ( <RANKFEATURES_SL> { features = token.image.substring(token.image.indexOf(":") + 1).trim(); } |
- <RANKFEATURES_ML> { features = token.image.substring(token.image.indexOf("{") + 1,
- token.image.lastIndexOf("}")).trim(); } )
- {
- profile.addRankFeatures(getFeatureList(features));
- }
-}
-
-/**
- * This rule consumes a ignore-default-rank-features statement for a rank profile.
- *
- * @param profile The rank profile to modify.
- */
-void ignoreRankFeatures(RankProfile profile) : { }
-{
- <IGNOREDEFAULTRANKFEATURES> { profile.setIgnoreDefaultRankFeatures(true); }
-}
-
-/**
- * This rule consumes a num-threads-per-search statement for a rank profile.
- *
- * @param profile The rank profile to modify.
- */
-void numThreadsPerSearch(RankProfile profile) :
-{
- int num;
-}
-{
- (<NUMTHREADSPERSEARCH> <COLON> num = integer()) { profile.setNumThreadsPerSearch(num); }
-}
-
-/**
- * This rule consumes a min-hits-per-thread statement for a rank profile.
- *
- * @param profile The rank profile to modify.
- */
-void minHitsPerThread(RankProfile profile) :
-{
- int num;
-}
-{
- (<MINHITSPERTHREAD> <COLON> num = integer()) { profile.setMinHitsPerThread(num); }
-}
-
-/**
- * This rule consumes a num-search-partitions statement for a rank profile.
- *
- * @param profile the rank profile to modify
- */
-void numSearchPartitions(RankProfile profile) :
-{
- int num;
-}
-{
- (<NUMSEARCHPARTITIONS> <COLON> num = integer()) { profile.setNumSearchPartitions(num); }
-}
-
-/**
- * This rule consumes a termwise-limit statement for a rank profile.
- *
- * @param profile the rank profile to modify
- */
-void termwiseLimit(RankProfile profile) :
-{
- double num;
-}
-{
- (<TERMWISELIMIT> <COLON> num = consumeFloat()) { profile.setTermwiseLimit(num); }
-}
-
-/**
- * This rule consumes a post-filter-threshold statement for a rank profile.
- *
- * @param profile the rank profile to modify
- */
-void postFilterThreshold(RankProfile profile) :
-{
- double threshold;
-}
-{
- (<POSTFILTERTHRESHOLD> <COLON> threshold = consumeFloat()) { profile.setPostFilterThreshold(threshold); }
-}
-
-/**
- * This rule consumes an approximate-threshold statement for a rank profile.
- *
- * @param profile the rank profile to modify
- */
-void approximateThreshold(RankProfile profile) :
-{
- double threshold;
-}
-{
- (<APPROXIMATETHRESHOLD> <COLON> threshold = consumeFloat()) { profile.setApproximateThreshold(threshold); }
-}
-
-/**
- * This rule consumes a rank-properties block of a rank profile. There is a little trick within this rule to allow the
- * final rank property to skip the terminating newline token.
- *
- * @param profile the rank profile to modify
- */
-void rankProperties(RankProfile profile) : { }
-{
- <RANKPROPERTIES> lbrace() (LOOKAHEAD(rankPropertyItem() <COLON> rankPropertyItem() <NL>)
- rankProperty(profile) (<NL>)+)* [rankProperty(profile)] <RBRACE>
-}
-
-/**
- * This rule consumes a single rank property pair for a rank profile.
- *
- * @param profile the rank profile to modify
- */
-void rankProperty(RankProfile profile) :
-{
- String key, val;
-}
-{
- key = rankPropertyItem() <COLON> val = rankPropertyItem()
- { profile.addRankProperty(key, val); }
-}
-
-
-/**
- * This rule consumes a single rank property for a rank-properties block.
- *
- * @return The token image of the consumed item.
- */
-String rankPropertyItem() :
-{
- String image;
- String ret = "";
- Token dToken;
-}
-{
- ( ( image = identifierWithDash() { ret += image; }
- | dToken = <DOUBLE> { ret += dToken.image; }
- | image = quotedString() { ret += image; }
- | ( "(" | ")" | <DOT> | <COMMA> ) { ret += token.image; } )+ )
- { return ret; }
-}
-
-/**
- * This rule consumes a field-weight statement of a rank profile.
- *
- * @param profile The rank profile to modify.
- */
-void fieldWeight(RankProfile profile) :
-{
- Integer num;
- String name;
-}
-{
- <WEIGHT> name = identifier() <COLON> num = integer()
- { profile.addRankSetting(name, RankProfile.RankSetting.Type.WEIGHT, num); }
-}
-
-/**
- * This rule consumes a rank-type statement of a rank profile.
- *
- * @param profile The rank profile to modify.
- */
-void fieldRankType(RankProfile profile) :
-{
- String name;
- String type;
-}
-{
- <RANKTYPE> name = identifier() <COLON> type = identifier()
- { profile.addRankSetting(name, RankProfile.RankSetting.Type.RANKTYPE, RankType.fromString(type)); }
-}
-
-/**
- * This rule consumes a rank filter statement of a rank profile.
- *
- * @param profile The rank profile to modify.
- */
-void fieldRankFilter(RankProfile profile) :
-{
- String name;
-}
-{
- <RANK> name = identifier() <COLON> <FILTER>
- { profile.addRankSetting(name, RankProfile.RankSetting.Type.PREFERBITVECTOR, Boolean.TRUE); }
-}
-
-/**
- * This rule consumes part of a rank-degradation statement of a rank profile.
- */
-void rankDegradationBinSize() :
-{
- double freq;
-}
-{
- <RPBINSIZE> <COLON> freq = consumeFloat()
- { deployLogger.logApplicationPackage(Level.WARNING, "Specifying 'doc-frequency' in 'rank-degradation' is deprecated and has no effect."); }
-}
-
-
-/**
- * This rule consumes part of a rank-degradation statement of a rank profile.
- */
-void rankDegradationBinLow() :
-{
- int n;
-}
-{
- <RPBINLOW> <COLON> n = integer()
- { deployLogger.logApplicationPackage(Level.WARNING, "Specifying 'min-fullrank-docs' in 'rank-degradation' is deprecated and has no effect."); }
-}
-
-
-/**
- * This rule consumes part of a rank-degradation statement of a rank profile.
- */
-void rankDegradationPosbinSize() :
-{
- double avgOcc;
-}
-{
- <RPPOSBINSIZE> <COLON> avgOcc = consumeFloat()
- { deployLogger.logApplicationPackage(Level.WARNING, "Specifying 'occurrences-per-doc' in 'rank-degradation' is deprecated and has no effect."); }
-}
-
-
-/**
- * This rule consumes part of a rank-degradation statement of a rank profile.
- */
-void rankDegradationItem() :
-{
-}
-{
- ( rankDegradationBinSize()
- | rankDegradationBinLow()
- | rankDegradationPosbinSize() )
-}
-
-/**
- * This rule consumes a rank-degradation statement of a rank profile.
- *
- * @param profile The rank profile to modify.
- */
-void rankDegradation(RankProfile profile) :
-{
- double freq;
-}
-{
- ( <RANKDEGRADATIONFREQ> <COLON> freq = consumeFloat()
- { deployLogger.logApplicationPackage(Level.WARNING, "Specifying 'rank-degradation-frequency' in 'rank-profile' is deprecated and has no effect."); }
- | <RANKDEGRADATION> lbrace() ( rankDegradationItem() (<NL>)*)+ <RBRACE>
- )
-}
-
-/** Consumes a set of constants available in ranking expressions in the enclosing profile. */
-void constants(RankProfile profile) :
-{
- String name;
-}
-{
- <CONSTANTS> <LBRACE> (<NL>)*
- ( name = identifier() ( constantValue(profile, name) |
- constantTensor(profile, name) ) (<NL>)* )*
- <RBRACE>
-}
-
-void constantValue(RankProfile profile, String name) :
-{
- Token value;
-}
-{
- <COLON> ( value = <DOUBLE> | value = <INTEGER> | value = <IDENTIFIER> ) { profile.addConstant(name, Value.parse(value.image)); }
-}
-
-void constantTensor(RankProfile profile, String name) :
-{
- String tensorString = "";
- TensorType tensorType = null;
-}
-{
- <LBRACE> (<NL>)*
- (( tensorString = tensorValue() |
- tensorType = tensorTypeWithPrefix(constantTensorErrorMessage(profile.name(), name)) ) (<NL>)* )* <RBRACE>
- {
- if (tensorType != null) {
- profile.addConstantTensor(name, new TensorValue(Tensor.from(tensorType, tensorString)));
- } else {
- profile.addConstantTensor(name, new TensorValue(Tensor.from(tensorString)));
- }
- }
-}
-
-String constantTensorErrorMessage(String rankProfileName, String constantTensorName) : {}
-{
- { return "For constant tensor '" + constantTensorName + "' in rank profile '" + rankProfileName + "'"; }
-}
-
-String tensorValue() :
-{
- String tensor;
-}
-{
- ( <TENSOR_VALUE_SL> { tensor = token.image.substring(token.image.indexOf(":") + 1); } |
- <TENSOR_VALUE_ML> { tensor = token.image.substring(token.image.indexOf("{") + 1,
- token.image.lastIndexOf("}")); } )
- {
- return tensor;
- }
-}
-
-TensorType tensorTypeWithPrefix(String errorMessage) :
-{ TensorType type; }
-{
- <TYPE> <COLON> type= tensorType(errorMessage)
- { return type; }
-}
-
-TensorType tensorType(String errorMessage) :
-{
- String tensorTypeString;
-}
-{
- <TENSOR_TYPE> { tensorTypeString = token.image; }
- {
- TensorType tensorType;
- try {
- tensorType = TensorType.fromSpec(tensorTypeString);
- } catch (IllegalArgumentException e) {
- throw new IllegalArgumentException(errorMessage + ": Illegal tensor type spec: " + e.getMessage());
- }
- return tensorType;
- }
-}
-
-void importField(Schema schema) :
-{
- String fieldRefSpec;
- String aliasFieldName;
-}
-{
- <IMPORT> <FIELD> fieldRefSpec = identifier() <AS> aliasFieldName = identifier() lbrace()
- <RBRACE>
- {
- long nDots = Utils.count(fieldRefSpec, '.');
- if (nDots != 1) {
- throw new IllegalArgumentException("Illegal field reference spec '" + fieldRefSpec + "': Does not include a single '.'");
- }
- int indexOfDot = fieldRefSpec.indexOf('.');
- String documentReferenceFieldName = fieldRefSpec.substring(0, indexOfDot);
- String foreignFieldName = fieldRefSpec.substring(indexOfDot + 1);
- TemporaryImportedFields importedFields = schema.temporaryImportedFields().get();
- if (importedFields.hasField(aliasFieldName)) {
- throw new IllegalArgumentException("For schema '" + schema.getName() + "', import field as '" + aliasFieldName + "': Field already imported");
- }
- importedFields.add(new TemporaryImportedField(aliasFieldName, documentReferenceFieldName, foreignFieldName));
- }
-}
-
-
-/**
- * This rule consumes an expression token and returns its image.
- *
- * @return The consumed token image.
- */
-String expression() :
-{
- String exp;
-}
-{
- ( <EXPRESSION_SL> { exp = token.image.substring(token.image.indexOf(":") + 1); } |
- <EXPRESSION_ML> { exp = token.image.substring(token.image.indexOf("{") + 1,
- token.image.lastIndexOf("}")); } )
- { return exp; }
-}
-
-String identifierWithDash() :
-{
- String identifier;
-}
-{
- ( identifier = identifier() { return identifier; } )
- |
- ( <IDENTIFIER_WITH_DASH> { return token.image; } )
-}
-
-/**
- * Consumes an identifier. This must be kept in sync with all word tokens that should be parseable as
- * identifiers.
- *
- * @return the identifier string
- */
-String identifier() : { }
-{
- ( <ALIAS>
- | <ALWAYS>
- | <ANNOTATION>
- | <ANNOTATIONREFERENCE>
- | <ARITY>
- | <ARRAY>
- | <AS>
- | <ASCENDING>
- | <ATTRIBUTE>
- | <BODY>
- | <BOLDING>
- | <BTREE>
- | <CASED>
- | <COMPRESSION>
- | <COMPRESSIONLEVEL>
- | <COMPRESSIONTHRESHOLD>
- | <CONSTANT>
- | <CONSTANTS>
- | <CONTEXT>
- | <CREATEIFNONEXISTENT>
- | <DENSEPOSTINGLISTTHRESHOLD>
- | <DESCENDING>
- | <DICTIONARY>
- | <DIRECT>
- | <DOCUMENT>
- | <DOCUMENTSUMMARY>
- | <DOUBLE_KEYWORD>
- | <DYNAMIC>
- | <ENABLEBITVECTORS>
- | <ENABLEONLYBITVECTOR>
- | <EXACT>
- | <EXACTTERMINATOR>
- | <FALSE>
- | <FASTACCESS>
- | <FASTSEARCH>
- | <FIELD>
- | <FIELDS>
- | <FIELDSET>
- | <FILE>
- | <FILTER>
- | <FIRSTPHASE>
- | <FULL>
- | <FUNCTION>
- | <GRAM>
- | <HASH>
- | <HEADER>
- | <HUGE>
- | <ID>
- | <IDENTICAL>
- | <IDENTIFIER>
- | <IGNOREDEFAULTRANKFEATURES>
- | <IMPORT>
- | <INDEX>
- | <INDEXING>
- | <INDEXINGREWRITE>
- | <INHERITS>
- | <INLINE>
- | <INPUTS>
- | <INTEGER>
- | <KEEPRANKCOUNT>
- | <LITERAL>
- | <LOCALE>
- | <LONG>
- | <LOOSE>
- | <LOWERBOUND>
- | <LOWERCASE>
- | <MACRO>
- | <MAP>
- | <MATCH>
- | <MATCHPHASE>
- | <MAXFILTERCOVERAGE>
- | <MAXHITS>
- | <MTOKEN>
- | <MUTABLE>
- | <NEVER>
- | <NONE>
- | <NORMAL>
- | <NORMALIZING>
- | <OFF>
- | <ON>
- | <ONDEMAND>
- | <ORDER>
- | <PREFIX>
- | <PRIMARY>
- | <PROPERTIES>
- | <QUATERNARY>
- | <QUERY>
- | <QUERYCOMMAND>
- | <RANK>
- | <MODEL>
- | <RANKPROFILE>
- | <RANKPROPERTIES>
- | <RANKSCOREDROPLIMIT>
- | <RANKTYPE>
- | <RAW>
- | <REFERENCE>
- | <REMOVEIFZERO>
- | <RERANKCOUNT>
- | <SCHEMA>
- | <SEARCH>
- | <SECONDARY>
- | <SECONDPHASE>
- | <SORTING>
- | <SOURCE>
- | <PAGED>
- | <SSCONTEXTUAL>
- | <SSOVERRIDE>
- | <SSTITLE>
- | <SSURL>
- | <STATIC>
- | <STEMMING>
- | <STRENGTH>
- | <STRICT>
- | <STRING>
- | <STRUCT>
- | <SUBSTRING>
- | <SUFFIX>
- | <SUMMARY>
- | <SUMMARYTO>
- | <SYMMETRIC>
- | <TERTIARY>
- | <TEXT>
- | <TO>
- | <TRUE>
- | <TYPE>
- | <UCA>
- | <UNCASED>
- | <URI>
- | <UPPERBOUND>
- | <USEDOCUMENT>
- | <VARIABLE>
- | <WEIGHT>
- | <WEIGHTEDSET>
- | <WORD>
- )
- { return token.image; }
-}
-
-/**
- * Consumes a string token and returns the token image.
- *
- * @return The consumed token image.
- */
-String string() : { }
-{
- <STRING> { return token.image; }
-}
-
-/**
- * Consumes a quoted string token and returns the token image minus the quotes. This does not perform
- * unescaping of the content, it simply removes the first and last character of the image. However, the token itself can
- * contain anything but a double quote.
- *
- * @return The unquoted token image.
- */
-String quotedString() : { }
-{
- <QUOTEDSTRING> { return token.image.substring(1, token.image.length() - 1); }
-}
-
-/**
- * This rule consumes a boolean value.
- *
- * @return The consumed boolean value.
- */
-Boolean bool() : { }
-{
- ( ( <ON> | <TRUE> ) { return true; } |
- ( <OFF> | <FALSE> ) { return false; } )
-}
-
-/**
- * This rule consumes an integer token and returns its numeric value.
- *
- * @return The consumed integer value.
- */
-int integer() : { }
-{
- <INTEGER> { return Integer.parseInt(token.image); }
-}
-
-/**
- * This rule consumes a long or integer token and returns its numeric value.
- *
- * @return The consumed long value.
- */
-long consumeLong() : { }
-{
- ( <INTEGER> { return Long.parseLong(token.image); } |
- <LONG> { return Long.parseLong(token.image.substring(0, token.image.length()-1)); }
- )
-}
-
-/**
- * This rule consumes a floating-point token and returns its numeric value.
- *
- * @return The consumed value.
- */
-double consumeFloat() : { }
-{
- <DOUBLE> { return Double.valueOf(token.image); }
-}
-
-Number consumeNumber() :
-{
- Number num;
-}
-{
- (num = consumeFloat() | num = consumeLong()) { return num; }
-}
-
-/**
- * This rule consumes an opening brace with leading and trailing newline tokens.
- */
-void lbrace() : { }
-{
- (<NL>)* <LBRACE> (<NL>)*
-}