From c64a57addba3c07586279bbc0e81993bbafcb620 Mon Sep 17 00:00:00 2001
From: Henrik <henrik.hoiness@online.no>
Date: Tue, 17 Jul 2018 16:52:22 +0200
Subject: Now generates config-file programmatically. Added the predetermined
 sources from the genreated config.json to the autocompletion of yql-syntax.
 Now also possible to add possible values for all parameters, just put the
 parameter (full name with underscore instead of dots) and a list of possible
 values in the config-file to be generated. Model.sources,
 Ranking.rankprofile, Ranking.properties, ranking.features have now possible
 values on value-input.

---
 container-search-gui/pom.xml                       |  6 ++
 .../com/yahoo/search/query/gui/GUIHandler.java     | 87 +++++++++++++++++---
 .../src/main/resources/gui/_includes/index.html    | 94 +++++++++++++++-------
 .../plugins/autocompletion/autocompletion.js       |  5 +-
 .../gui/editarea/edit_area/reg_syntax/yql.js       | 31 ++++++-
 .../src/main/resources/gui/gui_variables.json      |  4 -
 6 files changed, 180 insertions(+), 47 deletions(-)
 delete mode 100644 container-search-gui/src/main/resources/gui/gui_variables.json

(limited to 'container-search-gui')

diff --git a/container-search-gui/pom.xml b/container-search-gui/pom.xml
index f598758e1de..a0cb520aa75 100644
--- a/container-search-gui/pom.xml
+++ b/container-search-gui/pom.xml
@@ -44,6 +44,12 @@
             <version>${project.version}</version>
             <scope>provided</scope>
         </dependency>
+        <dependency>
+            <groupId>com.yahoo.vespa</groupId>
+            <artifactId>container-search</artifactId>
+            <version>${project.version}</version>
+            <scope>provided</scope>
+        </dependency>
     </dependencies>
     <build>
         <plugins>
diff --git a/container-search-gui/src/main/java/com/yahoo/search/query/gui/GUIHandler.java b/container-search-gui/src/main/java/com/yahoo/search/query/gui/GUIHandler.java
index 95adbbea3fc..552991d4dca 100644
--- a/container-search-gui/src/main/java/com/yahoo/search/query/gui/GUIHandler.java
+++ b/container-search-gui/src/main/java/com/yahoo/search/query/gui/GUIHandler.java
@@ -2,18 +2,36 @@
 package com.yahoo.search.query.gui;
 
 import com.google.inject.Inject;
+
+import com.yahoo.container.QrSearchersConfig;
 import com.yahoo.container.jdisc.HttpRequest;
 import com.yahoo.container.jdisc.HttpResponse;
 import com.yahoo.container.jdisc.LoggingRequestHandler;
+import com.yahoo.prelude.IndexModel;
+import com.yahoo.prelude.fastsearch.CacheControl;
+import com.yahoo.prelude.querytransform.RecallSearcher;
+import com.yahoo.search.Query;
+import com.yahoo.search.config.IndexInfoConfig;
+import com.yahoo.search.query.Model;
+import com.yahoo.search.query.Presentation;
+import com.yahoo.search.query.Ranking;
+import com.yahoo.search.query.ranking.Diversity;
+import com.yahoo.search.query.ranking.MatchPhase;
 import com.yahoo.search.query.restapi.ErrorResponse;
 import com.yahoo.yolean.Exceptions;
+import org.json.JSONException;
+import org.json.JSONObject;
 
-
+import java.io.ByteArrayInputStream;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
+import java.util.*;
 import java.util.logging.Level;
 
+import com.yahoo.search.yql.MinimalQueryInserter;
+
+
 /**
  * Takes requests on /querybuilder
  *
@@ -21,10 +39,12 @@ import java.util.logging.Level;
  */
 
 public class GUIHandler extends LoggingRequestHandler {
+    private final IndexModel indexModel;
 
     @Inject
-    public GUIHandler(Context parentCtx) {
-        super(parentCtx);
+    public GUIHandler(Context parentContext, IndexInfoConfig indexInfo, QrSearchersConfig clusters) {
+        super(parentContext);
+        indexModel = new IndexModel(indexInfo, clusters);
     }
 
     @Override
@@ -45,40 +65,51 @@ public class GUIHandler extends LoggingRequestHandler {
     private HttpResponse handleGET(HttpRequest request) {
         com.yahoo.restapi.Path path = new com.yahoo.restapi.Path(request.getUri().getPath());
         if (path.matches("/querybuilder/")) {
-            return new FileResponse("_includes/index.html");
+            return new FileResponse("_includes/index.html", null);
         }
         if (!path.matches("/querybuilder/{*}") ) {
             return ErrorResponse.notFoundError("Nothing at path:" + path);
         }
         String filepath = path.getRest();
-        if (!isValidPath(filepath)){
+        if (!isValidPath(filepath) && !filepath.equals("config.json")){
             return ErrorResponse.notFoundError("Nothing at path:" + filepath);
         }
-        return new FileResponse(filepath);
+        return new FileResponse(filepath, indexModel);
     }
 
     private static boolean isValidPath(String path) {
         InputStream in = GUIHandler.class.getClassLoader().getResourceAsStream("gui/"+path);
         boolean isValid = (in != null);
         if(isValid){
-            try { in.close(); } catch (IOException e) {/* Problem with closing inputstream */}
+            try { in.close(); } catch (IOException e) {/* Problem with closing input stream */}
         }
 
         return isValid;
     }
 
+
     private static class FileResponse extends HttpResponse {
 
         private final String path;
+        private final IndexModel indexModel;
 
-        public FileResponse(String relativePath) {
+        public FileResponse(String relativePath, IndexModel indexModel) {
             super(200);
             this.path = relativePath;
+            this.indexModel = indexModel;
         }
 
+
         @Override
         public void render(OutputStream out) throws IOException {
-            InputStream is = GUIHandler.class.getClassLoader().getResourceAsStream("gui/"+this.path);
+            InputStream is;
+            if (this.path.equals("config.json")){
+                String json = "{}";
+                try { json = getGUIConfig(); } catch (JSONException e) { /*Something happened while parsing JSON */ }
+                is = new ByteArrayInputStream(json.getBytes());
+            } else{
+                is = GUIHandler.class.getClassLoader().getResourceAsStream("gui/"+this.path);
+            }
             byte[] buf = new byte[1024];
             int numRead;
             while ( (numRead = is.read(buf) ) >= 0) {
@@ -121,5 +152,43 @@ public class GUIHandler extends LoggingRequestHandler {
             }
             return "text/html";
         }
+
+        private String getGUIConfig() throws JSONException {
+            JSONObject json = new JSONObject();
+            json.put("ranking_properties", Arrays.asList("propertyname"));
+            json.put("ranking_features", Arrays.asList("featurename"));
+            json.put("ranking_profile", Arrays.asList("rankprofile1", "rankprofile2"));
+            List<String> sources = new ArrayList<>();
+            try{
+                sources = new ArrayList<>(indexModel.getMasterClusters().keySet());
+            } catch (NullPointerException ex){ /* clusters are not set */}
+            json.put("model_sources", sources);
+
+            // Creating map from parent to children for GUI: parameter --> child-parameters
+            HashMap<String, List<String>> childMap = new HashMap<>();
+            childMap.put(Model.MODEL, Arrays.asList(Model.DEFAULT_INDEX, Model.ENCODING, Model.LANGUAGE, Model.QUERY_STRING, Model.RESTRICT, Model.SEARCH_PATH, Model.SOURCES, Model.TYPE));
+            childMap.put(Ranking.RANKING, Arrays.asList(Ranking.LOCATION, Ranking.FEATURES, Ranking.LIST_FEATURES, Ranking.PROFILE, Ranking.PROPERTIES, Ranking.SORTING, Ranking.FRESHNESS, Ranking.QUERYCACHE, Ranking.MATCH_PHASE));
+            childMap.put(Ranking.RANKING +"."+ Ranking.MATCH_PHASE, Arrays.asList(MatchPhase.MAX_HITS, MatchPhase.ATTRIBUTE, MatchPhase.ASCENDING, Ranking.DIVERSITY));
+            childMap.put(Ranking.RANKING +"."+ Ranking.MATCH_PHASE +"."+Ranking.DIVERSITY, Arrays.asList(Diversity.ATTRIBUTE, Diversity.MINGROUPS));
+            childMap.put(Presentation.PRESENTATION, Arrays.asList(Presentation.BOLDING, Presentation.FORMAT, Presentation.SUMMARY, "template", Presentation.TIMING ));
+            childMap.put("trace", Arrays.asList("timestamps"));
+            childMap.put("tracelevel", Arrays.asList("rules"));
+            childMap.put("metrics", Arrays.asList("ignore"));
+            childMap.put("collapse", Arrays.asList("summary"));
+            childMap.put("pos", Arrays.asList("ll", "radius", "bb", "attribute"));
+            childMap.put("streaming", Arrays.asList("userid", "groupname", "selection", "priority", "maxbucketspervisitor"));
+            childMap.put("rules", Arrays.asList("off", "rulebase"));
+            json.put("childMap", childMap);
+
+            List<String> levelZeroParameters = Arrays.asList(MinimalQueryInserter.YQL.toString(), Query.HITS.toString(), Query.OFFSET.toString(),
+                                "queryProfile", Query.NO_CACHE.toString(), Query.GROUPING_SESSION_CACHE.toString(),
+                                Query.SEARCH_CHAIN.toString(), Query.TIMEOUT.toString(), "trace", "tracelevel",
+                                Query.TRACE_LEVEL.toString(), Model.MODEL, Ranking.RANKING, "collapse", "collapsesize","collapsefield",
+                                Presentation.PRESENTATION, "pos", "streaming", "rules", RecallSearcher.recallName.toString(), "user",
+                                CacheControl.nocachewrite.toString(), "metrics", "");
+            json.put("levelZeroParameters", levelZeroParameters);
+
+            return json.toString();
+        }
     }
 }
\ No newline at end of file
diff --git a/container-search-gui/src/main/resources/gui/_includes/index.html b/container-search-gui/src/main/resources/gui/_includes/index.html
index ca3302dd404..ae6558d8f53 100644
--- a/container-search-gui/src/main/resources/gui/_includes/index.html
+++ b/container-search-gui/src/main/resources/gui/_includes/index.html
@@ -84,7 +84,7 @@
                 	<option class="options" value="POST">POST</option>
                 	<option class="options" value="GET">GET</option>
               </select>
-              <input type="text" class="textbox" name="value" value="http://httpbin.org/post" id="url" size="30">
+              <input type="text" class="textbox" name="value" value="http://localhost:8080/search/" id="url" size="30">
               <button class="button" onclick="startSending();" id="send">Send</button>
 
               <br/>
@@ -184,48 +184,36 @@
 
 
 
-
-
       <SCRIPT language="javascript">
 
-        const CONFIG = $.getJSON('/querybuilder/gui_variables.json', function(data){window.CONFIG = data;});
+        const CONFIG = $.getJSON('/querybuilder/config.json', function(data){window.CONFIG = data;});
         method = "POST";
       	var number = 0;
       	var childno = {};
       	var json = JSON.parse("{}");
         var searchApiReference = null;
-
-      	var possible = ["yql", "hits", "offset", "queryProfile", "nocache", "groupingSessionCache", "searchChain", "timeout", "trace","tracelevel","traceLevel", "",
-      , "model", "ranking", "collapse","collapsesize","collapsesize","presentation", "pos", "streaming", "rules", "recall", "user", "nocachewrite", "metrics"];
-
+        var possible = null;
       	var usedProps = [];
       	var removedIndexes = [0];
-      	var childrenProps = {
-      		"model" : ["defaultIndex", "encoding", "language", "queryString", "restrict", "searchPath", "sources", "type"],
-      		"ranking" : ["location", "features", "listFeatures", "profile", "properties", "sorting", "freshness", "queryCache", "matchPhase"],
-      		"ranking.matchPhase" : ["maxHits", "attribute", "ascending", "diversity"],
-      		"ranking.matchPhase.diversity" : ["attribute", "minGroups"],
-      		"presentation" : ["bolding", "format", "summary", "template", "timing"],
-          "trace" : ["timestamps"],
-          "tracelevel" : ["rules"],
-          "metrics" : ["ignore"],
-      		"collapse":["summary"],
-      		"pos" : ["ll", "radius", "bb", "attribute"],
-      		"streaming" : ["userid", "groupname", "selection", "priority", "maxbucketspervisitor"],
-      		"rules" : ["off", "rulebase"]
-      	};
+        var childrenProps = null;
+
 
         window.onload = function() {
-          // Adding variables from configuration file
-          if (window.CONFIG.hasOwnProperty("featurename")){
-            childrenProps["ranking.features"] = window.CONFIG.featurename;
-          }
-          if (window.CONFIG.hasOwnProperty("propertyname")){
-            childrenProps["ranking.properties"] = window.CONFIG.propertyname;
-          }
+          setTimeout(function(){
+            possible = window.CONFIG.levelZeroParameters;
+            childrenProps = window.CONFIG.childMap;
+
+            if (window.CONFIG.hasOwnProperty("ranking_features")){
+              childrenProps["ranking.features"] = window.CONFIG.ranking_features;
+            }
+            if (window.CONFIG.hasOwnProperty("ranking_properties")){
+              childrenProps["ranking.properties"] = window.CONFIG.ranking_properties;
+            }
 
-          addNewRow();
-          getSearchApiReference();
+
+            addNewRow();
+            getSearchApiReference();
+          }, 250);
         };
 
         var stringType = ["yql", "queryProfile", "searchChain", "model.defaultIndex", "model.encoding", "model.language",
@@ -439,10 +427,17 @@
         newInput.classList.add("input")
       	var newDatalist = document.createElement("datalist");
       	newDatalist.id = "prop"+temp;
+
       	newInputVal = document.createElement("input");
       	newInputVal.type = "text";
       	newInputVal.id = "v"+temp;
         newInputVal.classList.add("propvalue");
+        newInputVal.setAttribute("list", "val"+temp);
+        var newDatalist2 = document.createElement("datalist");
+      	newDatalist2.id = "val"+temp;
+        //newDatalist2.style = "display: none;";
+
+
       	var newButton = document.createElement("button");
       	newButton.id = "b"+temp;
       	newButton.innerHTML = "-";
@@ -478,6 +473,7 @@
         div.appendChild(a);
 
       	div.appendChild(newInputVal);
+        div.appendChild(newDatalist2);
       	div.appendChild(newButton);
       	div.appendChild(br);
 
@@ -525,6 +521,23 @@
         return false;
       }
 
+
+      function checkConfigOptions(key, no){
+        var jsonID = key.replace(/\./g , "_");
+        var datalist = document.getElementById("val"+no);
+        datalist.innerHTML = "";
+        if (window.CONFIG.hasOwnProperty(jsonID)){
+          var optionlist = eval("window.CONFIG."+jsonID);
+          optionlist.forEach(function(item){
+            var option = document.createElement("option");
+            option.value = item;
+            datalist.appendChild(option);
+          });
+
+        }
+
+      }
+
       	function keySelected(no, value){
       		var key = document.getElementById("i"+no).value;
           showInformation(no, key);
@@ -536,6 +549,10 @@
               editAreaLoader.delete_instance(window.yqlID);
             }
       			var button = document.createElement("button");
+
+            //var datalist = document.getElementById("val"+no); //Hide value-datalist
+            //datalist.style = "display:none;";
+
       			button.id="propb"+no
       			button.innerHTML=" +   Add property";
       			button.onclick = function(){addChildProp(no);};
@@ -549,6 +566,8 @@
       				newInputVal.type = "text";
       				newInputVal.id = "v"+no;
               newInputVal.classList.add("propvalue");
+              newInputVal.setAttribute("list", "val"+no);
+
               var parent = button.parentNode;
               showType(newInputVal, no);
               if (key === "yql"){
@@ -586,6 +605,7 @@
                 newInputVal.type = "text";
                 newInputVal.id = "v"+no;
                 newInputVal.classList.add("propvalue");
+                newInputVal.setAttribute("list", "val"+no);
                 showType(newInputVal, no);
                 var parent = inputval.parentNode;
                 parent.replaceChild(newInputVal, inputval);
@@ -627,8 +647,14 @@
             //keyInput.style = "border-width: 1px; border-color: red;"
             var valueInput = document.getElementById("v"+no);
             valueInput.placeholder = "Possible invalid parameter";
+
+            //Removes possible options for for former parameter
+            var datalist = document.getElementById("val"+no);
+            datalist.innerHTML = "";
           }
           if (validKey(no, key)){
+            // Check if datalist should be visible and add options
+            checkConfigOptions(fullKey, no);
             var keyInput = document.getElementById("i"+no);
             //keyInput.style = "border-width: 0px;"
           }
@@ -712,6 +738,11 @@
       	newInputVal.type = "text";
       	newInputVal.id = "v"+temp;
         newInputVal.classList.add("propvalue");
+
+        newInputVal.setAttribute("list", "val"+temp);
+        var newDatalist2 = document.createElement("datalist");
+        newDatalist2.id = "val"+temp;
+
       	var newButton = document.createElement("button");
       	newButton.id = "b"+temp;
       	newButton.innerHTML = "-";
@@ -746,6 +777,7 @@
       	div.appendChild(newDatalist);
         div.append(a);
       	div.appendChild(newInputVal);
+        div.appendChild(newDatalist2);
       	div.appendChild(newButton);
       	div.appendChild(br);
       	parentNode.appendChild(div);
diff --git a/container-search-gui/src/main/resources/gui/editarea/edit_area/plugins/autocompletion/autocompletion.js b/container-search-gui/src/main/resources/gui/editarea/edit_area/plugins/autocompletion/autocompletion.js
index d6984f59899..5531a2a4f69 100644
--- a/container-search-gui/src/main/resources/gui/editarea/edit_area/plugins/autocompletion/autocompletion.js
+++ b/container-search-gui/src/main/resources/gui/editarea/edit_area/plugins/autocompletion/autocompletion.js
@@ -480,7 +480,10 @@ var EditArea_autocompletion= {
 				{
 					var line= "<li><a href=\"#\" class=\"entry\" onmousedown=\"EditArea_autocompletion._select('"+ results[i][1]['replace_with'].replace(new RegExp('"', "g"), "&quot;") +"');return false;\">"+ results[i][1]['comment'];
 					if(results[i][0]['prefix_name'].length>0)
-						line+='<span class="prefix">'+ results[i][0]['prefix_name'] +'</span>';
+						var value = results[i][0]['prefix_name'];
+						value = (value == undefined) ? "" : value;
+						if (value == 'SOURCES' || value == ','){value = 'Source'}
+						line+='<span class="prefix">'+ value +'</span>';
 					line+='</a></li>';
 					lines[lines.length]=line;
 				}
diff --git a/container-search-gui/src/main/resources/gui/editarea/edit_area/reg_syntax/yql.js b/container-search-gui/src/main/resources/gui/editarea/edit_area/reg_syntax/yql.js
index e4e9cd3b32d..0a7dae862f3 100755
--- a/container-search-gui/src/main/resources/gui/editarea/edit_area/reg_syntax/yql.js
+++ b/container-search-gui/src/main/resources/gui/editarea/edit_area/reg_syntax/yql.js
@@ -1,6 +1,19 @@
 /**
 * Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 */
+function getSources(bool){
+		var options = [];
+		if (window.CONFIG.hasOwnProperty("model_sources")){
+			var sources = window.CONFIG.model_sources;
+			for (i in sources){
+				option = [sources[i]];
+				options.push(option);
+			}
+			if (bool == true){options.push(["*"])}
+		}
+		return options;
+}
+
 editAreaLoader.load_syntax["yql"] = {
 	'DISPLAY_NAME' : 'YQL'
 	,'QUOTEMARKS' : {1: "'", 2: '"', 3: '`'}
@@ -74,9 +87,23 @@ editAreaLoader.load_syntax["yql"] = {
 						 ['DESC','DESC'],['ASC','ASC'],['ALL','ALL'], ['GROUP','GROUP'],
 						 ['RANGE','RANGE'],['EACH','EACH'],['OUTPUT','OUTPUT'],
 						 ['SUM','SUM'],['LIMIT','LIMIT'],['OFFSET','OFFSET'],
-						 ['TIMEOUT','TIMEOUT'],
-						]
+						 ['TIMEOUT','TIMEOUT']
+					 ]
+			}
+		},
+		"sources": {	// the name of this definition group. It's posisble to have different rules inside the same definition file
+			"REGEXP": { "before_word": "[^a-zA-Z0-9_]|^"	// \\s|\\.|
+						,"possible_words_letters": "[a-zA-Z0-9_]+"
+						,"letter_after_word_must_match": "[^a-zA-Z0-9_]|$"
+						,"prefix_separator": " "
+					}
+			,"CASE_SENSITIVE": false
+			,"MAX_TEXT_LENGTH": 50		// the maximum length of the text being analyzed before the cursor position
+			,"KEYWORDS": {
+					'SOURCES' : getSources(true) ,
+					',' : getSources(false)
 			}
 		}
+
 	}
 };
diff --git a/container-search-gui/src/main/resources/gui/gui_variables.json b/container-search-gui/src/main/resources/gui/gui_variables.json
deleted file mode 100644
index 1b5f411611a..00000000000
--- a/container-search-gui/src/main/resources/gui/gui_variables.json
+++ /dev/null
@@ -1,4 +0,0 @@
-{
-  "propertyname" : ["propertyname"],
-  "featurename" : ["featurename"]
-}
-- 
cgit v1.2.3