diff options
116 files changed, 3742 insertions, 527 deletions
diff --git a/CMakeLists.txt b/CMakeLists.txt index 6c557179750..3961fd2a4ef 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -20,6 +20,8 @@ include(build_settings.cmake) # Enable CTest unit testing enable_testing() +vespa_install_data(valgrind-suppressions.txt etc/vespa) + # Include vespa config definitions in every target include_directories(BEFORE ${CMAKE_BINARY_DIR}/configdefinitions/src) diff --git a/config-model/src/main/python/ES_Vespa_parser.py b/config-model/src/main/python/ES_Vespa_parser.py new file mode 100644 index 00000000000..477b0db4744 --- /dev/null +++ b/config-model/src/main/python/ES_Vespa_parser.py @@ -0,0 +1,222 @@ +# Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +import json +import argparse +import os, sys + +# Parsing Elastic Search documents to Vespa documents +# Example of usage: python ES_Vespa_parser.py my_index.json my_index_mapping.json +__author__ = 'henrhoi' + + +class ElasticSearchParser: + document_file = None + mapping_file = None + application_name = None + search_definitions = {} + path = "" + _all = True + all_mappings = {} + no_index = [] + types = [] + + def __init__(self): + parser = argparse.ArgumentParser() + parser.add_argument("documents_path", help="location of file with documents to be parsed", type=str) + parser.add_argument("mappings_path", help="location of file with mappings", type=str) + parser.add_argument("--application_name", help="name of application", default="application_name", type=str) + args = parser.parse_args() + + self.document_file = args.documents_path + self.mapping_file = args.mappings_path + self.application_name = args.application_name + + def main(self): + self.path = os.getcwd() + "/application/" + try: + os.mkdir(self.path, 0o777) + print(" > Created folder '" + self.path + "'") + except OSError: + print(" > Folder '" + self.path + "' already existed") + + try: + os.makedirs(self.path + "searchdefinitions/", 0o777) + print(" > Created folder '" + self.path + "searchdefinitions/" + "'") + except OSError: + print(" > Folder '" + self.path + "searchdefinitions/" + "' already existed") + + self.parse() + self.createServices_xml() + self.createHosts_xml() + + def getMapping(self, type): + unparsed_mapping_file = open(self.mapping_file, "r") + type_mapping = {} + for line in unparsed_mapping_file: + data = json.loads(line) + index = list(data.keys())[0] + mappings = data[index]["mappings"][type]["properties"] + + # Checking if some fields could be no-index + try: + _all_enabled = data[index]["mappings"][type]["_all"]["enabled"] + if not _all_enabled: + self._all = False + print(" > All fields in the document type '" + type + "' is not searchable. Go inside " + self.path + type + ".sd to add which fields that should be searchable") + except KeyError: + print(" > All fields in the document type '" + type + "' is searchable") + + self.walk(mappings, type_mapping, "properties") + + unparsed_mapping_file.close() + if type not in self.search_definitions: + self.search_definitions[type] = True + self.types.append(type) + self.createSearchDefinition(type, type_mapping) + + # Adding mapping to global map with mappings + self.all_mappings[type] = type_mapping + return type_mapping + + def parse(self): + file_path = self.path + "documents" + ".json" + unparsed_document_file = open(self.document_file, "r") + vespa_docs = open(file_path, "w") + + for line in unparsed_document_file: + data = json.loads(line) + type = data["_type"] + + parsed_data = { + "put": "id:"+self.application_name+":" + type + "::" + data["_id"], + "fields": {} + } + + # Checking for already existing mapping for a type, if not create a new + if type in self.all_mappings: + mapping = self.all_mappings[type] + else: + mapping = self.getMapping(type) + + for key, item in mapping.items(): + try: + parsed_data["fields"][key] = data["_source"][key] + except KeyError: + continue + + json.dump(parsed_data, vespa_docs) + vespa_docs.write("\n") + + vespa_docs.close() + unparsed_document_file.close() + print(" > Parsed all documents '" + ", ".join(self.types) + "'" + "' at '" + file_path + "'") + + def createSearchDefinition(self, type, type_mapping): + file_path = self.path + "searchdefinitions/" + type + ".sd" + new_sd = open(file_path, "w") + new_sd.write("search " + type + " {\n") + new_sd.write(" document " + type + " {\n") + + for key, item in type_mapping.items(): + new_sd.write(" field " + key + " type " + self.get_type(item) + " {\n") + new_sd.write(" indexing: " + self.get_indexing(key, self.get_type(item)) + "\n") + new_sd.write(" }\n") + + new_sd.write(" }\n") + new_sd.write("}\n") + new_sd.close() + print(" > Created search definition for '" + type + "' at '" + file_path + "'") + + def createServices_xml(self): + file_path = self.path + "services.xml" + new_services = open(file_path, "w") + template = ("<?xml version='1.0' encoding='UTF-8'?>" + "<services version='1.0'>\n\n" + " <container id='default' version='1.0'>\n" + " <search/>\n" + " <document-api/>\n" + " <nodes>\n" + " <node hostalias='node1'/>\n" + " </nodes>\n" + " </container>\n\n" + " <content id='content' version='1.0'>\n" + " <redundancy>1</redundancy>\n" + " <search>\n" + " <visibility-delay>1.0</visibility-delay>\n" + " </search>\n" + " <documents>\n") + + for i in range(0, len(self.types)): + template += " <document mode='index' type='" + self.types[i] + "'/>\n" + + template += (" </documents>\n" + " <nodes>\n" + " <node hostalias='node1' distribution-key=\"0\"/>\n" + " </nodes>\n" + " <engine>\n" + " <proton>\n" + " <searchable-copies>1</searchable-copies>\n" + " </proton>\n" + " </engine>\n" + " </content>\n\n" + "</services>") + + new_services.write(template) + new_services.close() + print(" > Created services.xml at '" + file_path + "'") + + def createHosts_xml(self): + file_path = self.path + "hosts.xml" + new_hosts = open(file_path, "w") + template = ("<?xml version=\"1.0\" encoding=\"utf-8\" ?>\n" + "<hosts>\n" + " <host name=\"localhost\">\n" + " <alias>node1</alias>\n" + " </host>\n" + "</hosts>") + + new_hosts.write(template) + new_hosts.close() + print(" > Created hosts.xml at '" + file_path + "'") + + def get_type(self, type): + return { + "text": "string", + "keyword": "string", + "date": "string", + "long": "long", + "double": "double", + "boolean": "string", + "ip": "text", + "byte": "byte", + "float": "float", + + }[type] + + def get_indexing(self, key, key_type): + if not self._all: + return "summary" + + if key not in self.no_index: + if key_type == "string": + return "summary | index" + else: + return "summary | attribute" + + return "summary" + + def walk(self, node, mapping, parent): + for key, item in node.items(): + if isinstance(item, dict): + self.walk(item, mapping, key) + elif key == "type": + mapping[parent] = item + elif key == "include_in_all": + if not item: # Field should not be searchable + self.no_index.append(parent) + elif key == "index" and parent != "properties": + if item == "no": # Field should not be searchable + self.no_index.append(parent) + + +if __name__ == '__main__': + ElasticSearchParser().main() diff --git a/container-search/src/main/java/com/yahoo/search/Query.java b/container-search/src/main/java/com/yahoo/search/Query.java index cbc8b15eff9..cfb1c9a26be 100644 --- a/container-search/src/main/java/com/yahoo/search/Query.java +++ b/container-search/src/main/java/com/yahoo/search/Query.java @@ -20,6 +20,7 @@ import com.yahoo.search.query.Presentation; import com.yahoo.search.query.Properties; import com.yahoo.search.query.QueryTree; import com.yahoo.search.query.Ranking; +import com.yahoo.search.query.Select; import com.yahoo.search.query.SessionId; import com.yahoo.search.query.Sorting; import com.yahoo.search.query.Sorting.AttributeSorter; @@ -100,7 +101,8 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { ADVANCED(3,"adv"), WEB(4,"web"), PROGRAMMATIC(5, "prog"), - YQL(6, "yql"); + YQL(6, "yql"), + SELECT(7, "select");; private final int intValue; private final String stringValue; @@ -170,6 +172,9 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { /** How results of this query should be presented */ private Presentation presentation = new Presentation(this); + /** The selection of where-clause and grouping */ + private Select select = new Select(this); + //---------------- Tracing ---------------------------------------------------- private static Logger log = Logger.getLogger(Query.class.getName()); @@ -188,6 +193,7 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { public static final CompoundName GROUPING_SESSION_CACHE = new CompoundName("groupingSessionCache"); public static final CompoundName TIMEOUT = new CompoundName("timeout"); + private static QueryProfileType argumentType; static { argumentType = new QueryProfileType("native"); @@ -206,10 +212,12 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { argumentType.addField(new FieldDescription(Presentation.PRESENTATION, new QueryProfileFieldType(Presentation.getArgumentType()))); argumentType.addField(new FieldDescription(Ranking.RANKING, new QueryProfileFieldType(Ranking.getArgumentType()))); argumentType.addField(new FieldDescription(Model.MODEL, new QueryProfileFieldType(Model.getArgumentType()))); + argumentType.addField(new FieldDescription(Select.SELECT, new QueryProfileFieldType(Select.getArgumentType()))); argumentType.freeze(); } public static QueryProfileType getArgumentType() { return argumentType; } + /** The aliases of query properties */ private static Map<String,CompoundName> propertyAliases; static { @@ -218,6 +226,7 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { addAliases(Ranking.getArgumentType(), propertyAliasesBuilder); addAliases(Model.getArgumentType(), propertyAliasesBuilder); addAliases(Presentation.getArgumentType(), propertyAliasesBuilder); + addAliases(Select.getArgumentType(), propertyAliasesBuilder); propertyAliases = ImmutableMap.copyOf(propertyAliasesBuilder); } private static void addAliases(QueryProfileType arguments, Map<String, CompoundName> aliases) { @@ -237,6 +246,7 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { registry.register(Query.getArgumentType().unfrozen()); registry.register(Ranking.getArgumentType().unfrozen()); registry.register(Model.getArgumentType().unfrozen()); + registry.register(Select.getArgumentType().unfrozen()); registry.register(Presentation.getArgumentType().unfrozen()); registry.register(DefaultProperties.argumentType.unfrozen()); } @@ -381,6 +391,7 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { setFrom(properties,Model.getArgumentType(), context); setFrom(properties,Presentation.getArgumentType(), context); setFrom(properties,Ranking.getArgumentType(), context); + setFrom(properties, Select.getArgumentType(), context); } /** @@ -980,6 +991,9 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { /** Returns the presentation to be used for this query, never null */ public Presentation getPresentation() { return presentation; } + /** Returns the select to be used for this query, never null */ + public Select getSelect() { return select; } + /** Returns the ranking to be used for this query, never null */ public Ranking getRanking() { return ranking; } diff --git a/container-search/src/main/java/com/yahoo/search/federation/vespa/VespaSearcher.java b/container-search/src/main/java/com/yahoo/search/federation/vespa/VespaSearcher.java index b59578ab6a3..fdbee9c8f11 100644 --- a/container-search/src/main/java/com/yahoo/search/federation/vespa/VespaSearcher.java +++ b/container-search/src/main/java/com/yahoo/search/federation/vespa/VespaSearcher.java @@ -140,6 +140,8 @@ public class VespaSearcher extends ConfiguredHTTPProviderSearcher { return Query.Type.PROGRAMMATIC; } else if (providerQueryType == ProviderConfig.QueryType.YQL) { return Query.Type.YQL; + } else if (providerQueryType == ProviderConfig.QueryType.SELECT) { + return Query.Type.SELECT; } else { throw new RuntimeException("Query type " + providerQueryType + " unsupported."); } diff --git a/container-search/src/main/java/com/yahoo/search/handler/SearchHandler.java b/container-search/src/main/java/com/yahoo/search/handler/SearchHandler.java index 3bfaee658f9..5ac1f834031 100644 --- a/container-search/src/main/java/com/yahoo/search/handler/SearchHandler.java +++ b/container-search/src/main/java/com/yahoo/search/handler/SearchHandler.java @@ -283,13 +283,16 @@ public class SearchHandler extends LoggingRequestHandler { private HttpSearchResponse handleBody(HttpRequest request){ - // Find query profile - String queryProfileName = request.getProperty("queryProfile"); + + Map<String, String> requestMap = requestMapFromRequest(request); + + // Get query profile + String queryProfileName = requestMap.getOrDefault("queryProfile", null); CompiledQueryProfile queryProfile = queryProfileRegistry.findQueryProfile(queryProfileName); - boolean benchmarkOutput = VespaHeaders.benchmarkOutput(request); - Query query = queryFromRequest(request, queryProfile); + Query query = new Query(request, requestMap, queryProfile); + boolean benchmarkOutput = VespaHeaders.benchmarkOutput(request); boolean benchmarkCoverage = VespaHeaders.benchmarkCoverage(benchmarkOutput, request.getJDiscRequest().headers()); // Find and execute search chain if we have a valid query @@ -558,7 +561,8 @@ public class SearchHandler extends LoggingRequestHandler { return searchChainRegistry; } - private Query queryFromRequest(HttpRequest request, CompiledQueryProfile queryProfile){ + private Map<String, String> requestMapFromRequest(HttpRequest request) { + if (request.getMethod() == com.yahoo.jdisc.http.HttpRequest.Method.POST && JSON_CONTENT_TYPE.equals(request.getHeader(com.yahoo.jdisc.http.HttpHeaders.Names.CONTENT_TYPE))) { Inspector inspector; @@ -576,11 +580,21 @@ public class SearchHandler extends LoggingRequestHandler { // Create request-mapping Map<String, String> requestMap = new HashMap<>(); createRequestMapping(inspector, requestMap, ""); - return new Query(request, requestMap, queryProfile); + // Throws QueryException if query contains both yql- and select-parameter + if (requestMap.containsKey("yql") && (requestMap.containsKey("select.where") || requestMap.containsKey("select.grouping")) ) { + throw new QueryException("Illegal query: Query contains both yql- and select-parameter"); + } + + // Throws QueryException if query contains both query- and select-parameter + if (requestMap.containsKey("query") && (requestMap.containsKey("select.where") || requestMap.containsKey("select.grouping")) ) { + throw new QueryException("Illegal query: Query contains both query- and select-parameter"); + } + + return requestMap; } else { - return new Query(request, queryProfile); + return request.propertyMap(); } } @@ -605,6 +619,10 @@ public class SearchHandler extends LoggingRequestHandler { map.put(qualifiedKey, value.asString()); break; case OBJECT: + if (qualifiedKey.equals("select.where") || qualifiedKey.equals("select.grouping")){ + map.put(qualifiedKey, value.toString()); + break; + } createRequestMapping(value, map, qualifiedKey+"."); break; } diff --git a/container-search/src/main/java/com/yahoo/search/query/Model.java b/container-search/src/main/java/com/yahoo/search/query/Model.java index 167bb312f61..ddd33cb7c78 100644 --- a/container-search/src/main/java/com/yahoo/search/query/Model.java +++ b/container-search/src/main/java/com/yahoo/search/query/Model.java @@ -249,7 +249,7 @@ public class Model implements Cloneable { public String getQueryString() { return queryString; } /** - * Returns the query as an object structure. + * Returns the query as an object structure. Remember to have the correct Query.Type set. * This causes parsing of the query string if it has changed since this was last called * (i.e query parsing is lazy) */ diff --git a/container-search/src/main/java/com/yahoo/search/query/Select.java b/container-search/src/main/java/com/yahoo/search/query/Select.java new file mode 100644 index 00000000000..3ffc6bddb24 --- /dev/null +++ b/container-search/src/main/java/com/yahoo/search/query/Select.java @@ -0,0 +1,110 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.query; + +import com.yahoo.processing.request.CompoundName; +import com.yahoo.search.Query; +import com.yahoo.search.grouping.GroupingRequest; +import com.yahoo.search.query.parser.ParserEnvironment; +import com.yahoo.search.query.parser.ParserFactory; +import com.yahoo.search.query.profile.types.FieldDescription; +import com.yahoo.search.query.profile.types.QueryProfileType; +import com.yahoo.search.yql.VespaGroupingStep; + + + +/** + * The parameters defining the where-clause and groping of a query + * + * @author henrhoi + */ +public class Select implements Cloneable { + + /** The type representing the property arguments consumed by this */ + private static final QueryProfileType argumentType; + private static final CompoundName argumentTypeName; + + public static final String SELECT = "select"; + public static final String WHERE = "where"; + public static final String GROUPING = "grouping"; + + + private static Model model; + private Query parent; + private String where = ""; + private String grouping = ""; + + static { + argumentType = new QueryProfileType(SELECT); + argumentType.setStrict(true); + argumentType.setBuiltin(true); + argumentType.addField(new FieldDescription(WHERE, "string", "where")); + argumentType.addField(new FieldDescription(GROUPING, "string", "grouping")); + argumentType.freeze(); + argumentTypeName=new CompoundName(argumentType.getId().getName()); + } + + public static QueryProfileType getArgumentType() { return argumentType; } + + public Select(String where, String grouping){ + this.where = where; + this.grouping = grouping; + } + + public Select(Query query) { + setParent(query); + model = query.getModel(); + } + + + /** Returns the query owning this, never null */ + private Query getParent() { return parent; } + + + /** Assigns the query owning this */ + public void setParent(Query parent) { + if (parent==null) throw new NullPointerException("A query models owner cannot be null"); + this.parent = parent; + } + + + /** Set the where-clause for the query. Must be a JSON-string, with the format described in the Select Reference doc - https://docs.vespa.ai/documentation/reference/select-reference.html. */ + public void setWhere(String where) { + this.where = where; + model.setType(SELECT); + + // Setting the queryTree to null + model.setQueryString(null); + } + + + /** Returns the where-clause in the query */ + public String getWhereString(){ + return this.where; + } + + + /** Set the grouping-string for the query. Must be a JSON-string, with the format described in the Select Reference doc - https://docs.vespa.ai/documentation/reference/select-reference.html. */ + public void setGrouping(String grouping){ + this.grouping = grouping; + SelectParser parser = (SelectParser) ParserFactory.newInstance(Query.Type.SELECT, new ParserEnvironment()); + + for (VespaGroupingStep step : parser.getGroupingSteps(grouping)) { + GroupingRequest.newInstance(parent) + .setRootOperation(step.getOperation()) + .continuations().addAll(step.continuations()); + } + } + + + /** Returns the grouping in the query */ + public String getGroupingString(){ + return this.grouping; + } + + + @Override + public String toString() { + return "where: [" + where + "], grouping: [" + grouping+ "]"; + } + +} diff --git a/container-search/src/main/java/com/yahoo/search/query/SelectParser.java b/container-search/src/main/java/com/yahoo/search/query/SelectParser.java new file mode 100644 index 00000000000..13ebacb62ef --- /dev/null +++ b/container-search/src/main/java/com/yahoo/search/query/SelectParser.java @@ -0,0 +1,1185 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.query; + + +import com.google.common.base.Preconditions; +import com.yahoo.collections.LazyMap; +import com.yahoo.language.Language; +import com.yahoo.language.process.Normalizer; +import com.yahoo.prelude.IndexFacts; +import com.yahoo.prelude.query.AndItem; +import com.yahoo.prelude.query.CompositeItem; +import com.yahoo.prelude.query.DotProductItem; +import com.yahoo.prelude.query.EquivItem; +import com.yahoo.prelude.query.ExactStringItem; +import com.yahoo.prelude.query.IntItem; +import com.yahoo.prelude.query.Item; +import com.yahoo.prelude.query.Limit; +import com.yahoo.prelude.query.NearItem; +import com.yahoo.prelude.query.NotItem; +import com.yahoo.prelude.query.ONearItem; +import com.yahoo.prelude.query.OrItem; +import com.yahoo.prelude.query.PhraseItem; +import com.yahoo.prelude.query.PredicateQueryItem; +import com.yahoo.prelude.query.PrefixItem; +import com.yahoo.prelude.query.QueryException; +import com.yahoo.prelude.query.RangeItem; +import com.yahoo.prelude.query.RankItem; +import com.yahoo.prelude.query.RegExpItem; +import com.yahoo.prelude.query.SameElementItem; +import com.yahoo.prelude.query.SegmentingRule; +import com.yahoo.prelude.query.Substring; +import com.yahoo.prelude.query.SubstringItem; +import com.yahoo.prelude.query.SuffixItem; +import com.yahoo.prelude.query.TaggableItem; +import com.yahoo.prelude.query.WandItem; +import com.yahoo.prelude.query.WeakAndItem; +import com.yahoo.prelude.query.WeightedSetItem; +import com.yahoo.prelude.query.WordAlternativesItem; +import com.yahoo.prelude.query.WordItem; +import com.yahoo.search.grouping.request.GroupingOperation; +import com.yahoo.search.query.parser.Parsable; +import com.yahoo.search.query.parser.Parser; +import com.yahoo.search.query.parser.ParserEnvironment; +import com.yahoo.search.yql.VespaGroupingStep; +import com.yahoo.slime.ArrayTraverser; +import com.yahoo.slime.Inspector; +import com.yahoo.slime.ObjectTraverser; +import com.yahoo.vespa.config.SlimeUtils; +import edu.umd.cs.findbugs.annotations.NonNull; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +import static com.yahoo.slime.Type.ARRAY; +import static com.yahoo.slime.Type.DOUBLE; +import static com.yahoo.slime.Type.LONG; +import static com.yahoo.slime.Type.OBJECT; +import static com.yahoo.slime.Type.STRING; + +/** + * The Select query language. + * + * This class will be parsing the Select parameters, and will be used when the query has the SELECT-type. + * + * @author henrhoi + */ + + +public class SelectParser implements Parser { + + Parsable query; + private final IndexFacts indexFacts; + private final Map<Integer, TaggableItem> identifiedItems = LazyMap.newHashMap(); + private final List<ConnectedItem> connectedItems = new ArrayList<>(); + private final Normalizer normalizer; + private final ParserEnvironment environment; + private IndexFacts.Session indexFactsSession; + + + + /** YQL parameters and functions */ + + private static final String DESCENDING_HITS_ORDER = "descending"; + private static final String ASCENDING_HITS_ORDER = "ascending"; + private static final Integer DEFAULT_TARGET_NUM_HITS = 10; + private static final String ORIGIN_LENGTH = "length"; + private static final String ORIGIN_OFFSET = "offset"; + private static final String ORIGIN = "origin"; + private static final String ORIGIN_ORIGINAL = "original"; + private static final String CONNECTION_ID = "id"; + private static final String CONNECTION_WEIGHT = "weight"; + private static final String CONNECTIVITY = "connectivity"; + private static final String ANNOTATIONS = "annotations"; + private static final String NFKC = "nfkc"; + private static final String USER_INPUT_LANGUAGE = "language"; + private static final String ACCENT_DROP = "accentDrop"; + private static final String ALTERNATIVES = "alternatives"; + private static final String AND_SEGMENTING = "andSegmenting"; + private static final String DISTANCE = "distance"; + private static final String DOT_PRODUCT = "dotProduct"; + private static final String EQUIV = "equiv"; + private static final String FILTER = "filter"; + private static final String HIT_LIMIT = "hitLimit"; + private static final String IMPLICIT_TRANSFORMS = "implicitTransforms"; + private static final String LABEL = "label"; + private static final String NEAR = "near"; + private static final String NORMALIZE_CASE = "normalizeCase"; + private static final String ONEAR = "onear"; + private static final String PHRASE = "phrase"; + private static final String PREDICATE = "predicate"; + private static final String PREFIX = "prefix"; + private static final String RANKED = "ranked"; + private static final String RANK = "rank"; + private static final String SAME_ELEMENT = "sameElement"; + private static final String SCORE_THRESHOLD = "scoreThreshold"; + private static final String SIGNIFICANCE = "significance"; + private static final String STEM = "stem"; + private static final String SUBSTRING = "substring"; + private static final String SUFFIX = "suffix"; + private static final String TARGET_NUM_HITS = "targetNumHits"; + private static final String THRESHOLD_BOOST_FACTOR = "thresholdBoostFactor"; + private static final String UNIQUE_ID = "id"; + private static final String USE_POSITION_DATA = "usePositionData"; + private static final String WAND = "wand"; + private static final String WEAK_AND = "weakAnd"; + private static final String WEIGHTED_SET = "weightedSet"; + private static final String WEIGHT = "weight"; + private static final String AND = "and"; + private static final String AND_NOT = "and_not"; + private static final String OR = "or"; + private static final String EQ = "equals"; + private static final String RANGE = "range"; + private static final String CONTAINS = "contains"; + private static final String MATCHES = "matches"; + private static final String CALL = "call"; + private static final List<String> FUNCTION_CALLS = Arrays.asList(WAND, WEIGHTED_SET, DOT_PRODUCT, PREDICATE, RANK, WEAK_AND); + + /**************************************/ + + + + public SelectParser(ParserEnvironment environment) { + indexFacts = environment.getIndexFacts(); + normalizer = environment.getLinguistics().getNormalizer(); + + this.environment = environment; + } + + + @Override + public QueryTree parse(Parsable query) { + indexFactsSession = indexFacts.newSession(query.getSources(), query.getRestrict()); + connectedItems.clear(); + identifiedItems.clear(); + this.query = query; + + return buildTree(); + } + + + + private QueryTree buildTree() { + Inspector inspector = SlimeUtils.jsonToSlime(this.query.getSelect().getWhereString().getBytes()).get(); + if (inspector.field("error_message").valid()){ + throw new QueryException("Illegal query: "+inspector.field("error_message").asString() + ", at: "+ new String(inspector.field("offending_input").asData(), StandardCharsets.UTF_8)); + } + + Item root = walkJson(inspector); + connectItems(); + QueryTree newTree = new QueryTree(root); + + return newTree; + } + + + private Item walkJson(Inspector inspector){ + final Item[] item = {null}; + inspector.traverse((ObjectTraverser) (key, value) -> { + String type = (FUNCTION_CALLS.contains(key)) ? CALL : key; + + switch (type) { + + case AND: + item[0] = buildAnd(key, value); + break; + case AND_NOT: + item[0] = buildNotAnd(key, value); + break; + case OR: + item[0] = buildOr(key, value); + break; + case EQ: + item[0] = buildEquals(key, value); + break; + case RANGE: + item[0] = buildRange(key, value); + break; + case CONTAINS: + item[0] = buildTermSearch(key, value); + break; + case MATCHES: + item[0] = buildRegExpSearch(key, value); + break; + case CALL: + item[0] = buildFunctionCall(key, value); + break; + default: + throw newUnexpectedArgumentException(key, AND, CALL, CONTAINS, EQ, OR, RANGE, AND_NOT); + } + }); + return item[0]; + } + + + public List<VespaGroupingStep> getGroupingSteps(String grouping){ + List<VespaGroupingStep> groupingSteps = new ArrayList<>(); + List<String> groupingOperations = getOperations(grouping); + for (String groupingString : groupingOperations){ + GroupingOperation groupingOperation = GroupingOperation.fromString(groupingString); + VespaGroupingStep groupingStep = new VespaGroupingStep(groupingOperation); + groupingSteps.add(groupingStep); + } + return groupingSteps; + } + + private List<String> getOperations(String grouping) { + List<String> operations = new ArrayList<>(); + Inspector inspector = SlimeUtils.jsonToSlime(grouping.getBytes()).get(); + if (inspector.field("error_message").valid()){ + throw new QueryException("Illegal query: "+inspector.field("error_message").asString() + ", at: "+ new String(inspector.field("offending_input").asData(), StandardCharsets.UTF_8)); + } + + inspector.traverse( (ArrayTraverser) (key, value) -> { + String groupingString = value.toString(); + groupingString = groupingString.replace(" ", "").replace("\"", "").replace("\'", "").replace(":{", "(").replace(":", "(").replace("}", ")").replace(",", ")"); + groupingString = groupingString.substring(1, groupingString.length()); + operations.add(groupingString); + }); + + return operations; + + } + + + @NonNull + private Item buildFunctionCall(String key, Inspector value) { + switch (key) { + case WAND: + return buildWand(key, value); + case WEIGHTED_SET: + return buildWeightedSet(key, value); + case DOT_PRODUCT: + return buildDotProduct(key, value); + case PREDICATE: + return buildPredicate(key, value); + case RANK: + return buildRank(key, value); + case WEAK_AND: + return buildWeakAnd(key, value); + default: + throw newUnexpectedArgumentException(key, DOT_PRODUCT, RANK, WAND, WEAK_AND, WEIGHTED_SET, PREDICATE); + } + } + + + private void addItemsFromInspector(CompositeItem item, Inspector inspector){ + if (inspector.type() == ARRAY){ + inspector.traverse((ArrayTraverser) (index, new_value) -> { + item.addItem(walkJson(new_value)); + }); + + } else if (inspector.type() == OBJECT){ + if (inspector.field("children").valid()){ + inspector.field("children").traverse((ArrayTraverser) (index, new_value) -> { + item.addItem(walkJson(new_value)); + }); + } + + } + } + + + private Inspector getChildren(Inspector inspector){ + if (inspector.type() == ARRAY){ + return inspector; + + } else if (inspector.type() == OBJECT){ + if (inspector.field("children").valid()){ + return inspector.field("children"); + } + if (inspector.field(1).valid()){ + return inspector.field(1); + } + } + return null; + } + + + private HashMap<Integer, Inspector> getChildrenMap(Inspector inspector){ + HashMap<Integer, Inspector> children = new HashMap<>(); + if (inspector.type() == ARRAY){ + inspector.traverse((ArrayTraverser) (index, new_value) -> { + children.put(index, new_value); + }); + + } else if (inspector.type() == OBJECT){ + if (inspector.field("children").valid()){ + inspector.field("children").traverse((ArrayTraverser) (index, new_value) -> { + children.put(index, new_value); + }); + } + } + return children; + } + + + private Inspector getAnnotations(Inspector inspector){ + if (inspector.type() == OBJECT && inspector.field("attributes").valid()){ + return inspector.field("attributes"); + } + return null; + } + + + private HashMap<String, Inspector> getAnnotationMapFromAnnotationInspector(Inspector annotation){ + HashMap<String, Inspector> attributes = new HashMap<>(); + if (annotation.type() == OBJECT){ + annotation.traverse((ObjectTraverser) (index, new_value) -> { + attributes.put(index, new_value); + }); + } + return attributes; + } + + + private HashMap<String, Inspector> getAnnotationMap(Inspector inspector){ + HashMap<String, Inspector> attributes = new HashMap<>(); + if (inspector.type() == OBJECT && inspector.field("attributes").valid()){ + inspector.field("attributes").traverse((ObjectTraverser) (index, new_value) -> { + attributes.put(index, new_value); + }); + } + return attributes; + } + + + private <T> T getAnnotation(String annotationName, HashMap<String, Inspector> annotations, Class<T> expectedClass, T defaultValue) { + return (annotations.get(annotationName) == null) ? defaultValue : expectedClass.cast(annotations.get(annotationName).asString()); + } + + + private Boolean getBoolAnnotation(String annotationName, HashMap<String, Inspector> annotations, Boolean defaultValue) { + if (annotations != null){ + Inspector annotation = annotations.getOrDefault(annotationName, null); + if (annotation != null){ + return annotation.asBool(); + } + } + return defaultValue; + } + + + private Integer getIntegerAnnotation(String annotationName, HashMap<String, Inspector> annotations, Integer defaultValue) { + if (annotations != null){ + Inspector annotation = annotations.getOrDefault(annotationName, null); + if (annotation != null){ + return (int)annotation.asLong(); + } + } + return defaultValue; + } + + + private Double getDoubleAnnotation(String annotationName, HashMap<String, Inspector> annotations, Double defaultValue) { + if (annotations != null){ + Inspector annotation = annotations.getOrDefault(annotationName, null); + if (annotation != null){ + return annotation.asDouble(); + } + } + return defaultValue; + } + + + private Inspector getAnnotationAsInspectorOrNull(String annotationName, HashMap<String, Inspector> annotations) { + return annotations.get(annotationName); + } + + + @NonNull + private CompositeItem buildAnd(String key, Inspector value) { + AndItem andItem = new AndItem(); + addItemsFromInspector(andItem, value); + + return andItem; + } + + + @NonNull + private CompositeItem buildNotAnd(String key, Inspector value) { + NotItem notItem = new NotItem(); + addItemsFromInspector(notItem, value); + + return notItem; + } + + + @NonNull + private CompositeItem buildOr(String key, Inspector value) { + OrItem orItem = new OrItem(); + addItemsFromInspector(orItem, value); + return orItem; + } + + + @NonNull + private CompositeItem buildWeakAnd(String key, Inspector value) { + WeakAndItem weakAnd = new WeakAndItem(); + addItemsFromInspector(weakAnd, value); + Inspector annotations = getAnnotations(value); + + if (annotations != null){ + annotations.traverse((ObjectTraverser) (annotation_name, annotation_value) -> { + if (TARGET_NUM_HITS.equals(annotation_name)){ + weakAnd.setN((int)(annotation_value.asDouble())); + } + if (SCORE_THRESHOLD.equals(annotation_name)){ + weakAnd.setScoreThreshold((int)(annotation_value.asDouble())); + } + }); + } + + return weakAnd; + } + + + @NonNull + private <T extends TaggableItem> T leafStyleSettings(Inspector annotations, @NonNull T out) { + { + if (annotations != null) { + Inspector itemConnectivity= getAnnotationAsInspectorOrNull(CONNECTIVITY, getAnnotationMapFromAnnotationInspector(annotations)); + if (itemConnectivity != null) { + Integer[] id = {null}; + Double[] weight = {null}; + itemConnectivity.traverse((ObjectTraverser) (key, value) -> { + switch (key){ + case CONNECTION_ID: + id[0] = (int) value.asLong(); + break; + case CONNECTION_WEIGHT: + weight[0] = value.asDouble(); + break; + } + }); + connectedItems.add(new ConnectedItem(out, id[0], weight[0])); + } + + annotations.traverse((ObjectTraverser) (annotation_name, annotation_value) -> { + + if (SIGNIFICANCE.equals(annotation_name)) { + if (annotation_value != null) { + out.setSignificance(annotation_value.asDouble()); + } + } + if (UNIQUE_ID.equals(annotation_name)) { + if (annotation_value != null) { + out.setUniqueID((int)annotation_value.asLong()); + identifiedItems.put((int)annotation_value.asLong(), out); + } + } + }); + } + } + { + Item leaf = (Item) out; + if (annotations != null) { + Inspector itemAnnotations = getAnnotationAsInspectorOrNull(ANNOTATIONS, getAnnotationMapFromAnnotationInspector(annotations)); + if (itemAnnotations != null) { + itemAnnotations.traverse((ObjectTraverser) (key, value) -> { + leaf.addAnnotation(key, value.asString()); + }); + } + + annotations.traverse((ObjectTraverser) (annotation_name, annotation_value) -> { + if (FILTER.equals(annotation_name)) { + if (annotation_value != null) { + leaf.setFilter(annotation_value.asBool()); + } + } + if (RANKED.equals(annotation_name)) { + if (annotation_value != null) { + leaf.setRanked(annotation_value.asBool()); + } + } + if (LABEL.equals(annotation_name)) { + if (annotation_value != null) { + leaf.setLabel(annotation_value.asString()); + } + } + if (WEIGHT.equals(annotation_name)) { + if (annotation_value != null) { + leaf.setWeight((int)annotation_value.asDouble()); + } + } + }); + } + if (out instanceof IntItem && annotations != null) { + IntItem number = (IntItem) out; + Integer hitLimit = getCappedRangeSearchParameter(annotations); + if (hitLimit != null) { + number.setHitLimit(hitLimit); + } + + } + } + + return out; + } + + + private Integer getCappedRangeSearchParameter(Inspector annotations) { + final Integer[] hitLimit = {null}; + annotations.traverse((ObjectTraverser) (annotation_name, annotation_value) -> { + if (HIT_LIMIT.equals(annotation_name)) { + if (annotation_value != null) { + hitLimit[0] = (int)(annotation_value.asDouble()); + } + } + }); + final Boolean[] ascending = {null}; + final Boolean[] descending = {null}; + + if (hitLimit[0] != null) { + annotations.traverse((ObjectTraverser) (annotation_name, annotation_value) -> { + if (ASCENDING_HITS_ORDER.equals(annotation_name)) { + ascending[0] = annotation_value.asBool(); + } + if (DESCENDING_HITS_ORDER.equals(annotation_name)) { + descending[0] = annotation_value.asBool(); + } + + }); + Preconditions.checkArgument(ascending[0] == null || descending[0] == null, + "Settings for both ascending and descending ordering set, only one of these expected."); + + if (Boolean.TRUE.equals(descending[0]) || Boolean.FALSE.equals(ascending[0])) { + hitLimit[0] = hitLimit[0] * -1; + } + } + return hitLimit[0]; + } + + + @NonNull + private Item buildRange(String key, Inspector value) { + HashMap<Integer, Inspector> children = getChildrenMap(value); + Inspector annotations = getAnnotations(value); + + final boolean[] equals = {false}; + + String field; + Inspector boundInspector; + if (children.get(0).type() == STRING){ + field = children.get(0).asString(); + boundInspector = children.get(1); + } else { + field = children.get(1).asString(); + boundInspector = children.get(0); + } + + final Number[] bounds = {null, null}; + final String[] operators = {null, null}; + boundInspector.traverse((ObjectTraverser) (operator, bound) -> { + if (bound.type() == STRING) { + throw new IllegalArgumentException("Expected operator LITERAL, got READ_FIELD."); + } + if (operator.equals("=")) { + bounds[0] = (bound.type() == DOUBLE) ? Number.class.cast(bound.asDouble()) : Number.class.cast(bound.asLong()); + operators[0] = operator; + equals[0] = true; + } + if (operator.equals(">=") || operator.equals(">")){ + bounds[0] = (bound.type() == DOUBLE) ? Number.class.cast(bound.asDouble()) : Number.class.cast(bound.asLong()); + operators[0] = operator; + } else if (operator.equals("<=") || operator.equals("<")){ + bounds[1] = (bound.type() == DOUBLE) ? Number.class.cast(bound.asDouble()) : Number.class.cast(bound.asLong()); + operators[1] = operator; + } + + }); + IntItem range = null; + if (equals[0]){ + range = new IntItem(bounds[0].toString(), field); + } else if (operators[0]==null || operators[1]==null){ + Integer index = (operators[0] == null) ? 1 : 0; + switch (operators[index]){ + case ">=": + range = buildGreaterThanOrEquals(field, bounds[index].toString()); + break; + case ">": + range = buildGreaterThan(field, bounds[index].toString()); + break; + case "<": + range = buildLessThan(field, bounds[index].toString()); + break; + case "<=": + range = buildLessThanOrEquals(field, bounds[index].toString()); + break; + } + } + else { + range = instantiateRangeItem(bounds[0], bounds[1], field, operators[0].equals(">"), operators[1].equals("<")); + } + + return leafStyleSettings(annotations, range); + } + + @NonNull + private IntItem buildGreaterThanOrEquals(String field, String bound) { + return new IntItem("[" + bound + ";]", field); + + } + + + @NonNull + private IntItem buildLessThanOrEquals(String field, String bound) { + return new IntItem("[;" + bound + "]", field); + } + + + @NonNull + private IntItem buildGreaterThan(String field, String bound) { + return new IntItem(">" + bound, field); + + } + + + @NonNull + private IntItem buildLessThan(String field, String bound) { + return new IntItem("<" + bound, field); + } + + + @NonNull + private IntItem instantiateRangeItem(Number lowerBound, Number upperBound, String field, boolean bounds_left_open, boolean bounds_right_open) { + Preconditions.checkArgument(lowerBound != null && upperBound != null && field != null, + "Expected 3 NonNull-arguments"); + + if (!bounds_left_open && !bounds_right_open) { + return new RangeItem(lowerBound, upperBound, field); + } else { + Limit from; + Limit to; + if (bounds_left_open && bounds_right_open) { + from = new Limit(lowerBound, false); + to = new Limit(upperBound, false); + } else if (bounds_left_open) { + from = new Limit(lowerBound, false); + to = new Limit(upperBound, true); + } else { + from = new Limit(lowerBound, true); + to = new Limit(upperBound, false); + } + return new IntItem(from, to, field); + } + } + + + @NonNull + private Item buildEquals(String key, Inspector value) { + return buildRange(key, value); + } + + + @NonNull + private Item buildWand(String key, Inspector value) { + HashMap<String, Inspector> annotations = getAnnotationMap(value); + HashMap<Integer, Inspector> children = getChildrenMap(value); + + Preconditions.checkArgument(children.size() == 2, "Expected 2 arguments, got %s.", children.size()); + Integer target_num_hits= getIntegerAnnotation(TARGET_NUM_HITS, annotations, DEFAULT_TARGET_NUM_HITS); + + WandItem out = new WandItem(children.get(0).asString(), target_num_hits); + + Double scoreThreshold = getDoubleAnnotation(SCORE_THRESHOLD, annotations, null); + + if (scoreThreshold != null) { + out.setScoreThreshold(scoreThreshold); + } + + Double thresholdBoostFactor = getDoubleAnnotation(THRESHOLD_BOOST_FACTOR, annotations, null); + if (thresholdBoostFactor != null) { + out.setThresholdBoostFactor(thresholdBoostFactor); + } + return fillWeightedSet(value, children, out); + } + + + @NonNull + private WeightedSetItem fillWeightedSet(Inspector value, HashMap<Integer, Inspector> children, @NonNull WeightedSetItem out) { + addItems(children, out); + + return leafStyleSettings(getAnnotations(value), out); + } + + + private static void addItems(HashMap<Integer, Inspector> children, WeightedSetItem out) { + switch (children.get(1).type()) { + case OBJECT: + addStringItems(children, out); + break; + case ARRAY: + addLongItems(children, out); + break; + default: + throw newUnexpectedArgumentException(children.get(1).type(), ARRAY, OBJECT); + } + } + + + private static void addStringItems(HashMap<Integer, Inspector> children, WeightedSetItem out) { + //{"a":1, "b":2} + children.get(1).traverse((ObjectTraverser) (key, value) -> { + if (value.type() == STRING){ + throw new IllegalArgumentException("Expected operator LITERAL, got READ_FIELD."); + } + out.addToken(key, (int)value.asLong()); + }); + } + + + private static void addLongItems(HashMap<Integer, Inspector> children, WeightedSetItem out) { + //[[11,1], [37,2]] + children.get(1).traverse((ArrayTraverser) (index, pair) -> { + List<Integer> pairValues = new ArrayList<>(); + pair.traverse((ArrayTraverser) (pairIndex, pairValue) -> { + pairValues.add((int)pairValue.asLong()); + }); + Preconditions.checkArgument(pairValues.size() == 2, + "Expected item and weight, got %s.", pairValues); + out.addToken(pairValues.get(0).longValue(), pairValues.get(1)); + }); + } + + + @NonNull + private Item buildRegExpSearch(String key, Inspector value) { + assertHasOperator(key, MATCHES); + HashMap<Integer, Inspector> children = getChildrenMap(value); + String field = children.get(0).asString(); + String wordData = children.get(1).asString(); + RegExpItem regExp = new RegExpItem(field, true, wordData); + return leafStyleSettings(getAnnotations(value), regExp); + } + + + @NonNull + private Item buildWeightedSet(String key, Inspector value) { + HashMap<Integer, Inspector> children = getChildrenMap(value); + String field = children.get(0).asString(); + Preconditions.checkArgument(children.size() == 2, "Expected 2 arguments, got %s.", children.size()); + return fillWeightedSet(value, children, new WeightedSetItem(field)); + } + + + @NonNull + private Item buildDotProduct(String key, Inspector value) { + HashMap<Integer, Inspector> children = getChildrenMap(value); + String field = children.get(0).asString(); + Preconditions.checkArgument(children.size() == 2, "Expected 2 arguments, got %s.", children.size()); + return fillWeightedSet(value, children, new DotProductItem(field)); + } + + + @NonNull + private Item buildPredicate(String key, Inspector value) { + HashMap<Integer, Inspector> children = getChildrenMap(value); + String field = children.get(0).asString(); + Inspector args = children.get(1); + + Preconditions.checkArgument(children.size() == 3, "Expected 3 arguments, got %s.", children.size()); + + PredicateQueryItem item = new PredicateQueryItem(); + item.setIndexName(field); + + List<Inspector> argumentList = valueListFromInspector(getChildren(value)); + + // Adding attributes + argumentList.get(1).traverse((ObjectTraverser) (attrKey, attrValue) -> { + if (attrValue.type() == ARRAY){ + List<Inspector> attributes = valueListFromInspector(attrValue); + attributes.forEach( (attribute) -> item.addFeature(attrKey, attribute.asString())); + } else { + item.addFeature(attrKey, attrValue.asString()); + } + }); + + // Adding range attributes + argumentList.get(2).traverse((ObjectTraverser) (attrKey, attrValue) -> item.addRangeFeature(attrKey, (int)attrValue.asDouble())); + + return leafStyleSettings(getAnnotations(value), item); + } + + + @NonNull + private CompositeItem buildRank(String key, Inspector value) { + RankItem rankItem = new RankItem(); + addItemsFromInspector(rankItem, value); + return rankItem; + } + + + @NonNull + private Item buildTermSearch(String key, Inspector value) { + HashMap<Integer, Inspector> children = getChildrenMap(value); + String field = children.get(0).asString(); + + return instantiateLeafItem(field, key, value); + } + + + private String getInspectorKey(Inspector inspector){ + String[] actualKey = {""}; + if (inspector.type() == OBJECT){ + inspector.traverse((ObjectTraverser) (key, value) -> { + actualKey[0] = key; + + }); + } + return actualKey[0]; + } + + + @NonNull + private Item instantiateLeafItem(String field, String key, Inspector value) { + List<Inspector> possibleLeafFunction = valueListFromInspector(value); + String possibleLeafFunctionName = (possibleLeafFunction.size() > 1) ? getInspectorKey(possibleLeafFunction.get(1)) : ""; + if (FUNCTION_CALLS.contains(key)) { + return instantiateCompositeLeaf(field, key, value); + } else if(!possibleLeafFunctionName.equals("")){ + return instantiateCompositeLeaf(field, possibleLeafFunctionName, valueListFromInspector(value).get(1).field(possibleLeafFunctionName)); + } else { + return instantiateWordItem(field, key, value); + } + } + + + @NonNull + private Item instantiateCompositeLeaf(String field, String key, Inspector value) { + switch (key) { + case SAME_ELEMENT: + return instantiateSameElementItem(field, key, value); + case PHRASE: + return instantiatePhraseItem(field, key, value); + case NEAR: + return instantiateNearItem(field, key, value); + case ONEAR: + return instantiateONearItem(field, key, value); + case EQUIV: + return instantiateEquivItem(field, key, value); + case ALTERNATIVES: + return instantiateWordAlternativesItem(field, key, value); + default: + throw newUnexpectedArgumentException(key, EQUIV, NEAR, ONEAR, PHRASE, SAME_ELEMENT); + } + } + + + @NonNull + private Item instantiateWordItem(String field, String key, Inspector value) { + String wordData = getChildrenMap(value).get(1).asString(); + return instantiateWordItem(field, wordData, key, value, false, decideParsingLanguage(value, wordData)); + } + + + @NonNull + private Item instantiateWordItem(String field, String rawWord, String key, Inspector value, boolean exactMatch, Language language) { + String wordData = rawWord; + HashMap<String, Inspector> annotations = getAnnotationMap(value); + + if (getBoolAnnotation(NFKC, annotations, Boolean.FALSE)) { + // NOTE: If this is set to FALSE (default), we will still NFKC normalize text data + // during tokenization/segmentation, as that is always turned on also on the indexing side. + wordData = normalizer.normalize(wordData); + } + boolean fromQuery = getBoolAnnotation(IMPLICIT_TRANSFORMS, annotations, Boolean.TRUE); + boolean prefixMatch = getBoolAnnotation(PREFIX, annotations, Boolean.FALSE); + boolean suffixMatch = getBoolAnnotation(SUFFIX, annotations, Boolean.FALSE); + boolean substrMatch = getBoolAnnotation(SUBSTRING,annotations, Boolean.FALSE); + + Preconditions.checkArgument((prefixMatch ? 1 : 0) + + (substrMatch ? 1 : 0) + (suffixMatch ? 1 : 0) < 2, + "Only one of prefix, substring and suffix can be set."); + @NonNull + final TaggableItem wordItem; + + if (exactMatch) { + wordItem = new ExactStringItem(wordData, fromQuery); + } else if (prefixMatch) { + wordItem = new PrefixItem(wordData, fromQuery); + } else if (suffixMatch) { + wordItem = new SuffixItem(wordData, fromQuery); + } else if (substrMatch) { + wordItem = new SubstringItem(wordData, fromQuery); + } else { + wordItem = new WordItem(wordData, fromQuery); + } + + if (wordItem instanceof WordItem) { + prepareWord(field, value, (WordItem) wordItem); + } + if (language != Language.ENGLISH) + ((Item)wordItem).setLanguage(language); + + return (Item) leafStyleSettings(getAnnotations(value), wordItem); + } + + + private Language decideParsingLanguage(Inspector value, String wordData) { + String languageTag = getAnnotation(USER_INPUT_LANGUAGE, getAnnotationMap(value), String.class, null); + + Language language = Language.fromLanguageTag(languageTag); + if (language != Language.UNKNOWN) return language; + + Optional<Language> explicitLanguage = query.getExplicitLanguage(); + if (explicitLanguage.isPresent()) return explicitLanguage.get(); + + return Language.ENGLISH; + } + + + private void prepareWord(String field, Inspector value, WordItem wordItem) { + wordItem.setIndexName(field); + wordStyleSettings(value, wordItem); + } + + + private void wordStyleSettings(Inspector value, WordItem out) { + HashMap<String, Inspector> annotations = getAnnotationMap(value); + + Substring origin = getOrigin(getAnnotations(value)); + if (origin != null) { + out.setOrigin(origin); + } + if (annotations != null){ + Boolean usePositionData = Boolean.getBoolean(getAnnotation(USE_POSITION_DATA, annotations, String.class, null)); + if (usePositionData != null) { + out.setPositionData(usePositionData); + } + Boolean stem = getBoolAnnotation(STEM, annotations, null); + if (stem != null) { + out.setStemmed(!stem); + } + + Boolean normalizeCase = getBoolAnnotation(NORMALIZE_CASE, annotations, null); + if (normalizeCase != null) { + out.setLowercased(!normalizeCase); + } + Boolean accentDrop = getBoolAnnotation(ACCENT_DROP, annotations, null); + if (accentDrop != null) { + out.setNormalizable(accentDrop); + } + Boolean andSegmenting = getBoolAnnotation(AND_SEGMENTING, annotations, null); + if (andSegmenting != null) { + if (andSegmenting) { + out.setSegmentingRule(SegmentingRule.BOOLEAN_AND); + } else { + out.setSegmentingRule(SegmentingRule.PHRASE); + } + } + } + } + + + private Substring getOrigin(Inspector annotations) { + if (annotations != null) { + Inspector origin = getAnnotationAsInspectorOrNull(ORIGIN, getAnnotationMapFromAnnotationInspector(annotations)); + if (origin == null) { + return null; + } + final String[] original = {null}; + final Integer[] offset = {null}; + final Integer[] length = {null}; + + origin.traverse((ObjectTraverser) (key, value) -> { + switch (key) { + case (ORIGIN_ORIGINAL): + original[0] = value.asString(); + break; + case (ORIGIN_OFFSET): + offset[0] = (int) value.asDouble(); + break; + case (ORIGIN_LENGTH): + length[0] = (int) value.asDouble(); + break; + } + + + }); + return new Substring(offset[0], length[0] + offset[0], original[0]); + } + return null; + } + + + @NonNull + private Item instantiateSameElementItem(String field, String key, Inspector value) { + assertHasOperator(key, SAME_ELEMENT); + + SameElementItem sameElement = new SameElementItem(field); + // All terms below sameElement are relative to this. + getChildren(value).traverse((ArrayTraverser) (index, term) -> { + sameElement.addItem(walkJson(term)); + }); + + return sameElement; + } + + + @NonNull + private Item instantiatePhraseItem(String field, String key, Inspector value) { + assertHasOperator(key, PHRASE); + HashMap<String, Inspector> annotations = getAnnotationMap(value); + + PhraseItem phrase = new PhraseItem(); + phrase.setIndexName(field); + HashMap<Integer, Inspector> children = getChildrenMap(value); + + for (Inspector word : children.values()) + if (word.type() == STRING) phrase.addItem(new WordItem(word.asString())); + else if (word.type() == OBJECT && word.field(PHRASE).valid()) { + phrase.addItem(instantiatePhraseItem(field, key, getChildren(word))); + } + return leafStyleSettings(getAnnotations(value), phrase); + } + + + @NonNull + private Item instantiateNearItem(String field, String key, Inspector value) { + assertHasOperator(key, NEAR); + + NearItem near = new NearItem(); + near.setIndexName(field); + + HashMap<Integer, Inspector> children = getChildrenMap(value); + + for (Inspector word : children.values()){ + near.addItem(new WordItem(word.asString(), field)); + } + + Integer distance = getIntegerAnnotation(DISTANCE, getAnnotationMap(value), null); + + if (distance != null) { + near.setDistance((int)distance); + } + return near; + } + + + @NonNull + private Item instantiateONearItem(String field, String key, Inspector value) { + assertHasOperator(key, ONEAR); + + NearItem onear = new ONearItem(); + onear.setIndexName(field); + HashMap<Integer, Inspector> children = getChildrenMap(value); + + for (Inspector word : children.values()){ + onear.addItem(new WordItem(word.asString(), field)); + } + + Integer distance = getIntegerAnnotation(DISTANCE, getAnnotationMap(value), null); + if (distance != null) { + onear.setDistance(distance); + } + return onear; + } + + + @NonNull + private Item instantiateEquivItem(String field, String key, Inspector value) { + + HashMap<Integer, Inspector> children = getChildrenMap(value); + Preconditions.checkArgument(children.size() >= 2, "Expected 2 or more arguments, got %s.", children.size()); + + EquivItem equiv = new EquivItem(); + equiv.setIndexName(field); + + for (Inspector word : children.values()){ + if (word.type() == STRING || word.type() == LONG || word.type() == DOUBLE){ + equiv.addItem(new WordItem(word.asString(), field)); + } + if (word.type() == OBJECT){ + word.traverse((ObjectTraverser) (key2, value2) -> { + assertHasOperator(key2, PHRASE); + equiv.addItem(instantiatePhraseItem(field, key2, value2)); + }); + } + } + + return leafStyleSettings(getAnnotations(value), equiv); + } + + + private Item instantiateWordAlternativesItem(String field, String key, Inspector value) { + HashMap<Integer, Inspector> children = getChildrenMap(value); + Preconditions.checkArgument(children.size() >= 1, "Expected 1 or more arguments, got %s.", children.size()); + Preconditions.checkArgument(children.get(0).type() == OBJECT, "Expected OBJECT, got %s.", children.get(0).type()); + + List<WordAlternativesItem.Alternative> terms = new ArrayList<>(); + + children.get(0).traverse((ObjectTraverser) (keys, values) -> { + terms.add(new WordAlternativesItem.Alternative(keys, values.asDouble())); + }); + return leafStyleSettings(getAnnotations(value), new WordAlternativesItem(field, Boolean.TRUE, null, terms)); + } + + + // Not in use yet + @NonNull + private String getIndex(String field) { + Preconditions.checkArgument(indexFactsSession.isIndex(field), "Field '%s' does not exist.", field); + //return indexFactsSession.getCanonicName(field); + return field; + } + + + private static void assertHasOperator(String key, String expectedKey) { + Preconditions.checkArgument(key.equals(expectedKey), "Expected operator %s, got %s.", expectedKey, key); + } + + + private static IllegalArgumentException newUnexpectedArgumentException(Object actual, Object... expected) { + StringBuilder out = new StringBuilder("Expected "); + for (int i = 0, len = expected.length; i < len; ++i) { + out.append(expected[i]); + if (i < len - 2) { + out.append(", "); + } else if (i < len - 1) { + out.append(" or "); + } + } + out.append(", got ").append(actual).append("."); + return new IllegalArgumentException(out.toString()); + } + + + private List<Inspector> valueListFromInspector(Inspector inspector){ + List<Inspector> inspectorList = new ArrayList<>(); + inspector.traverse((ArrayTraverser) (key, value) -> inspectorList.add(value)); + return inspectorList; + } + + + private void connectItems() { + for (ConnectedItem entry : connectedItems) { + TaggableItem to = identifiedItems.get(entry.toId); + Preconditions.checkNotNull(to, + "Item '%s' was specified to connect to item with ID %s, which does not " + + "exist in the query.", entry.fromItem, + entry.toId); + entry.fromItem.setConnectivity((Item) to, entry.weight); + } + } + + + private static final class ConnectedItem { + + final double weight; + final int toId; + final TaggableItem fromItem; + + ConnectedItem(TaggableItem fromItem, int toId, double weight) { + this.weight = weight; + this.toId = toId; + this.fromItem = fromItem; + } + } + + +} diff --git a/container-search/src/main/java/com/yahoo/search/query/parser/Parsable.java b/container-search/src/main/java/com/yahoo/search/query/parser/Parsable.java index e5941a90b83..64fb201fe21 100644 --- a/container-search/src/main/java/com/yahoo/search/query/parser/Parsable.java +++ b/container-search/src/main/java/com/yahoo/search/query/parser/Parsable.java @@ -3,6 +3,7 @@ package com.yahoo.search.query.parser; import com.yahoo.language.Language; import com.yahoo.search.query.Model; +import com.yahoo.search.query.Select; import java.util.Collection; import java.util.HashSet; @@ -36,6 +37,7 @@ public final class Parsable { private String defaultIndexName; private Language language; // TODO: Initialize to UNKNOWN private Optional<Language> explicitLanguage = Optional.empty(); + private Select select; /** If this is set it will be used to determine the language, if not set explicitly */ private Optional<Model> model = Optional.empty(); @@ -133,6 +135,15 @@ public final class Parsable { return this; } + public Parsable setSelect(Select select){ + this.select = select; + return this; + } + + public Select getSelect(){ + return this.select; + } + public static Parsable fromQueryModel(Model model) { return new Parsable() .setModel(model) @@ -141,7 +152,11 @@ public final class Parsable { .setExplicitLanguage(Optional.ofNullable(model.getLanguage())) .setDefaultIndexName(model.getDefaultIndex()) .addSources(model.getSources()) - .addRestricts(model.getRestrict()); + .addRestricts(model.getRestrict()) + .setSelect(model.getParent().getSelect()); } + + + } diff --git a/container-search/src/main/java/com/yahoo/search/query/parser/ParserFactory.java b/container-search/src/main/java/com/yahoo/search/query/parser/ParserFactory.java index 8d008abaac2..69d46527255 100644 --- a/container-search/src/main/java/com/yahoo/search/query/parser/ParserFactory.java +++ b/container-search/src/main/java/com/yahoo/search/query/parser/ParserFactory.java @@ -3,6 +3,7 @@ package com.yahoo.search.query.parser; import com.yahoo.prelude.query.parser.*; import com.yahoo.search.Query; +import com.yahoo.search.query.SelectParser; import com.yahoo.search.yql.YqlParser; /** @@ -40,6 +41,8 @@ public final class ParserFactory { return new ProgrammaticParser(); case YQL: return new YqlParser(environment); + case SELECT: + return new SelectParser(environment); default: throw new UnsupportedOperationException(type.toString()); } diff --git a/container-search/src/main/java/com/yahoo/search/query/properties/QueryProperties.java b/container-search/src/main/java/com/yahoo/search/query/properties/QueryProperties.java index 0aea5e96161..71002166b11 100644 --- a/container-search/src/main/java/com/yahoo/search/query/properties/QueryProperties.java +++ b/container-search/src/main/java/com/yahoo/search/query/properties/QueryProperties.java @@ -113,6 +113,10 @@ public class QueryProperties extends Properties { if (key.get(1).equals(Ranking.PROPERTIES)) return ranking.getProperties().get(key.rest().rest().toString()); } } + else if (key.size()==2 && key.first().equals(Select.SELECT)) { + if (key.last().equals(Select.WHERE)) return query.getSelect().getWhereString(); + if (key.last().equals(Select.GROUPING)) return query.getSelect().getGroupingString(); + } else if (key.size()==2 && key.first().equals(Presentation.PRESENTATION)) { if (key.last().equals(Presentation.BOLDING)) return query.getPresentation().getBolding(); if (key.last().equals(Presentation.SUMMARY)) return query.getPresentation().getSummary(); @@ -247,6 +251,13 @@ public class QueryProperties extends Properties { else if ( ! key.last().equals(Presentation.REPORT_COVERAGE)) // TODO: Change this line to "else" on Vespa 7 throwIllegalParameter(key.last(), Presentation.PRESENTATION); } + else if (key.size()==2 && key.first().equals(Select.SELECT)) { + if (key.last().equals(Select.WHERE)){ + query.getSelect().setWhere(asString(value, "")); + } else if (key.last().equals(Select.GROUPING)) { + query.getSelect().setGrouping(asString(value, "")); + } + } else if (key.first().equals("rankfeature") || key.first().equals("featureoverride") ) { // featureoverride is deprecated setRankingFeature(query, key.rest().toString(), toSpecifiedType(key.rest().toString(), value, profileRegistry.getTypeRegistry().getComponent("features"))); } else if (key.first().equals("rankproperty")) { diff --git a/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java b/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java index 6bad032600c..e0e9042e1a3 100644 --- a/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java +++ b/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java @@ -789,7 +789,7 @@ public class YqlParser implements Parser { OperatorNode<ExpressionOperator> groupingAst = ast.<List<OperatorNode<ExpressionOperator>>> getArgument(2).get(0); GroupingOperation groupingOperation = GroupingOperation.fromString(groupingAst.<String> getArgument(0)); VespaGroupingStep groupingStep = new VespaGroupingStep(groupingOperation); - List<String> continuations = getAnnotation(groupingAst, "continuations", List.class, + List<String> continuations = getAnnotation(groupingAst, "continuations", List.class, Collections.emptyList(), "grouping continuations"); for (String continuation : continuations) { groupingStep.continuations().add(Continuation.fromString(continuation)); diff --git a/container-search/src/main/resources/configdefinitions/provider.def b/container-search/src/main/resources/configdefinitions/provider.def index 79b09913b49..f9ab305b114 100644 --- a/container-search/src/main/resources/configdefinitions/provider.def +++ b/container-search/src/main/resources/configdefinitions/provider.def @@ -35,7 +35,7 @@ yca.ttl int default=0 yca.retry int default=0 # The form of the serialized query. -queryType enum { LEGACY, PROGRAMMATIC, YQL } default=LEGACY +queryType enum { LEGACY, PROGRAMMATIC, YQL, SELECT } default=LEGACY # How to do pinging against a backend. pingOption enum { DISABLE, NORMAL, YCA } default=NORMAL diff --git a/container-search/src/test/java/com/yahoo/search/handler/test/JSONSearchHandlerTestCase.java b/container-search/src/test/java/com/yahoo/search/handler/test/JSONSearchHandlerTestCase.java index eea58d5444e..e85a945cc67 100644 --- a/container-search/src/test/java/com/yahoo/search/handler/test/JSONSearchHandlerTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/handler/test/JSONSearchHandlerTestCase.java @@ -11,7 +11,10 @@ import com.yahoo.io.IOUtils; import com.yahoo.net.HostName; import com.yahoo.search.handler.SearchHandler; import com.yahoo.search.searchchain.config.test.SearchChainConfigurerTestCase; +import com.yahoo.slime.ArrayTraverser; import com.yahoo.slime.Inspector; +import com.yahoo.slime.ObjectTraverser; +import com.yahoo.slime.Type; import com.yahoo.vespa.config.SlimeUtils; import org.json.JSONObject; import org.junit.After; @@ -21,8 +24,10 @@ import org.junit.Test; import org.junit.rules.TemporaryFolder; import java.io.File; import java.io.IOException; +import java.util.ArrayList; import java.util.Map; import java.util.HashMap; +import java.util.stream.Collectors; import static com.yahoo.jdisc.http.HttpRequest.Method.GET; import static org.hamcrest.CoreMatchers.containsString; @@ -338,6 +343,35 @@ public class JSONSearchHandlerTestCase { } + @Test + public void testSelectParameter() throws Exception { + JSONObject json = new JSONObject(); + + JSONObject select = new JSONObject(); + + JSONObject where = new JSONObject(); + where.put("where", "where"); + + JSONObject grouping = new JSONObject(); + grouping.put("grouping", "grouping"); + + select.put("where", where); + select.put("grouping", grouping); + + json.put("select", select); + + + // Create mapping + Inspector inspector = SlimeUtils.jsonToSlime(json.toString().getBytes("utf-8")).get(); + Map<String, String> map = new HashMap<>(); + searchHandler.createRequestMapping(inspector, map, ""); + + JSONObject processedWhere = new JSONObject(map.get("select.where")); + assertEquals(where.toString(), processedWhere.toString()); + + JSONObject processedGrouping = new JSONObject(map.get("select.grouping")); + assertEquals(grouping.toString(), processedGrouping.toString()); + } @Test public void testRequestMapping() throws Exception { diff --git a/container-search/src/test/java/com/yahoo/select/SelectParserTestCase.java b/container-search/src/test/java/com/yahoo/select/SelectParserTestCase.java new file mode 100644 index 00000000000..031ba386ad4 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/select/SelectParserTestCase.java @@ -0,0 +1,779 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.select; + +import com.yahoo.prelude.query.AndItem; +import com.yahoo.prelude.query.ExactStringItem; +import com.yahoo.prelude.query.Item; +import com.yahoo.prelude.query.PhraseItem; +import com.yahoo.prelude.query.PrefixItem; +import com.yahoo.prelude.query.RegExpItem; +import com.yahoo.prelude.query.SegmentingRule; +import com.yahoo.prelude.query.Substring; +import com.yahoo.prelude.query.SubstringItem; +import com.yahoo.prelude.query.SuffixItem; +import com.yahoo.prelude.query.WeakAndItem; +import com.yahoo.prelude.query.WordAlternativesItem; +import com.yahoo.prelude.query.WordItem; +import com.yahoo.search.Query; +import com.yahoo.search.federation.ProviderConfig; +import com.yahoo.search.query.QueryTree; +import com.yahoo.search.query.Select; +import com.yahoo.search.query.SelectParser; +import com.yahoo.search.query.parser.Parsable; +import com.yahoo.search.query.parser.ParserEnvironment; +import com.yahoo.search.yql.VespaGroupingStep; +import org.apache.http.client.utils.URIBuilder; +import org.json.JSONException; +import org.json.JSONObject; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + + +/** + * Specification for the conversion of Select expressions to Vespa search queries. + * + * @author henrhoi + */ + +public class SelectParserTestCase { + + private final SelectParser parser = new SelectParser(new ParserEnvironment()); + + + /** WHERE TESTS */ + + @Test + public void test_contains() throws Exception { + JSONObject json = new JSONObject(); + List<String> contains = Arrays.asList("default", "foo"); + json.put("contains", contains); + assertParse(json.toString(), "default:foo"); + } + + @Test + public void test() { + assertParse("{'contains' : ['title', 'madonna']}", + "title:madonna"); + } + + + @Test + public void testDottedFieldNames() { + assertParse("{ 'contains' : ['my.nested.title', 'madonna']}", + "my.nested.title:madonna"); + } + + + + @Test + public void testOr() throws Exception { + JSONObject json_two_or = new JSONObject(); + JSONObject json_three_or = new JSONObject(); + List<String> contains1 = Arrays.asList("title", "madonna"); + List<String> contains2 = Arrays.asList("title", "saint"); + List<String> contains3 = Arrays.asList("title", "angel"); + + JSONObject contains_json1 = new JSONObject(); + JSONObject contains_json2 = new JSONObject(); + JSONObject contains_json3 = new JSONObject(); + contains_json1.put("contains", contains1); + contains_json2.put("contains", contains2); + contains_json3.put("contains", contains3); + + json_two_or.put("or", Arrays.asList(contains_json1, contains_json2)); + json_three_or.put("or", Arrays.asList(contains_json1, contains_json2, contains_json3)); + + assertParse(json_two_or.toString(), "OR title:madonna title:saint"); + assertParse(json_three_or.toString(), "OR title:madonna title:saint title:angel"); + } + + @Test + public void testAnd() throws Exception{ + JSONObject json_two_and = new JSONObject(); + JSONObject json_three_and = new JSONObject(); + List<String> contains1 = Arrays.asList("title", "madonna"); + List<String> contains2 = Arrays.asList("title", "saint"); + List<String> contains3 = Arrays.asList("title", "angel"); + + JSONObject contains_json1 = new JSONObject(); + JSONObject contains_json2 = new JSONObject(); + JSONObject contains_json3 = new JSONObject(); + contains_json1.put("contains", contains1); + contains_json2.put("contains", contains2); + contains_json3.put("contains", contains3); + + json_two_and.put("and", Arrays.asList(contains_json1, contains_json2)); + json_three_and.put("and", Arrays.asList(contains_json1, contains_json2, contains_json3)); + + assertParse(json_two_and.toString(), "AND title:madonna title:saint"); + assertParse(json_three_and.toString(), "AND title:madonna title:saint title:angel"); + } + + @Test + public void testAndNot() throws JSONException { + JSONObject json_and_not = new JSONObject(); + List<String> contains1 = Arrays.asList("title", "madonna"); + List<String> contains2 = Arrays.asList("title", "saint"); + + JSONObject contains_json1 = new JSONObject(); + JSONObject contains_json2 = new JSONObject(); + contains_json1.put("contains", contains1); + contains_json2.put("contains", contains2); + + json_and_not.put("and_not", Arrays.asList(contains_json1, contains_json2)); + + assertParse(json_and_not.toString(), + "+title:madonna -title:saint"); + } + + + @Test + public void testLessThan() throws JSONException { + JSONObject range_json = new JSONObject(); + JSONObject operators = new JSONObject(); + operators.put("<", 500); + + List<Object> range = Arrays.asList("price", operators); + + range_json.put("range", range); + + assertParse(range_json.toString(), + "price:<500"); + } + + @Test + public void testGreaterThan() throws JSONException { + JSONObject range_json = new JSONObject(); + JSONObject operators = new JSONObject(); + operators.put(">", 500); + + List<Object> range = Arrays.asList("price", operators); + + range_json.put("range", range); + + assertParse(range_json.toString(), + "price:>500"); + } + + + @Test + public void testLessThanOrEqual() throws JSONException { + JSONObject range_json = new JSONObject(); + JSONObject operators = new JSONObject(); + operators.put("<=", 500); + + List<Object> range = Arrays.asList("price", operators); + + range_json.put("range", range); + + assertParse(range_json.toString(), + "price:[;500]"); + } + + @Test + public void testGreaterThanOrEqual() throws JSONException { + JSONObject range_json = new JSONObject(); + JSONObject operators = new JSONObject(); + operators.put(">=", 500); + + List<Object> range = Arrays.asList("price", operators); + + range_json.put("range", range); + + assertParse(range_json.toString(), + "price:[500;]"); + } + + @Test + public void testEquality() throws JSONException { + JSONObject range_json = new JSONObject(); + JSONObject operators = new JSONObject(); + operators.put("=", 500); + + List<Object> range = Arrays.asList("price", operators); + + range_json.put("range", range); + + assertParse(range_json.toString(), + "price:500"); + } + + @Test + public void testNegativeLessThan() throws JSONException { + JSONObject range_json = new JSONObject(); + JSONObject operators = new JSONObject(); + operators.put("<", -500); + + List<Object> range = Arrays.asList("price", operators); + + range_json.put("range", range); + + assertParse(range_json.toString(), + "price:<-500"); + } + + @Test + public void testNegativeGreaterThan() throws JSONException { + JSONObject range_json = new JSONObject(); + JSONObject operators = new JSONObject(); + operators.put(">", -500); + + List<Object> range = Arrays.asList("price", operators); + + range_json.put("range", range); + + assertParse(range_json.toString(), + "price:>-500"); + } + + @Test + public void testNegativeLessThanOrEqual() throws JSONException { + JSONObject range_json = new JSONObject(); + JSONObject operators = new JSONObject(); + operators.put("<=", -500); + + List<Object> range = Arrays.asList("price", operators); + + range_json.put("range", range); + + assertParse(range_json.toString(), + "price:[;-500]"); + } + + @Test + public void testNegativeGreaterThanOrEqual() throws JSONException { + JSONObject range_json = new JSONObject(); + JSONObject operators = new JSONObject(); + operators.put(">=", -500); + + List<Object> range = Arrays.asList("price", operators); + + range_json.put("range", range); + + assertParse(range_json.toString(), + "price:[-500;]"); + } + + @Test + public void testNegativeEquality() throws JSONException { + JSONObject range_json = new JSONObject(); + JSONObject operators = new JSONObject(); + operators.put("=", -500); + + List<Object> range = Arrays.asList("price", operators); + + range_json.put("range", range); + + assertParse(range_json.toString(), + "price:-500"); + } + + @Test + public void testAnnotatedLessThan() { + String jsonString = "{ \"range\": { \"children\" : [\"price\", {\"<\" : -500}], \"attributes\" : {\"filter\" : true} } }"; + assertParse(jsonString, "|price:<-500"); + } + + @Test + public void testAnnotatedGreaterThan() { + String jsonString = "{ \"range\": { \"children\" : [\"price\", {\">\" : 500}], \"attributes\" : {\"filter\" : true} } }"; + assertParse(jsonString, "|price:>500"); + } + + @Test + public void testAnnotatedLessThanOrEqual() { + String jsonString = "{ \"range\": { \"children\" : [\"price\", {\"<=\" : -500}], \"attributes\" : {\"filter\" : true} } }"; + assertParse(jsonString, "|price:[;-500]"); + } + + @Test + public void testAnnotatedGreaterThanOrEqual() { + String jsonString = "{ \"range\": { \"children\" : [\"price\", {\">=\" : 500}], \"attributes\" : {\"filter\" : true} } }"; + assertParse(jsonString, "|price:[500;]"); + } + + + @Test + public void testAnnotatedEquality() { + String jsonString = "{ \"range\": { \"children\" : [\"price\", {\"=\" : -500}], \"attributes\" : {\"filter\" : true} } }"; + assertParse(jsonString, "|price:-500"); + } + + @Test + public void testTermAnnotations() { + assertEquals("merkelapp", + getRootWord("{ \"contains\": { \"children\" : [\"baz\", \"colors\"], \"attributes\" : {\"label\" : \"merkelapp\"} } }").getLabel()); + assertEquals("another", + getRootWord("{ \"contains\": { \"children\" : [\"baz\", \"colors\"], \"attributes\" : {\"annotations\" : {\"cox\" : \"another\"} } } }").getAnnotation("cox")); + assertEquals(23.0, getRootWord("{ \"contains\": { \"children\" : [\"baz\", \"colors\"], \"attributes\" : {\"significance\" : 23.0 } } }").getSignificance(), 1E-6); + assertEquals(150, getRootWord("{ \"contains\": { \"children\" : [\"baz\", \"colors\"], \"attributes\" : {\"weight\" : 150 } } }").getWeight()); + assertFalse(getRootWord("{ \"contains\": { \"children\" : [\"baz\", \"colors\"], \"attributes\" : {\"usePositionData\" : false } } }").usePositionData()); + assertTrue(getRootWord("{ \"contains\": { \"children\" : [\"baz\", \"colors\"], \"attributes\" : {\"filter\" : true } } }").isFilter()); + assertFalse(getRootWord("{ \"contains\": { \"children\" : [\"baz\", \"colors\"], \"attributes\" : {\"ranked\" : false } } }").isRanked()); + Substring origin = getRootWord("{ \"contains\": { \"children\" : [\"baz\", \"colors\"], \"attributes\" : {\"origin\": {\"original\": \"abc\", \"offset\": 1, \"length\": 2}} } }").getOrigin(); + assertEquals("abc", origin.string); + assertEquals(1, origin.start); + assertEquals(3, origin.end); + } + + + @Test + public void testSameElement() { + assertParse("{ \"contains\": [ \"baz\", {\"sameElement\" : [ { \"contains\" : [\"f1\", \"a\"] }, { \"contains\" : [\"f2\", \"b\"] } ]} ] }", + "baz:{f1:a f2:b}"); + + assertParse("{ \"contains\": [ \"baz\", {\"sameElement\" : [ { \"contains\" : [\"f1\", \"a\"] }, {\"range\":[\"f2\",{\"=\":10}] } ]} ] }", + "baz:{f1:a f2:10}"); + + assertParse("{ \"contains\": [ \"baz\", {\"sameElement\" : [ { \"contains\" : [\"key\", \"a\"] }, {\"range\":[\"value.f2\",{\"=\":10}] } ]} ] }", + "baz:{key:a value.f2:10}"); + } + + @Test + public void testPhrase() { + assertParse("{ \"contains\": [ \"baz\", {\"phrase\" : [ \"a\", \"b\"] } ] }", + "baz:\"a b\""); + } + + @Test + public void testNestedPhrase() { + assertParse("{ \"contains\": [ \"baz\", {\"phrase\" : [ \"a\", \"b\", {\"phrase\" : [ \"c\", \"d\"] }] } ] }", + "baz:\"a b c d\""); + } + + @Test + public void testStemming() { + assertTrue(getRootWord("{ \"contains\": { \"children\" : [\"baz\", \"colors\"], \"attributes\" : {\"stem\" : false} } }").isStemmed()); + assertFalse(getRootWord("{ \"contains\": { \"children\" : [\"baz\", \"colors\"], \"attributes\" : {\"stem\" : true} } }").isStemmed()); + assertFalse(getRootWord("{ \"contains\": [\"baz\", \"colors\"] }").isStemmed()); + } + + @Test + public void testRaw() { + Item root = parseWhere("{ \"contains\":[ \"baz\", \"yoni jo dima\" ] }").getRoot(); + assertTrue(root instanceof WordItem); + assertFalse(root instanceof ExactStringItem); + assertEquals("yoni jo dima", ((WordItem)root).getWord()); + + root = parseWhere("{ \"contains\": { \"children\" : [\"baz\", \"yoni jo dima\"], \"attributes\" : {\"grammar\" : \"raw\"} } }").getRoot(); + assertTrue(root instanceof WordItem); + assertFalse(root instanceof ExactStringItem); + assertEquals("yoni jo dima", ((WordItem)root).getWord()); + } + + @Test + public void testAccentDropping() { + assertFalse(getRootWord("{ \"contains\": { \"children\" : [\"baz\", \"colors\"], \"attributes\" : {\"accentDrop\" : false} } }").isNormalizable()); + assertTrue(getRootWord("{ \"contains\": { \"children\" : [\"baz\", \"colors\"], \"attributes\" : {\"accentDrop\" : true} } }").isNormalizable()); + assertTrue(getRootWord("{ \"contains\": { \"children\" : [\"baz\", \"colors\"] } }").isNormalizable()); + } + + @Test + public void testCaseNormalization() { + assertTrue(getRootWord("{ \"contains\": { \"children\" : [\"baz\", \"colors\"], \"attributes\" : {\"normalizeCase\" : false} } }").isLowercased()); + assertFalse(getRootWord("{ \"contains\": { \"children\" : [\"baz\", \"colors\"], \"attributes\" : {\"normalizeCase\" : true} } }").isLowercased()); + assertFalse(getRootWord("{ \"contains\": { \"children\" : [\"baz\", \"colors\"] } }").isLowercased()); + } + + @Test + public void testSegmentingRule() { + assertEquals(SegmentingRule.PHRASE, + getRootWord("{ \"contains\": { \"children\" : [\"baz\", \"colors\"], \"attributes\" : {\"andSegmenting\" : false} } }").getSegmentingRule()); + assertEquals(SegmentingRule.BOOLEAN_AND, + getRootWord("{ \"contains\": { \"children\" : [\"baz\", \"colors\"], \"attributes\" : {\"andSegmenting\" : true} } }").getSegmentingRule()); + assertEquals(SegmentingRule.LANGUAGE_DEFAULT, + getRootWord("{ \"contains\": { \"children\" : [\"baz\", \"colors\"] } }").getSegmentingRule()); + } + + @Test + public void testNfkc() { + assertEquals("a\u030a", getRootWord("{ \"contains\": { \"children\" : [\"baz\", \"a\\u030a\"], \"attributes\" : {\"nfkc\" : false} } }").getWord()); + assertEquals("\u00e5", getRootWord("{ \"contains\": { \"children\" : [\"baz\", \"a\\u030a\"], \"attributes\" : {\"nfkc\" : true} } }").getWord()); + assertEquals("No NKFC by default", "a\u030a", getRootWord("{ \"contains\": { \"children\" : [\"baz\", \"a\\u030a\"] } } ").getWord()); + } + + @Test + public void testImplicitTransforms() { + assertFalse(getRootWord("{ \"contains\": { \"children\" : [\"baz\", \"cox\"], \"attributes\" : {\"implicitTransforms\" : false} } }").isFromQuery()); + assertTrue(getRootWord("{ \"contains\": { \"children\" : [\"baz\", \"cox\"], \"attributes\" : {\"implicitTransforms\" : true} } }").isFromQuery()); + assertTrue(getRootWord("{ \"contains\": { \"children\" : [\"baz\", \"cox\"] } }").isFromQuery()); + } + + @Test + public void testConnectivity() { + QueryTree parsed = parseWhere("{ \"and\": [ {\"contains\" : { \"children\" : [\"title\", \"madonna\"], \"attributes\" : {\"id\": 1, \"connectivity\": {\"id\": 3, \"weight\": 7.0}} } }, " + + "{ \"contains\" : { \"children\" : [\"title\", \"saint\"], \"attributes\" : {\"id\": 2} } }, " + + "{ \"contains\" : { \"children\" : [\"title\", \"angel\"], \"attributes\" : {\"id\": 3} } } ] }"); + assertEquals("AND title:madonna title:saint title:angel", parsed.toString()); + + AndItem root = (AndItem)parsed.getRoot(); + WordItem first = (WordItem)root.getItem(0); + WordItem second = (WordItem)root.getItem(1); + WordItem third = (WordItem)root.getItem(2); + assertTrue(first.getConnectedItem() == third); + assertEquals(first.getConnectivity(), 7.0d, 1E-6); + assertNull(second.getConnectedItem()); + + assertParseFail("{ \"and\": [ {\"contains\" : { \"children\" : [\"title\", \"madonna\"], \"attributes\" : {\"id\": 1, \"connectivity\": {\"id\": 4, \"weight\": 7.0}} } }, " + + "{ \"contains\" : { \"children\" : [\"title\", \"saint\"], \"attributes\" : {\"id\": 2} } }, " + + "{ \"contains\" : { \"children\" : [\"title\", \"angel\"], \"attributes\" : {\"id\": 3} } } ] }", + new NullPointerException("Item 'title:madonna' was specified to connect to item with ID 4, " + + "which does not exist in the query.")); + } + + @Test + public void testAnnotatedPhrase() { + QueryTree parsed = parseWhere("{ \"contains\": [\"baz\", { \"phrase\": { \"children\": [\"a\", \"b\"], \"attributes\": { \"label\": \"hello world\" } } }] }"); + assertEquals("baz:\"a b\"", parsed.toString()); + PhraseItem phrase = (PhraseItem)parsed.getRoot(); + assertEquals("hello world", phrase.getLabel()); + } + + @Test + public void testRange() { + QueryTree parsed = parseWhere("{ \"range\": [\"baz\", { \">=\": 1, \"<=\": 8 }] }"); + assertEquals("baz:[1;8]", parsed.toString()); + } + + @Test + public void testNegativeRange() { + QueryTree parsed = parseWhere("{ \"range\": [\"baz\", { \">=\": -8, \"<=\": -1 }] }"); + assertEquals("baz:[-8;-1]", parsed.toString()); + } + + @Test + public void testRangeIllegalArguments() { + assertParseFail("{ \"range\": [\"baz\", { \">=\": \"cox\", \"<=\": -1 }] }", + new IllegalArgumentException("Expected operator LITERAL, got READ_FIELD.")); + } + + @Test + public void testNear() { + assertParse("{ \"contains\": [\"description\", { \"near\": [\"a\", \"b\"] }] }", + "NEAR(2) description:a description:b"); + assertParse("{ \"contains\": [\"description\", { \"near\": { \"children\": [\"a\", \"b\"], \"attributes\": { \"distance\": 100 } } } ] }", + "NEAR(100) description:a description:b"); + } + + @Test + public void testOrderedNear() { + assertParse("{ \"contains\": [\"description\", { \"onear\": [\"a\", \"b\"] }] }", + "ONEAR(2) description:a description:b"); + assertParse("{ \"contains\": [\"description\", { \"onear\": { \"children\": [\"a\", \"b\"], \"attributes\": { \"distance\": 100 } } } ] }", + "ONEAR(100) description:a description:b"); + } + + @Test + public void testWand() { + assertParse("{ \"wand\": [\"description\", { \"a\": 1, \"b\": 2 }] }", + "WAND(10,0.0,1.0) description{[1]:\"a\",[2]:\"b\"}"); + assertParse("{ \"wand\": { \"children\": [\"description\", { \"a\": 1, \"b\": 2 }], \"attributes\": { \"scoreThreshold\": 13.3, \"targetNumHits\": 7, \"thresholdBoostFactor\": 2.3 } } }", + "WAND(7,13.3,2.3) description{[1]:\"a\",[2]:\"b\"}"); + } + + @Test + public void testNumericWand() { + String numWand = "WAND(10,0.0,1.0) description{[1]:\"11\",[2]:\"37\"}"; + assertParse("{ \"wand\" : [\"description\", [[11,1], [37,2]] ]}", numWand); + assertParseFail("{ \"wand\" : [\"description\", 12] }", + new IllegalArgumentException("Expected ARRAY or OBJECT, got LONG.")); + } + + @Test + public void testWeightedSet() { + assertParse("{ \"weightedSet\" : [\"description\", {\"a\":1, \"b\":2} ]}", + "WEIGHTEDSET description{[1]:\"a\",[2]:\"b\"}"); + assertParseFail("{ \"weightedSet\" : [\"description\", {\"a\":\"g\", \"b\":2} ]}", + new IllegalArgumentException("Expected operator LITERAL, got READ_FIELD.")); + assertParseFail("{ \"weightedSet\" : [\"description\" ]}", + new IllegalArgumentException("Expected 2 arguments, got 1.")); + } + + @Test + public void testDotProduct() { + assertParse("{ \"dotProduct\" : [\"description\", {\"a\":1, \"b\":2} ]}", + "DOTPRODUCT description{[1]:\"a\",[2]:\"b\"}"); + assertParse("{ \"dotProduct\" : [\"description\", {\"a\":2} ]}", + "DOTPRODUCT description{[2]:\"a\"}"); + } + + @Test + public void testPredicate() { + assertParse("{ \"predicate\" : [\"predicate_field\", {\"gender\":\"male\", \"hobby\":[\"music\", \"hiking\"]}, {\"age\":23} ]}", + "PREDICATE_QUERY_ITEM gender=male, hobby=music, hobby=hiking, age:23"); + assertParse("{ \"predicate\" : [\"predicate_field\", 0, \"void\" ]}", + "PREDICATE_QUERY_ITEM "); + } + + @Test + public void testRank() { + assertParse("{ \"rank\": [{ \"contains\": [\"a\", \"A\"] }, { \"contains\": [\"b\", \"B\"] } ] }", + "RANK a:A b:B"); + assertParse("{ \"rank\": [{ \"contains\": [\"a\", \"A\"] }, { \"contains\": [\"b\", \"B\"] }, { \"contains\": [\"c\", \"C\"] } ] }", + "RANK a:A b:B c:C"); + assertParse("{ \"rank\": [{ \"contains\": [\"a\", \"A\"] }, { \"or\": [{ \"contains\": [\"b\", \"B\"] }, { \"contains\": [\"c\", \"C\"] }] }] }", + "RANK a:A (OR b:B c:C)"); + } + + @Test + public void testWeakAnd() { + assertParse("{ \"weakAnd\": [{ \"contains\": [\"a\", \"A\"] }, { \"contains\": [\"b\", \"B\"] } ] }", + "WAND(100) a:A b:B"); + assertParse("{ \"weakAnd\": { \"children\" : [{ \"contains\": [\"a\", \"A\"] }, { \"contains\": [\"b\", \"B\"] } ], \"attributes\" : {\"targetNumHits\": 37} }}", + "WAND(37) a:A b:B"); + + QueryTree tree = parseWhere("{ \"weakAnd\": { \"children\" : [{ \"contains\": [\"a\", \"A\"] }, { \"contains\": [\"b\", \"B\"] } ], \"attributes\" : {\"scoreThreshold\": 41}}}"); + assertEquals("WAND(100) a:A b:B", tree.toString()); + assertEquals(WeakAndItem.class, tree.getRoot().getClass()); + assertEquals(41, ((WeakAndItem)tree.getRoot()).getScoreThreshold()); + } + + @Test + public void testEquiv() { + assertParse("{ \"contains\" : [\"fieldName\", {\"equiv\" : [\"A\",\"B\"]}]}", + "EQUIV fieldName:A fieldName:B"); + + assertParse("{ \"contains\" : [\"fieldName\", {\"equiv\" : [\"ny\",{\"phrase\" : [ \"new\",\"york\" ] } ] } ] }", + "EQUIV fieldName:ny fieldName:\"new york\""); + + assertParseFail("{ \"contains\" : [\"fieldName\", {\"equiv\" : [\"ny\"] } ] }", + new IllegalArgumentException("Expected 2 or more arguments, got 1.")); + assertParseFail("{ \"contains\" : [\"fieldName\", {\"equiv\" : [\"ny\",{\"nalle\" : [ \"void\" ] } ] } ] }", + new IllegalArgumentException("Expected operator phrase, got nalle.")); + assertParseFail("{ \"contains\" : [\"fieldName\", {\"equiv\" : [\"ny\", 42]}]}", + new IllegalArgumentException("Word item word can not be empty")); + } + + @Test + public void testAffixItems() { + assertRootClass("{ \"contains\" : { \"children\" : [\"baz\", \"colors\"], \"attributes\" : {\"suffix\": true} } }", + SuffixItem.class); + + + assertRootClass("{ \"contains\" : { \"children\" : [\"baz\", \"colors\"], \"attributes\" : {\"prefix\": true} } }", + PrefixItem.class); + assertRootClass("{ \"contains\" : { \"children\" : [\"baz\", \"colors\"], \"attributes\" : {\"substring\": true} } }", + SubstringItem.class); + assertParseFail("{ \"contains\" : { \"children\" : [\"baz\", \"colors\"], \"attributes\" : {\"suffix\": true, \"prefix\" : true} } }", + new IllegalArgumentException("Only one of prefix, substring and suffix can be set.")); + assertParseFail("{ \"contains\" : { \"children\" : [\"baz\", \"colors\"], \"attributes\" : {\"suffix\": true, \"substring\" : true} } }", + new IllegalArgumentException("Only one of prefix, substring and suffix can be set.")); + } + + @Test + public void testLongNumberInSimpleExpression() { + assertParse("{ \"range\" : [ \"price\", { \"=\" : 8589934592 }]}", + "price:8589934592"); + } + + @Test + public void testNegativeLongNumberInSimpleExpression() { + assertParse("{ \"range\" : [ \"price\", { \"=\" : -8589934592 }]}", + "price:-8589934592"); + } + + @Test + public void testNegativeHitLimit() { + assertParse( + "{ \"range\" : { \"children\":[ \"foo\", { \">=\" : 0, \"<=\" : 1 }], \"attributes\" : {\"hitLimit\": -38 } } }", + "foo:[0;1;-38]"); + } + + @Test + public void testRangeSearchHitPopulationOrdering() { + assertParse("{ \"range\" : { \"children\":[ \"foo\", { \">=\" : 0, \"<=\" : 1 }], \"attributes\" : {\"hitLimit\": 38 ,\"ascending\": true} } }", "foo:[0;1;38]"); + assertParse("{ \"range\" : { \"children\":[ \"foo\", { \">=\" : 0, \"<=\" : 1 }], \"attributes\" : {\"hitLimit\": 38 ,\"ascending\": false} } }", "foo:[0;1;-38]"); + assertParse("{ \"range\" : { \"children\":[ \"foo\", { \">=\" : 0, \"<=\" : 1 }], \"attributes\" : {\"hitLimit\": 38 ,\"descending\": true} } }", "foo:[0;1;-38]"); + assertParse("{ \"range\" : { \"children\":[ \"foo\", { \">=\" : 0, \"<=\" : 1 }], \"attributes\" : {\"hitLimit\": 38 ,\"descending\": false} } }", "foo:[0;1;38]"); + + boolean gotExceptionFromParse = false; + try { + parseWhere("{ \"range\" : { \"children\":[ \"foo\", { \">=\" : 0, \"<=\" : 1 }], \"attributes\" : {\"hitLimit\": 38, \"ascending\": true, \"descending\": false} } }"); + } catch (IllegalArgumentException e) { + assertTrue("Expected information about abuse of settings.", + e.getMessage().contains("both ascending and descending ordering set")); + gotExceptionFromParse = true; + } + assertTrue(gotExceptionFromParse); + } + + // NB: Uses operator-keys to set bounds, not annotations + @Test + public void testOpenIntervals() { + assertParse("{ \"range\" : { \"children\":[ \"title\", { \">=\" : 0.0, \"<=\" : 500.0 }] } }" + + "select * from sources * where range(title, 0.0, 500.0);", + "title:[0.0;500.0]"); + assertParse( + "{ \"range\" : { \"children\":[ \"title\", { \">\" : 0.0, \"<\" : 500.0 }] } }", + "title:<0.0;500.0>"); + assertParse( + "{ \"range\" : { \"children\":[ \"title\", { \">\" : 0.0, \"<=\" : 500.0 }] } }", + "title:<0.0;500.0]"); + assertParse( + "{ \"range\" : { \"children\":[ \"title\", { \">=\" : 0.0, \"<\" : 500.0 }] } }", + "title:[0.0;500.0>"); + } + + @Test + public void testRegexp() { + QueryTree x = parseWhere("{ \"matches\" : [\"foo\", \"a b\"]}"); + Item root = x.getRoot(); + assertSame(RegExpItem.class, root.getClass()); + assertEquals("a b", ((RegExpItem) root).stringValue()); + } + + @Test + public void testWordAlternatives() { + QueryTree x = parseWhere("{\"contains\" : [\"foo\", {\"alternatives\" : [{\"trees\": 1.0, \"tree\": 0.7}]}]}"); + Item root = x.getRoot(); + assertSame(WordAlternativesItem.class, root.getClass()); + WordAlternativesItem alternatives = (WordAlternativesItem) root; + checkWordAlternativesContent(alternatives); + } + + /** GROUPING TESTS */ + + @Test + public void testGrouping(){ + String grouping = "[ { \"all\" : { \"group\" : \"time.year(a)\", \"each\" : { \"output\" : \"count()\" } } } ]"; + String expected = "[[]all(group(time.year(a)) each(output(count())))]"; + assertGrouping(expected, parseGrouping(grouping)); + } + + + @Test + public void testMultipleGroupings() { + String grouping = "[ { \"all\" : { \"group\" : \"a\", \"each\" : { \"output\" : \"count()\"}}}, { \"all\" : { \"group\" : \"b\", \"each\" : { \"output\" : \"count()\"}}} ]"; + String expected = "[[]all(group(a) each(output(count()))), []all(group(b) each(output(count())))]"; + + assertGrouping(expected, parseGrouping(grouping)); + } + + + + /** OTHER TESTS */ + + @Test + public void testOverridingOtherQueryTree() { + Query query = new Query("?query=default:query"); + assertEquals("default:query", query.getModel().getQueryTree().toString()); + assertEquals(Query.Type.ALL, query.getModel().getType()); + + query.getSelect().setWhere("{\"contains\" : [\"default\", \"select\"] }"); + assertEquals("default:select", query.getModel().getQueryTree().toString()); + assertEquals(Query.Type.SELECT, query.getModel().getType()); + } + + + @Test + public void testOverridingWhereQueryTree() { + Query query = new Query(); + query.getSelect().setWhere("{\"contains\" : [\"default\", \"select\"] }"); + assertEquals("default:select", query.getModel().getQueryTree().toString()); + assertEquals(Query.Type.SELECT, query.getModel().getType()); + + query.getModel().setQueryString("default:query"); + query.getModel().setType("all"); + assertEquals("default:query", query.getModel().getQueryTree().toString()); + assertEquals(Query.Type.ALL, query.getModel().getType()); + } + + + + + /** Assert-methods */ + private void assertParse(String where, String expectedQueryTree) { + String queryTree = parseWhere(where).toString(); + assertEquals(expectedQueryTree, queryTree); + } + + private void assertParseFail(String where, Throwable expectedException) { + try { + parseWhere(where).toString(); + } catch (Throwable t) { + assertEquals(expectedException.getClass(), t.getClass()); + assertEquals(expectedException.getMessage(), t.getMessage()); + return; + } + fail("Parse succeeded: " + where); + } + + private void assertRootClass(String where, Class<? extends Item> expectedRootClass) { + assertEquals(expectedRootClass, parseWhere(where).getRoot().getClass()); + } + + private void assertGrouping(String expected, List<VespaGroupingStep> steps) { + List<String> actual = new ArrayList<>(steps.size()); + for (VespaGroupingStep step : steps) { + actual.add(step.continuations().toString() + + step.getOperation()); + } + assertEquals(expected, actual.toString()); + } + + + + + /** Parse-methods*/ + + private QueryTree parseWhere(String where) { + Select select = new Select(where, ""); + + return parser.parse(new Parsable().setSelect(select)); + } + + private List<VespaGroupingStep> parseGrouping(String grouping) { + + return parser.getGroupingSteps(grouping); + } + + private QueryTree parse(String where, String grouping) { + Select select = new Select(where, grouping); + + return parser.parse(new Parsable().setSelect(select)); + } + + + + + + /** Other methods */ + private WordItem getRootWord(String yqlQuery) { + Item root = parseWhere(yqlQuery).getRoot(); + assertTrue(root instanceof WordItem); + return (WordItem)root; + } + + private void checkWordAlternativesContent(WordAlternativesItem alternatives) { + boolean seenTree = false; + boolean seenForest = false; + final String forest = "trees"; + final String tree = "tree"; + assertEquals(2, alternatives.getAlternatives().size()); + for (WordAlternativesItem.Alternative alternative : alternatives.getAlternatives()) { + if (tree.equals(alternative.word)) { + assertFalse("Duplicate term introduced", seenTree); + seenTree = true; + assertEquals(.7d, alternative.exactness, 1e-15d); + } else if (forest.equals(alternative.word)) { + assertFalse("Duplicate term introduced", seenForest); + seenForest = true; + assertEquals(1.0d, alternative.exactness, 1e-15d); + } else { + fail("Unexpected term: " + alternative.word); + } + } + } + + +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/BuildService.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/BuildService.java index e91a5909f80..56c2ee8da6b 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/BuildService.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/BuildService.java @@ -20,11 +20,6 @@ public interface BuildService { */ JobState stateOf(BuildJob buildJob); - /** - * Returns whether the given build job should be performed by this build service. - */ - default boolean builds(BuildJob buildJob) { return true; } - enum JobState { /** Job is not running, and may be triggered. */ diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/Testers.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/TesterCloud.java index dccc0e47ceb..f1d07fc9097 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/Testers.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/TesterCloud.java @@ -7,7 +7,7 @@ import java.net.URI; * * @author jonmv */ -public interface Testers { +public interface TesterCloud { /** Signals the tester to run its tests. */ void startTests(URI testerUrl, Suite suite, byte[] config); diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/MockTesters.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/MockTesterCloud.java index 021e4d7f293..c2199c284f3 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/MockTesters.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/MockTesterCloud.java @@ -1,16 +1,14 @@ package com.yahoo.vespa.hosted.controller.api.integration.stubs; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.Testers; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterCloud; import java.net.URI; import java.util.Arrays; -import static com.yahoo.vespa.hosted.controller.api.integration.deployment.Testers.Status.FAILURE; -import static com.yahoo.vespa.hosted.controller.api.integration.deployment.Testers.Status.NOT_STARTED; -import static com.yahoo.vespa.hosted.controller.api.integration.deployment.Testers.Status.RUNNING; -import static com.yahoo.vespa.hosted.controller.api.integration.deployment.Testers.Status.SUCCESS; +import static com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterCloud.Status.NOT_STARTED; +import static com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterCloud.Status.RUNNING; -public class MockTesters implements Testers { +public class MockTesterCloud implements TesterCloud { private byte[] logs = new byte[0]; private Status status = NOT_STARTED; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java index 790d6d00035..f8be9f55b84 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java @@ -26,8 +26,6 @@ import com.yahoo.vespa.hosted.controller.api.integration.routing.GlobalRoutingSe import com.yahoo.vespa.hosted.controller.api.integration.routing.RotationStatus; import com.yahoo.vespa.hosted.controller.api.integration.routing.RoutingGenerator; import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneRegistry; -import com.yahoo.vespa.hosted.controller.deployment.DelegatingBuildService; -import com.yahoo.vespa.hosted.controller.deployment.InternalBuildService; import com.yahoo.vespa.hosted.controller.deployment.JobController; import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; import com.yahoo.vespa.hosted.controller.rotation.Rotation; @@ -126,8 +124,7 @@ public class Controller extends AbstractComponent { configServer, Objects.requireNonNull(artifactRepository, "ArtifactRepository cannot be null"), Objects.requireNonNull(routingGenerator, "RoutingGenerator cannot be null"), - new DelegatingBuildService(Objects.requireNonNull(buildService, "BuildService cannot be null"), - new InternalBuildService(jobController)), + Objects.requireNonNull(buildService, "BuildService cannot be null"), clock); tenantController = new TenantController(this, curator, athenzClientFactory); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DelegatingBuildService.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DelegatingBuildService.java deleted file mode 100644 index d2159841c9d..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DelegatingBuildService.java +++ /dev/null @@ -1,30 +0,0 @@ -package com.yahoo.vespa.hosted.controller.deployment; - -import com.yahoo.vespa.hosted.controller.api.integration.BuildService; - -/** - * Sends build jobs to an internal build system whenever it accepts them, or to an external one otherwise. - * - * @author jonmv - */ -public class DelegatingBuildService implements BuildService { - - private final BuildService external; - private final BuildService internal; - - public DelegatingBuildService(BuildService external, BuildService internal) { - this.external = external; - this.internal = internal; - } - - @Override - public void trigger(BuildJob buildJob) { - (internal.builds(buildJob) ? internal : external).trigger(buildJob); - } - - @Override - public JobState stateOf(BuildJob buildJob) { - return (internal.builds(buildJob) ? internal : external).stateOf(buildJob); - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java index e3b4b4cef8c..72bc1c61eae 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java @@ -69,11 +69,13 @@ public class DeploymentTrigger { private final Controller controller; private final Clock clock; private final BuildService buildService; + private final JobController jobs; public DeploymentTrigger(Controller controller, BuildService buildService, Clock clock) { this.controller = Objects.requireNonNull(controller, "controller cannot be null"); - this.buildService = Objects.requireNonNull(buildService, "buildService cannot be null"); this.clock = Objects.requireNonNull(clock, "clock cannot be null"); + this.buildService = Objects.requireNonNull(buildService, "buildService cannot be null"); + this.jobs = controller.jobController(); } public DeploymentSteps steps(DeploymentSpec spec) { @@ -161,14 +163,22 @@ public class DeploymentTrigger { * Attempts to trigger the given job for the given application and returns the outcome. * * If the build service can not find the given job, or claims it is illegal to trigger it, - * the project id is removed from the application owning the job, to prevent further trigger attemps. + * the project id is removed from the application owning the job, to prevent further trigger attempts. */ public boolean trigger(Job job) { log.log(LogLevel.INFO, String.format("Triggering %s: %s", job, job.triggering)); try { - buildService.trigger(job); - applications().lockOrThrow(job.applicationId(), application -> - applications().store(application.withJobTriggering(job.jobType, job.triggering))); + applications().lockOrThrow(job.applicationId(), application -> { + if (application.get().deploymentJobs().builtInternally()) + jobs.start(job.applicationId(), job.jobType, new Versions(job.triggering.platform(), + job.triggering.application(), + job.triggering.sourcePlatform(), + job.triggering.sourceApplication())); + else + buildService.trigger(job); + + applications().store(application.withJobTriggering(job.jobType, job.triggering)); + }); return true; } catch (RuntimeException e) { @@ -184,6 +194,9 @@ public class DeploymentTrigger { public List<JobType> forceTrigger(ApplicationId applicationId, JobType jobType, String user) { Application application = applications().require(applicationId); if (jobType == component) { + if (application.deploymentJobs().builtInternally()) + throw new IllegalArgumentException(applicationId + " has no component job we can trigger."); + buildService.trigger(BuildJob.of(applicationId, application.deploymentJobs().projectId().getAsLong(), jobType.jobName())); return singletonList(component); } @@ -339,6 +352,7 @@ public class DeploymentTrigger { if (!jobStatus.get().lastCompleted().isPresent()) return true; // Never completed if (!jobStatus.get().firstFailing().isPresent()) return true; // Should not happen as firstFailing should be set for an unsuccessful job if (!versions.targetsMatch(jobStatus.get().lastCompleted().get())) return true; // Always trigger as targets have changed + if (application.deploymentSpec().upgradePolicy() == DeploymentSpec.UpgradePolicy.canary) return true; // Don't throttle canaries Instant firstFailing = jobStatus.get().firstFailing().get().at(); Instant lastCompleted = jobStatus.get().lastCompleted().get().at(); @@ -376,6 +390,10 @@ public class DeploymentTrigger { } private JobState jobStateOf(Application application, JobType jobType) { + if (application.deploymentJobs().builtInternally()) { + Optional<RunStatus> run = controller.jobController().last(application.id(), jobType); + return run.isPresent() && ! run.get().hasEnded() ? JobState.running : JobState.idle; + } return buildService.stateOf(BuildJob.of(application.id(), application.deploymentJobs().projectId().getAsLong(), jobType.jobName())); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalBuildService.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalBuildService.java deleted file mode 100644 index 18b62f5ea0f..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalBuildService.java +++ /dev/null @@ -1,39 +0,0 @@ -package com.yahoo.vespa.hosted.controller.deployment; - -import com.yahoo.vespa.hosted.controller.api.integration.BuildService; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; - -import java.util.Optional; - -/** - * Wraps a JobController as a BuildService. - * - * Shall be inlined when the {@link DelegatingBuildService} delegates all jobs to it. - * - * @author jonmv - */ -public class InternalBuildService implements BuildService { - - private final JobController jobs; - - public InternalBuildService(JobController jobs) { - this.jobs = jobs; - } - - @Override - public void trigger(BuildJob buildJob) { - jobs.start(buildJob.applicationId(), JobType.fromJobName(buildJob.jobName())); - } - - @Override - public JobState stateOf(BuildJob buildJob) { - Optional<RunStatus> run = jobs.last(buildJob.applicationId(), JobType.fromJobName(buildJob.jobName())); - return run.isPresent() && ! run.get().hasEnded() ? JobState.running : JobState.idle; - } - - @Override - public boolean builds(BuildJob buildJob) { - return jobs.builds(buildJob.applicationId()); - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java index c5bcffd7ffe..5c848ba279b 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java @@ -20,7 +20,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.configserver.PrepareRes import com.yahoo.vespa.hosted.controller.api.integration.configserver.ServiceConvergence; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.Testers; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterCloud; import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; import com.yahoo.vespa.hosted.controller.application.ApplicationVersion; @@ -84,12 +84,12 @@ public class InternalStepRunner implements StepRunner { } private final Controller controller; - private final Testers testers; + private final TesterCloud testerCloud; private final ThreadLocal<ByteArrayLogger> logger = new ThreadLocal<>(); - public InternalStepRunner(Controller controller, Testers testers) { + public InternalStepRunner(Controller controller, TesterCloud testerCloud) { this.controller = controller; - this.testers = testers; + this.testerCloud = testerCloud; } @Override @@ -301,9 +301,9 @@ public class InternalStepRunner implements StepRunner { Optional<URI> testerEndpoint = testerEndpoint(id); if (testerEndpoint.isPresent()) { logger.get().log("Starting tests ..."); - testers.startTests(testerEndpoint.get(), - Testers.Suite.of(id.type()), - testConfig(id.application(), zone(id.type()), controller.system(), endpoints)); + testerCloud.startTests(testerEndpoint.get(), + TesterCloud.Suite.of(id.type()), + testConfig(id.application(), zone(id.type()), controller.system(), endpoints)); return succeeded; } @@ -321,7 +321,7 @@ public class InternalStepRunner implements StepRunner { .orElseThrow(() -> new NoSuchElementException("Endpoint for tester vanished again before tests were complete!")); Status status; - switch (testers.getStatus(testerEndpoint)) { + switch (testerCloud.getStatus(testerEndpoint)) { case NOT_STARTED: throw new IllegalStateException("Tester reports tests not started, even though they should have!"); case RUNNING: @@ -339,7 +339,7 @@ public class InternalStepRunner implements StepRunner { default: throw new AssertionError("Unknown status!"); } - logger.get().log(new String(testers.getLogs(testerEndpoint))); // TODO jvenstad: Replace with something less hopeless! + logger.get().log(new String(testerCloud.getLogs(testerEndpoint))); // TODO jvenstad: Replace with something less hopeless! return status; } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java index ba79364fa34..06af36a2e26 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java @@ -1,6 +1,7 @@ package com.yahoo.vespa.hosted.controller.deployment; import com.google.common.collect.ImmutableMap; +import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; import com.yahoo.vespa.curator.Lock; import com.yahoo.vespa.hosted.controller.Application; @@ -195,7 +196,7 @@ public class JobController { } /** Orders a run of the given type, or throws an IllegalStateException if that job type is already running. */ - public void start(ApplicationId id, JobType type) { + public void start(ApplicationId id, JobType type, Versions versions) { controller.applications().lockIfPresent(id, application -> { if ( ! application.get().deploymentJobs().builtInternally()) throw new IllegalArgumentException(id + " is not built here!"); @@ -206,7 +207,7 @@ public class JobController { throw new IllegalStateException("Can not start " + type + " for " + id + "; it is already running!"); RunId newId = new RunId(id, type, last.map(run -> run.id().number()).orElse(0L) + 1); - curator.writeLastRun(RunStatus.initial(newId, controller.clock().instant())); + curator.writeLastRun(RunStatus.initial(newId, versions, controller.clock().instant())); }); }); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunResult.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunResult.java index aaf43097908..c4aac48503f 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunResult.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunResult.java @@ -1,7 +1,7 @@ package com.yahoo.vespa.hosted.controller.deployment; /** - * Outcomes of jobs run by an {@link InternalBuildService}. + * Outcomes of jobs run by a {@link JobController}. * * @author jonmv */ diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunStatus.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunStatus.java index 1fd32524c88..c30b1aee7f1 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunStatus.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunStatus.java @@ -1,7 +1,9 @@ package com.yahoo.vespa.hosted.controller.deployment; import com.google.common.collect.ImmutableList; +import com.yahoo.component.Version; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; +import com.yahoo.vespa.hosted.controller.application.ApplicationVersion; import java.time.Instant; import java.util.Collections; @@ -16,7 +18,7 @@ import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.unfinishe import static java.util.Objects.requireNonNull; /** - * Immutable class containing status information for a deployment job run by an {@link InternalBuildService}. + * Immutable class containing status information for a deployment job run by a {@link JobController}. * * @author jonmv */ @@ -24,24 +26,26 @@ public class RunStatus { private final RunId id; private final Map<Step, Step.Status> steps; + private final Versions versions; private final Instant start; private final Optional<Instant> end; private final boolean aborted; - // TODO jvenstad: Add a Versions object and a reason String. Requires shortcutting of triggering of these runs. // For deserialisation only -- do not use! - public RunStatus(RunId id, Map<Step, Step.Status> steps, Instant start, Optional<Instant> end, boolean aborted) { + public RunStatus(RunId id, Map<Step, Step.Status> steps, Versions versions, + Instant start, Optional<Instant> end, boolean aborted) { this.id = id; this.steps = Collections.unmodifiableMap(new EnumMap<>(steps)); + this.versions = versions; this.start = start; this.end = end; this.aborted = aborted; } - public static RunStatus initial(RunId id, Instant now) { + public static RunStatus initial(RunId id, Versions versions, Instant now) { EnumMap<Step, Step.Status> steps = new EnumMap<>(Step.class); JobProfile.of(id.type()).steps().forEach(step -> steps.put(step, unfinished)); - return new RunStatus(id, steps, requireNonNull(now), Optional.empty(), false); + return new RunStatus(id, steps, requireNonNull(versions), requireNonNull(now), Optional.empty(), false); } public RunStatus with(Step.Status status, LockedStep step) { @@ -50,21 +54,21 @@ public class RunStatus { EnumMap<Step, Step.Status> steps = new EnumMap<>(this.steps); steps.put(step.get(), requireNonNull(status)); - return new RunStatus(id, steps, start, end, aborted); + return new RunStatus(id, steps, versions, start, end, aborted); } public RunStatus finished(Instant now) { if (hasEnded()) throw new AssertionError("This step ended at " + end.get() + " -- it can't be ended again!"); - return new RunStatus(id, new EnumMap<>(steps), start, Optional.of(now), aborted); + return new RunStatus(id, new EnumMap<>(steps), versions, start, Optional.of(now), aborted); } public RunStatus aborted() { if (hasEnded()) throw new AssertionError("This step ended at " + end.get() + " -- it can't be aborted now!"); - return new RunStatus(id, new EnumMap<>(steps), start, end, true); + return new RunStatus(id, new EnumMap<>(steps), versions, start, end, true); } /** Returns the id of this run. */ @@ -83,7 +87,7 @@ public class RunStatus { public Optional<RunResult> result() { // No result of not finished yet - if (!hasEnded()) return Optional.empty(); + if ( ! hasEnded()) return Optional.empty(); // If any steps has failed - then we need to figure out what - for now return fixed error result if (hasFailed()) return Optional.of(RunResult.testError); @@ -116,6 +120,11 @@ public class RunStatus { return end.isPresent(); } + /** Returns the target, and possibly source, versions for this run. */ + public Versions versions() { + return versions; + } + @Override public boolean equals(Object o) { if (this == o) return true; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Step.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Step.java index e1e2281c5ea..f734dde9440 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Step.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Step.java @@ -66,10 +66,6 @@ public enum Step { public List<Step> prerequisites() { return prerequisites; } - public static Step last() { - return report; - } - public enum Status { /** Step still has unsatisfied finish criteria -- it may not even have started. */ diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Versions.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Versions.java index bf58bac177c..e0f7a955a80 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Versions.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Versions.java @@ -9,8 +9,11 @@ import com.yahoo.vespa.hosted.controller.application.Change; import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.vespa.hosted.controller.application.JobStatus; +import java.util.Objects; import java.util.Optional; +import static java.util.Objects.requireNonNull; + /** * Source and target versions for an application. * @@ -26,10 +29,13 @@ public class Versions { public Versions(Version targetPlatform, ApplicationVersion targetApplication, Optional<Version> sourcePlatform, Optional<ApplicationVersion> sourceApplication) { - this.targetPlatform = targetPlatform; - this.targetApplication = targetApplication; - this.sourcePlatform = sourcePlatform; - this.sourceApplication = sourceApplication; + if (sourcePlatform.isPresent() ^ sourceApplication.isPresent()) + throw new IllegalArgumentException("Sources must both be present or absent."); + + this.targetPlatform = requireNonNull(targetPlatform); + this.targetApplication = requireNonNull(targetApplication); + this.sourcePlatform = requireNonNull(sourcePlatform); + this.sourceApplication = requireNonNull(sourceApplication); } /** Target platform version for this */ diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java index 7fa16a02649..e663b0154d7 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java @@ -4,13 +4,12 @@ package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.component.AbstractComponent; import com.yahoo.jdisc.Metric; import com.yahoo.vespa.hosted.controller.Controller; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.Testers; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterCloud; import com.yahoo.vespa.hosted.controller.api.integration.dns.NameService; import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeRepositoryClientInterface; import com.yahoo.vespa.hosted.controller.api.integration.organization.OwnershipIssues; import com.yahoo.vespa.hosted.controller.api.integration.organization.DeploymentIssues; import com.yahoo.vespa.hosted.controller.api.integration.chef.Chef; -import com.yahoo.vespa.hosted.controller.deployment.DummyStepRunner; import com.yahoo.vespa.hosted.controller.deployment.InternalStepRunner; import com.yahoo.vespa.hosted.controller.maintenance.config.MaintainerConfig; import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; @@ -45,7 +44,7 @@ public class ControllerMaintenance extends AbstractComponent { @SuppressWarnings("unused") // instantiated by Dependency Injection public ControllerMaintenance(MaintainerConfig maintainerConfig, Controller controller, CuratorDb curator, - JobControl jobControl, Metric metric, Chef chefClient, Testers testers, + JobControl jobControl, Metric metric, Chef chefClient, TesterCloud testerCloud, DeploymentIssues deploymentIssues, OwnershipIssues ownershipIssues, NameService nameService, NodeRepositoryClientInterface nodeRepositoryClient) { Duration maintenanceInterval = Duration.ofMinutes(maintainerConfig.intervalMinutes()); @@ -63,7 +62,7 @@ public class ControllerMaintenance extends AbstractComponent { applicationOwnershipConfirmer = new ApplicationOwnershipConfirmer(controller, Duration.ofHours(12), jobControl, ownershipIssues); dnsMaintainer = new DnsMaintainer(controller, Duration.ofHours(12), jobControl, nameService); systemUpgrader = new SystemUpgrader(controller, Duration.ofMinutes(1), jobControl); - jobRunner = new JobRunner(controller, Duration.ofSeconds(30), jobControl, new InternalStepRunner(controller, testers)); + jobRunner = new JobRunner(controller, Duration.ofSeconds(30), jobControl, new InternalStepRunner(controller, testerCloud)); } public Upgrader upgrader() { return upgrader; } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunner.java index 7dbf1a2c05e..9fa67cb5f89 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunner.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunner.java @@ -2,7 +2,6 @@ package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.log.LogLevel; import com.yahoo.vespa.hosted.controller.Controller; -import com.yahoo.vespa.hosted.controller.deployment.InternalBuildService; import com.yahoo.vespa.hosted.controller.deployment.JobController; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; import com.yahoo.vespa.hosted.controller.deployment.RunStatus; @@ -20,9 +19,8 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.logging.Logger; /** - * Advances the set of {@link RunStatus}es for an {@link InternalBuildService}. + * Advances the set of {@link RunStatus}es for a {@link JobController}. * - * @see JobController * @author jonmv */ public class JobRunner extends Maintainer { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java index 7df60278390..153f4e327a0 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java @@ -1,5 +1,6 @@ package com.yahoo.vespa.hosted.controller.persistence; +import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; import com.yahoo.slime.ArrayTraverser; import com.yahoo.slime.Cursor; @@ -8,9 +9,12 @@ import com.yahoo.slime.ObjectTraverser; import com.yahoo.slime.Slime; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; +import com.yahoo.vespa.hosted.controller.application.ApplicationVersion; +import com.yahoo.vespa.hosted.controller.application.SourceRevision; import com.yahoo.vespa.hosted.controller.deployment.RunStatus; import com.yahoo.vespa.hosted.controller.deployment.Step; import com.yahoo.vespa.hosted.controller.deployment.Step.Status; +import com.yahoo.vespa.hosted.controller.deployment.Versions; import java.time.Instant; import java.util.EnumMap; @@ -47,6 +51,13 @@ public class RunSerializer { private static final String startField = "start"; private static final String endField = "end"; private static final String abortedField = "aborted"; + private static final String versionsField = "versions"; + private static final String platformVersionField = "platform"; + private static final String repositoryField = "repository"; + private static final String branchField = "branch"; + private static final String commitField = "commit"; + private static final String buildField = "build"; + private static final String sourceField = "source"; RunStatus runFromSlime(Slime slime) { return runFromSlime(slime.get()); @@ -71,6 +82,7 @@ public class RunSerializer { JobType.fromJobName(runObject.field(jobTypeField).asString()), runObject.field(numberField).asLong()), steps, + versionsFromSlime(runObject.field(versionsField)), Instant.ofEpochMilli(runObject.field(startField).asLong()), Optional.of(runObject.field(endField)) .filter(Inspector::valid) @@ -78,6 +90,25 @@ public class RunSerializer { runObject.field(abortedField).asBool()); } + private Versions versionsFromSlime(Inspector versionsObject) { + Version targetPlatformVersion = Version.fromString(versionsObject.field(platformVersionField).asString()); + ApplicationVersion targetApplicationVersion = ApplicationVersion.from(new SourceRevision(versionsObject.field(repositoryField).asString(), + versionsObject.field(branchField).asString(), + versionsObject.field(commitField).asString()), + versionsObject.field(buildField).asLong()); + Optional<Version> sourcePlatformVersion = versionsObject.field(sourceField).valid() + ? Optional.of(Version.fromString(versionsObject.field(sourceField).field(platformVersionField).asString())) + : Optional.empty(); + Optional<ApplicationVersion> sourceApplicationVersion = versionsObject.field(sourceField).valid() + ? Optional.of(ApplicationVersion.from(new SourceRevision(versionsObject.field(repositoryField).asString(), + versionsObject.field(branchField).asString(), + versionsObject.field(commitField).asString()), + versionsObject.field(buildField).asLong())) + : Optional.empty(); + + return new Versions(targetPlatformVersion, targetApplicationVersion, sourcePlatformVersion, sourceApplicationVersion); + } + Slime toSlime(Iterable<RunStatus> runs) { Slime slime = new Slime(); Cursor runArray = slime.setArray(); @@ -98,59 +129,84 @@ public class RunSerializer { runObject.setLong(startField, run.start().toEpochMilli()); run.end().ifPresent(end -> runObject.setLong(endField, end.toEpochMilli())); if (run.isAborted()) runObject.setBool(abortedField, true); + Cursor stepsObject = runObject.setObject(stepsField); run.steps().forEach((step, status) -> stepsObject.setString(valueOf(step), valueOf(status))); + + Cursor versionsObject = runObject.setObject(versionsField); + toSlime(run.versions().targetPlatform(), run.versions().targetApplication(), versionsObject); + run.versions().sourcePlatform().ifPresent(sourcePlatformVersion -> { + toSlime(sourcePlatformVersion, + run.versions().sourceApplication() + .orElseThrow(() -> new IllegalArgumentException("Source versions must be both present or absent.")), + versionsObject.setObject(sourceField)); + }); + } + + private void toSlime(Version platformVersion, ApplicationVersion applicationVersion, Cursor versionsObject) { + versionsObject.setString(platformVersionField, platformVersion.toString()); + SourceRevision targetSourceRevision = applicationVersion.source() + .orElseThrow(() -> new IllegalArgumentException("Source revision must be present in target application version.")); + versionsObject.setString(repositoryField, targetSourceRevision.repository()); + versionsObject.setString(branchField, targetSourceRevision.branch()); + versionsObject.setString(commitField, targetSourceRevision.commit()); + versionsObject.setLong(buildField, applicationVersion.buildNumber() + .orElseThrow(() -> new IllegalArgumentException("Build number must be present in target application version."))); } static String valueOf(Step step) { switch (step) { - case deployInitialReal : return "DIR"; - case installInitialReal : return "IIR"; - case deployReal : return "DR" ; - case installReal : return "IR" ; - case deactivateReal : return "DAR"; - case deployTester : return "DT" ; - case installTester : return "IT" ; - case deactivateTester : return "DAT"; - case startTests : return "ST" ; - case endTests : return "ET" ; - case report : return "R" ; + case deployInitialReal : return "deployInitialReal"; + case installInitialReal : return "installInitialReal"; + case deployReal : return "deployReal"; + case installReal : return "installReal"; + case deactivateReal : return "deactivateReal"; + case deployTester : return "deployTester"; + case installTester : return "installTester"; + case deactivateTester : return "deactivateTester"; + case startTests : return "startTests"; + case endTests : return "endTests"; + case report : return "report"; + default : throw new AssertionError("No value defined for '" + step + "'!"); } } static Step stepOf(String step) { switch (step) { - case "DIR" : return deployInitialReal ; - case "IIR" : return installInitialReal; - case "DR" : return deployReal ; - case "IR" : return installReal ; - case "DAR" : return deactivateReal ; - case "DT" : return deployTester ; - case "IT" : return installTester ; - case "DAT" : return deactivateTester ; - case "ST" : return startTests ; - case "ET" : return endTests ; - case "R" : return report ; - default : throw new IllegalArgumentException("No step defined by '" + step + "'!"); + case "deployInitialReal" : return deployInitialReal; + case "installInitialReal" : return installInitialReal; + case "deployReal" : return deployReal; + case "installReal" : return installReal; + case "deactivateReal" : return deactivateReal; + case "deployTester" : return deployTester; + case "installTester" : return installTester; + case "deactivateTester" : return deactivateTester; + case "startTests" : return startTests; + case "endTests" : return endTests; + case "report" : return report; + + default : throw new IllegalArgumentException("No step defined by '" + step + "'!"); } } static String valueOf(Status status) { switch (status) { - case unfinished : return "U"; - case failed : return "F"; - case succeeded : return "S"; - default : throw new AssertionError("No value defined for '" + status + "'!"); + case unfinished : return "unfinished"; + case failed : return "failed"; + case succeeded : return "succeeded"; + + default : throw new AssertionError("No value defined for '" + status + "'!"); } } static Status statusOf(String status) { switch (status) { - case "U" : return unfinished; - case "F" : return failed ; - case "S" : return succeeded ; - default : throw new IllegalArgumentException("No status defined by '" + status + "'!"); + case "unfinished" : return unfinished; + case "failed" : return failed; + case "succeeded" : return succeeded; + + default : throw new IllegalArgumentException("No status defined by '" + status + "'!"); } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java index af7261149ad..95eb9117c5a 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java @@ -480,7 +480,6 @@ public class DeploymentTriggerTest { .environment(Environment.prod) .region("us-central-1") .region("eu-west-1") - .upgradePolicy("canary") .build(); tester.deployCompletely(application, applicationPackage); @@ -497,7 +496,7 @@ public class DeploymentTriggerTest { tester.clock().advance(Duration.ofHours(1)); Version v1 = new Version("7.1"); - tester.upgradeSystem(v1); // Downgrade, but it works, since the app is a canary. + tester.upgradeSystem(v1); // Downgrade to cut down on the amount of code. assertEquals(Change.of(v1), app.get().change()); // 7.1 proceeds 'til the last job, where it fails; us-central-1 is skipped, as current change is strictly dominated by what's deployed there. diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java index e084e9aa46d..e5416672bb7 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java @@ -1,8 +1,10 @@ package com.yahoo.vespa.hosted.controller.maintenance; +import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; import com.yahoo.vespa.hosted.controller.TestIdentities; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; +import com.yahoo.vespa.hosted.controller.application.ApplicationVersion; import com.yahoo.vespa.hosted.controller.application.SourceRevision; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; import com.yahoo.vespa.hosted.controller.deployment.JobController; @@ -10,6 +12,7 @@ import com.yahoo.vespa.hosted.controller.deployment.RunStatus; import com.yahoo.vespa.hosted.controller.deployment.Step; import com.yahoo.vespa.hosted.controller.deployment.Step.Status; import com.yahoo.vespa.hosted.controller.deployment.StepRunner; +import com.yahoo.vespa.hosted.controller.deployment.Versions; import org.junit.Test; import java.time.Duration; @@ -40,7 +43,6 @@ import static com.yahoo.vespa.hosted.controller.deployment.Step.deactivateReal; import static com.yahoo.vespa.hosted.controller.deployment.Step.deactivateTester; import static com.yahoo.vespa.hosted.controller.deployment.Step.deployReal; import static com.yahoo.vespa.hosted.controller.deployment.Step.deployTester; -import static com.yahoo.vespa.hosted.controller.deployment.Step.installInitialReal; import static com.yahoo.vespa.hosted.controller.deployment.Step.installReal; import static com.yahoo.vespa.hosted.controller.deployment.Step.installTester; import static com.yahoo.vespa.hosted.controller.deployment.Step.report; @@ -56,6 +58,14 @@ import static org.junit.Assert.fail; */ public class JobRunnerTest { + static final Versions versions = new Versions(Version.fromString("1.2.3"), + ApplicationVersion.from(new SourceRevision("repo", + "branch", + "bada55"), + 321), + Optional.empty(), + Optional.empty()); + @Test public void multiThreadedExecutionFinishes() throws InterruptedException { DeploymentTester tester = new DeploymentTester(); @@ -67,15 +77,15 @@ public class JobRunnerTest { Executors.newFixedThreadPool(32), notifying(stepRunner, latch)); ApplicationId id = tester.createApplication("real", "tenant", 1, 1L).id(); - jobs.submit(id, new SourceRevision("repo", "branch", "bada55"), new byte[0], new byte[0]); + jobs.submit(id, versions.targetApplication().source().get(), new byte[0], new byte[0]); - jobs.start(id, systemTest); + jobs.start(id, systemTest, versions); try { - jobs.start(id, systemTest); + jobs.start(id, systemTest, versions); fail("Job is already running, so this should not be allowed!"); } catch (IllegalStateException e) { } - jobs.start(id, stagingTest); + jobs.start(id, stagingTest, versions); assertTrue(jobs.last(id, systemTest).get().steps().values().stream().allMatch(unfinished::equals)); runner.maintain(); @@ -100,10 +110,10 @@ public class JobRunnerTest { inThreadExecutor(), mappedRunner(outcomes)); ApplicationId id = tester.createApplication("real", "tenant", 1, 1L).id(); - jobs.submit(id, new SourceRevision("repo", "branch", "bada55"), new byte[0], new byte[0]); + jobs.submit(id, versions.targetApplication().source().get(), new byte[0], new byte[0]); Supplier<RunStatus> run = () -> jobs.last(id, systemTest).get(); - jobs.start(id, systemTest); + jobs.start(id, systemTest, versions); RunId first = run.get().id(); Map<Step, Status> steps = run.get().steps(); @@ -155,7 +165,7 @@ public class JobRunnerTest { assertTrue(run.get().isAborted()); // A new run is attempted. - jobs.start(id, systemTest); + jobs.start(id, systemTest, versions); assertEquals(first.number() + 1, run.get().id().number()); // Run fails on tester deployment -- remaining run-always steps succeed, and the run finishes. @@ -171,7 +181,7 @@ public class JobRunnerTest { assertEquals(2, jobs.runs(id, systemTest).size()); // Start a third run, then unregister and wait for data to be deleted. - jobs.start(id, systemTest); + jobs.start(id, systemTest, versions); jobs.unregister(id); runner.maintain(); assertFalse(jobs.last(id, systemTest).isPresent()); @@ -188,10 +198,10 @@ public class JobRunnerTest { Executors.newFixedThreadPool(32), waitingRunner(barrier)); ApplicationId id = tester.createApplication("real", "tenant", 1, 1L).id(); - jobs.submit(id, new SourceRevision("repo", "branch", "bada55"), new byte[0], new byte[0]); + jobs.submit(id, versions.targetApplication().source().get(), new byte[0], new byte[0]); RunId runId = new RunId(id, systemTest, 1); - jobs.start(id, systemTest); + jobs.start(id, systemTest, versions); runner.maintain(); barrier.await(); try { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/testdata/run-status.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/testdata/run-status.json index d659bd9fff0..9b3de20acc4 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/testdata/run-status.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/testdata/run-status.json @@ -5,17 +5,31 @@ "number": 112358, "start": 1196676930000, "steps": { - "DIR": "U", - "IIR": "F", - "DR": "S", - "IR": "U", - "DAR": "F", - "DT": "S", - "IT": "U", - "DAT": "F", - "ST": "S", - "ET": "U", - "R": "F" + "deployInitialReal": "unfinished", + "installInitialReal": "failed", + "deployReal": "succeeded", + "installReal": "unfinished", + "deactivateReal": "failed", + "deployTester": "succeeded", + "installTester": "unfinished", + "deactivateTester": "failed", + "startTests": "succeeded", + "endTests": "unfinished", + "report": "failed" + }, + "versions": { + "platform": "1.2.3", + "repository": "git@github.com:user/repo.git", + "branch": "master", + "commit": "f00bad", + "build": 123, + "source": { + "platform": "1.2.3", + "repository": "git@github.com:user/repo.git", + "branch": "master", + "commit": "badb17", + "build": 122 + } } } ]
\ No newline at end of file diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java index de645cff96c..639cfad9958 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java @@ -81,7 +81,7 @@ public class ControllerContainerTest { " <component id='com.yahoo.vespa.hosted.controller.maintenance.JobControl'/>\n" + " <component id='com.yahoo.vespa.hosted.controller.integration.RoutingGeneratorMock'/>\n" + " <component id='com.yahoo.vespa.hosted.controller.integration.ArtifactRepositoryMock'/>\n" + - " <component id='com.yahoo.vespa.hosted.controller.api.integration.stubs.MockTesters'/>\n" + + " <component id='com.yahoo.vespa.hosted.controller.api.integration.stubs.MockTesterCloud'/>\n" + " <handler id='com.yahoo.vespa.hosted.controller.restapi.application.ApplicationApiHandler'>\n" + " <binding>http://*/application/v4/*</binding>\n" + " </handler>\n" + diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelperTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelperTest.java index bc671d2375e..f350cc8627b 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelperTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelperTest.java @@ -1,5 +1,6 @@ package com.yahoo.vespa.hosted.controller.restapi.application; +import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.TenantName; import com.yahoo.container.jdisc.HttpResponse; @@ -7,10 +8,12 @@ import com.yahoo.vespa.hosted.controller.ControllerTester; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; import com.yahoo.vespa.hosted.controller.api.integration.stubs.MockLogStore; +import com.yahoo.vespa.hosted.controller.application.ApplicationVersion; import com.yahoo.vespa.hosted.controller.application.SourceRevision; import com.yahoo.vespa.hosted.controller.deployment.JobController; import com.yahoo.vespa.hosted.controller.deployment.RunStatus; import com.yahoo.vespa.hosted.controller.deployment.Step; +import com.yahoo.vespa.hosted.controller.deployment.Versions; import org.json.JSONException; import org.json.JSONObject; import org.junit.Assert; @@ -24,9 +27,7 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.time.Instant; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collections; -import java.util.Comparator; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -38,14 +39,23 @@ public class JobControllerApiHandlerHelperTest { private final ApplicationId appId = ApplicationId.from("vespa", "music", "default"); private final Instant start = Instant.parse("2018-06-27T10:12:35Z"); + static final Versions versions = new Versions(Version.fromString("1.2.3"), + ApplicationVersion.from(new SourceRevision("repo", + "branch", + "bada55"), + 321), + Optional.empty(), + Optional.empty()); + + private static Step lastStep = Step.values()[Step.values().length - 1]; @Test public void jobTypeResponse() { Map<JobType, RunStatus> jobMap = new HashMap<>(); List<JobType> jobList = new ArrayList<>(); - jobMap.put(JobType.systemTest, createStatus(JobType.systemTest, 1, 30, Step.last(), Step.Status.succeeded)); + jobMap.put(JobType.systemTest, createStatus(JobType.systemTest, 1, 30, lastStep, Step.Status.succeeded)); jobList.add(JobType.systemTest); - jobMap.put(JobType.productionApNortheast1, createStatus(JobType.productionApNortheast1, 1, 60, Step.last(), Step.Status.succeeded)); + jobMap.put(JobType.productionApNortheast1, createStatus(JobType.productionApNortheast1, 1, 60, lastStep, Step.Status.succeeded)); jobList.add(JobType.productionApNortheast1); jobMap.put(JobType.productionUsWest1, createStatus(JobType.productionUsWest1, 1, 60, Step.startTests, Step.Status.failed)); jobList.add(JobType.productionUsWest1); @@ -61,13 +71,13 @@ public class JobControllerApiHandlerHelperTest { Map<RunId, RunStatus> statusMap = new HashMap<>(); RunStatus status; - status = createStatus(JobType.systemTest, 3, 30, Step.last(), Step.Status.succeeded); + status = createStatus(JobType.systemTest, 3, 30, lastStep, Step.Status.succeeded); statusMap.put(status.id(), status); status = createStatus(JobType.systemTest, 2, 56, Step.installReal, Step.Status.failed); statusMap.put(status.id(), status); - status = createStatus(JobType.systemTest, 1, 44, Step.last(), Step.Status.succeeded); + status = createStatus(JobType.systemTest, 1, 44, lastStep, Step.Status.succeeded); statusMap.put(status.id(), status); URI jobTypeUrl = URI.create("https://domain.tld/application/v4/tenant/sometenant/application/someapp/instance/usuallydefault/job/systemtest"); @@ -85,7 +95,7 @@ public class JobControllerApiHandlerHelperTest { tester.curator().writeHistoricRuns( runId.application(), runId.type(), - Collections.singleton(createStatus(JobType.systemTest, 42, 44, Step.last(), Step.Status.succeeded))); + Collections.singleton(createStatus(JobType.systemTest, 42, 44, lastStep, Step.Status.succeeded))); logStore.append(runId, Step.deployTester.name(), "INFO\t1234567890\tSUCCESS".getBytes()); logStore.append(runId, Step.installTester.name(), "INFO\t1234598760\tSUCCESS".getBytes()); @@ -114,7 +124,7 @@ public class JobControllerApiHandlerHelperTest { RunId runId = new RunId(appId, type, runid); Map<Step, Step.Status> stepStatusMap = new HashMap<>(); - Arrays.stream(Step.values()).sorted(Comparator.naturalOrder()).forEach(step -> { + for (Step step : Step.values()) { if (step.ordinal() < lastStep.ordinal()) { stepStatusMap.put(step, Step.Status.succeeded); } else if (step.equals(lastStep)) { @@ -122,14 +132,14 @@ public class JobControllerApiHandlerHelperTest { } else { stepStatusMap.put(step, Step.Status.unfinished); } - }); + } Optional<Instant> end = Optional.empty(); - if (lastStepStatus == Step.Status.failed || (lastStepStatus != Step.Status.unfinished && lastStep == Step.last())) { + if (lastStepStatus == Step.Status.failed || stepStatusMap.get(lastStep) == Step.Status.succeeded) { end = Optional.of(start.plusSeconds(duration)); } - return new RunStatus(runId, stepStatusMap, start, end, false); + return new RunStatus(runId, stepStatusMap, versions, start, end, false); } private void compare(HttpResponse response, String expected) { diff --git a/dist/vespa.spec b/dist/vespa.spec index 0e3cabcea80..bd2bf798e58 100644 --- a/dist/vespa.spec +++ b/dist/vespa.spec @@ -56,7 +56,7 @@ BuildRequires: gtest-devel BuildRequires: gmock-devel %endif %if 0%{?fc29} -BuildRequires: llvm4.0-devel >= 4.0 +BuildRequires: llvm3.9-devel >= 3.9.1 BuildRequires: boost-devel >= 1.66 BuildRequires: gtest-devel BuildRequires: gmock-devel @@ -130,10 +130,10 @@ Requires: llvm4.0-libs >= 4.0 %define _vespa_llvm_include_directory /usr/include/llvm4.0 %endif %if 0%{?fc29} -Requires: llvm4.0-libs >= 4.0 -%define _vespa_llvm_version 4.0 -%define _vespa_llvm_link_directory /usr/lib64/llvm4.0/lib -%define _vespa_llvm_include_directory /usr/include/llvm4.0 +Requires: llvm3.9-libs >= 3.9.1 +%define _vespa_llvm_version 3.9 +%define _vespa_llvm_link_directory /usr/lib64/llvm3.9/lib +%define _vespa_llvm_include_directory /usr/include/llvm3.9 %endif %define _extra_link_directory /opt/vespa-cppunit/lib%{?_vespa_llvm_link_directory:;%{_vespa_llvm_link_directory}}%{?_vespa_gtest_link_directory:;%{_vespa_gtest_link_directory}} %define _extra_include_directory /opt/vespa-cppunit/include%{?_vespa_llvm_include_directory:;%{_vespa_llvm_include_directory}}%{?_vespa_gtest_include_directory:;%{_vespa_gtest_include_directory}} @@ -162,7 +162,7 @@ source %{_devtoolset_enable} || true source %{_rhmaven35_enable} || true %endif sh bootstrap.sh java -mvn --batch-mode -nsu -T 2C install -Dmaven.test.skip=true -Dmaven.javadoc.skip=true +mvn --batch-mode -nsu -T 1 install -Dmaven.test.skip=true -Dmaven.javadoc.skip=true cmake3 -DCMAKE_INSTALL_PREFIX=%{_prefix} \ -DJAVA_HOME=/usr/lib/jvm/java-openjdk \ -DEXTRA_LINK_DIRECTORY="%{_extra_link_directory}" \ diff --git a/document/src/tests/fieldpathupdatetestcase.cpp b/document/src/tests/fieldpathupdatetestcase.cpp index fb3ba3f7e40..f5db5831912 100644 --- a/document/src/tests/fieldpathupdatetestcase.cpp +++ b/document/src/tests/fieldpathupdatetestcase.cpp @@ -33,6 +33,7 @@ struct FieldPathUpdateTestCase : public CppUnit::TestFixture { void tearDown() override; void testWhereClause(); + void testBrokenWhereClause(); void testNoIterateMapValues(); void testRemoveField(); void testApplyRemoveEntireListField(); @@ -73,6 +74,7 @@ struct FieldPathUpdateTestCase : public CppUnit::TestFixture { CPPUNIT_TEST_SUITE(FieldPathUpdateTestCase); CPPUNIT_TEST(testWhereClause); + CPPUNIT_TEST(testBrokenWhereClause); CPPUNIT_TEST(testNoIterateMapValues); CPPUNIT_TEST(testRemoveField); CPPUNIT_TEST(testApplyRemoveEntireListField); @@ -356,6 +358,17 @@ FieldPathUpdateTestCase::testWhereClause() } void +FieldPathUpdateTestCase::testBrokenWhereClause() +{ + DocumentTypeRepo repo(getRepoConfig()); + Document::UP doc(createTestDocument(repo)); + std::string where = "l1s1.structmap.value.smap{$x} == \"dicaprio\""; + TestFieldPathUpdate update("l1s1.structmap.value.smap{$x}", where); + update.applyTo(*doc); + CPPUNIT_ASSERT_EQUAL(std::string(""), update._str); +} + +void FieldPathUpdateTestCase::testNoIterateMapValues() { DocumentTypeRepo repo(getRepoConfig()); diff --git a/document/src/vespa/document/update/fieldpathupdate.cpp b/document/src/vespa/document/update/fieldpathupdate.cpp index e7d5824b6e0..fa7a8b38aba 100644 --- a/document/src/vespa/document/update/fieldpathupdate.cpp +++ b/document/src/vespa/document/update/fieldpathupdate.cpp @@ -3,7 +3,9 @@ #include <vespa/document/datatype/datatype.h> #include <vespa/document/fieldvalue/document.h> #include <vespa/document/fieldvalue/iteratorhandler.h> +#include <vespa/document/select/constant.h> #include <vespa/document/select/parser.h> +#include <vespa/document/select/parsing_failed_exception.h> #include <vespa/document/util/serializableexceptions.h> #include <vespa/vespalib/objects/nbostream.h> #include <ostream> @@ -11,6 +13,7 @@ #include <vespa/log/log.h> LOG_SETUP(".document.update.fieldpathupdate"); +using document::select::ParsingFailedException; using vespalib::make_string; using vespalib::IllegalArgumentException; @@ -26,8 +29,13 @@ std::unique_ptr<select::Node> parseDocumentSelection(vespalib::stringref query, const DocumentTypeRepo& repo) { BucketIdFactory factory; - select::Parser parser(repo, factory); - return parser.parse(query); + try { + select::Parser parser(repo, factory); + return parser.parse(query); + } catch (const ParsingFailedException &e) { + LOG(warning, "Failed to parse selection for field path update: %s", e.getMessage().c_str()); + return std::make_unique<select::Constant>(false); + } } } // namespace diff --git a/eval/src/tests/eval/compiled_function/compiled_function_test.cpp b/eval/src/tests/eval/compiled_function/compiled_function_test.cpp index 0e9806d5381..151a4cf5dd5 100644 --- a/eval/src/tests/eval/compiled_function/compiled_function_test.cpp +++ b/eval/src/tests/eval/compiled_function/compiled_function_test.cpp @@ -178,7 +178,7 @@ TEST("dump ir code to verify lazy casting") { Function function = Function::parse({"a", "b"}, "12==2+if(a==3&&a<10||b,10,5)"); LLVMWrapper wrapper; size_t id = wrapper.make_function(function.num_params(), PassParams::SEPARATE, function.root(), {}); - wrapper.compile(true); // dump module before compiling it + wrapper.compile(llvm::dbgs()); // dump module before compiling it using fun_type = double (*)(double, double); fun_type fun = (fun_type) wrapper.get_function_address(id); EXPECT_EQUAL(0.0, fun(0.0, 0.0)); diff --git a/eval/src/vespa/eval/eval/llvm/llvm_wrapper.cpp b/eval/src/vespa/eval/eval/llvm/llvm_wrapper.cpp index f314f8a69cb..71ca74f4167 100644 --- a/eval/src/vespa/eval/eval/llvm/llvm_wrapper.cpp +++ b/eval/src/vespa/eval/eval/llvm/llvm_wrapper.cpp @@ -48,8 +48,7 @@ double vespalib_eval_forest_proxy(Forest::eval_function eval_forest, const Fores } } -namespace vespalib { -namespace eval { +namespace vespalib::eval { using namespace nodes; @@ -84,6 +83,19 @@ struct FunctionBuilder : public NodeVisitor, public NodeTraverser { std::vector<gbdt::Forest::UP> &forests; std::vector<PluginState::UP> &plugin_state; + llvm::FunctionType *make_call_1_fun_t() { + std::vector<llvm::Type*> param_types; + param_types.push_back(builder.getDoubleTy()); + return llvm::FunctionType::get(builder.getDoubleTy(), param_types, false); + } + + llvm::FunctionType *make_call_2_fun_t() { + std::vector<llvm::Type*> param_types; + param_types.push_back(builder.getDoubleTy()); + param_types.push_back(builder.getDoubleTy()); + return llvm::FunctionType::get(builder.getDoubleTy(), param_types, false); + } + llvm::PointerType *make_eval_forest_funptr_t() { std::vector<llvm::Type*> param_types; param_types.push_back(builder.getVoidTy()->getPointerTo()); @@ -320,9 +332,7 @@ struct FunctionBuilder : public NodeVisitor, public NodeTraverser { make_call_1(llvm::Intrinsic::getDeclaration(&module, id, builder.getDoubleTy())); } void make_call_1(const char *name) { - make_call_1(dynamic_cast<llvm::Function*>(module.getOrInsertFunction(name, - builder.getDoubleTy(), - builder.getDoubleTy(), nullptr))); + make_call_1(llvm::dyn_cast<llvm::Function>(module.getOrInsertFunction(name, make_call_1_fun_t()))); } void make_call_2(llvm::Function *fun) { @@ -337,10 +347,7 @@ struct FunctionBuilder : public NodeVisitor, public NodeTraverser { make_call_2(llvm::Intrinsic::getDeclaration(&module, id, builder.getDoubleTy())); } void make_call_2(const char *name) { - make_call_2(dynamic_cast<llvm::Function*>(module.getOrInsertFunction(name, - builder.getDoubleTy(), - builder.getDoubleTy(), - builder.getDoubleTy(), nullptr))); + make_call_2(llvm::dyn_cast<llvm::Function>(module.getOrInsertFunction(name, make_call_2_fun_t()))); } //------------------------------------------------------------------------- @@ -659,11 +666,11 @@ LLVMWrapper::make_forest_fragment(size_t num_params, const std::vector<const Nod } void -LLVMWrapper::compile(bool dump_module) +LLVMWrapper::compile(llvm::raw_ostream * dumpStream) { std::lock_guard<std::recursive_mutex> guard(_global_llvm_lock); - if (dump_module) { - _module->dump(); + if (dumpStream) { + _module->print(*dumpStream, nullptr); } _engine.reset(llvm::EngineBuilder(std::move(_module)).setOptLevel(llvm::CodeGenOpt::Aggressive).create()); assert(_engine && "llvm jit not available for your platform"); @@ -687,5 +694,4 @@ LLVMWrapper::~LLVMWrapper() { _context.reset(); } -} // namespace vespalib::eval -} // namespace vespalib +} diff --git a/eval/src/vespa/eval/eval/llvm/llvm_wrapper.h b/eval/src/vespa/eval/eval/llvm/llvm_wrapper.h index 6860be922f4..db29771ee9e 100644 --- a/eval/src/vespa/eval/eval/llvm/llvm_wrapper.h +++ b/eval/src/vespa/eval/eval/llvm/llvm_wrapper.h @@ -21,8 +21,7 @@ extern "C" { double vespalib_eval_elu(double a); }; -namespace vespalib { -namespace eval { +namespace vespalib::eval { /** * Simple interface used to track and clean up custom state. This is @@ -52,6 +51,7 @@ private: static std::recursive_mutex _global_llvm_lock; + void compile(llvm::raw_ostream * dumpStream); public: LLVMWrapper(); LLVMWrapper(LLVMWrapper &&rhs) = default; @@ -60,11 +60,11 @@ public: const gbdt::Optimize::Chain &forest_optimizers); size_t make_forest_fragment(size_t num_params, const std::vector<const nodes::Node *> &fragment); const std::vector<gbdt::Forest::UP> &get_forests() const { return _forests; } - void compile(bool dump_module = false); + void compile(llvm::raw_ostream & dumpStream) { compile(&dumpStream); } + void compile() { compile(nullptr); } void *get_function_address(size_t function_id); ~LLVMWrapper(); }; -} // namespace vespalib::eval -} // namespace vespalib +} diff --git a/functions.cmake b/functions.cmake index c0e07c1d362..f0dd5a31949 100644 --- a/functions.cmake +++ b/functions.cmake @@ -422,6 +422,14 @@ function(vespa_install_script) endif() endfunction() +function(vespa_install_data) + if(ARGC GREATER 2) + install(FILES ${ARGV0} RENAME ${ARGV1} PERMISSIONS OWNER_READ OWNER_WRITE GROUP_READ WORLD_READ DESTINATION ${ARGV2}) + else() + install(FILES ${ARGV0} PERMISSIONS OWNER_READ OWNER_WRITE GROUP_READ WORLD_READ DESTINATION ${ARGV1}) + endif() +endfunction() + function(vespa_workaround_gcc_bug_67055 SOURCE_FILE) if(CMAKE_COMPILER_IS_GNUCC) execute_process(COMMAND ${CMAKE_CPP_COMPILER} -dumpversion OUTPUT_VARIABLE GCC_VERSION) diff --git a/jrt/tests/com/yahoo/jrt/AbortTest.java b/jrt/tests/com/yahoo/jrt/AbortTest.java index e3a1685c9e2..9093158162d 100644 --- a/jrt/tests/com/yahoo/jrt/AbortTest.java +++ b/jrt/tests/com/yahoo/jrt/AbortTest.java @@ -19,8 +19,8 @@ public class AbortTest { public void setUp() throws ListenFailedException { server = new Supervisor(new Transport()); client = new Supervisor(new Transport()); - acceptor = server.listen(new Spec(Test.PORT)); - target = client.connect(new Spec("localhost", Test.PORT)); + acceptor = server.listen(new Spec(0)); + target = client.connect(new Spec("localhost", acceptor.port())); server.addMethod(new Method("test", "i", "i", this, "rpc_test")); barrier = new Test.Barrier(); } diff --git a/jrt/tests/com/yahoo/jrt/BackTargetTest.java b/jrt/tests/com/yahoo/jrt/BackTargetTest.java index 3524d47dc1f..ade24f40c55 100644 --- a/jrt/tests/com/yahoo/jrt/BackTargetTest.java +++ b/jrt/tests/com/yahoo/jrt/BackTargetTest.java @@ -22,8 +22,8 @@ public class BackTargetTest { public void setUp() throws ListenFailedException { server = new Supervisor(new Transport()); client = new Supervisor(new Transport()); - acceptor = server.listen(new Spec(Test.PORT)); - target = client.connect(new Spec("localhost", Test.PORT)); + acceptor = server.listen(new Spec(0)); + target = client.connect(new Spec("localhost", acceptor.port())); server.addMethod(new Method("inc", "", "", this, "server_inc")); server.addMethod(new Method("sample_target", "", "", this, diff --git a/jrt/tests/com/yahoo/jrt/ConnectTest.java b/jrt/tests/com/yahoo/jrt/ConnectTest.java index 64ed3f5f954..efe4a018c70 100644 --- a/jrt/tests/com/yahoo/jrt/ConnectTest.java +++ b/jrt/tests/com/yahoo/jrt/ConnectTest.java @@ -9,12 +9,12 @@ public class ConnectTest { public void testConnect() throws ListenFailedException { Test.Orb server = new Test.Orb(new Transport()); Test.Orb client = new Test.Orb(new Transport()); - Acceptor acceptor = server.listen(new Spec(Test.PORT)); + Acceptor acceptor = server.listen(new Spec(0)); assertTrue(server.checkLifeCounts(0, 0)); assertTrue(client.checkLifeCounts(0, 0)); - Target target = client.connect(new Spec("localhost", Test.PORT)); + Target target = client.connect(new Spec("localhost", acceptor.port())); for (int i = 0; i < 100; i++) { if (client.initCount == 1 && server.initCount == 1) { diff --git a/jrt/tests/com/yahoo/jrt/DetachTest.java b/jrt/tests/com/yahoo/jrt/DetachTest.java index 1c1256f0689..808d029b5a5 100644 --- a/jrt/tests/com/yahoo/jrt/DetachTest.java +++ b/jrt/tests/com/yahoo/jrt/DetachTest.java @@ -20,8 +20,8 @@ public class DetachTest { public void setUp() throws ListenFailedException { server = new Test.Orb(new Transport()); client = new Test.Orb(new Transport()); - acceptor = server.listen(new Spec(Test.PORT)); - target = client.connect(new Spec("localhost", Test.PORT)); + acceptor = server.listen(new Spec(0)); + target = client.connect(new Spec("localhost", acceptor.port())); server.addMethod(new Method("d_inc", "i", "i", this, "rpc_detach_inc")); diff --git a/jrt/tests/com/yahoo/jrt/EchoTest.java b/jrt/tests/com/yahoo/jrt/EchoTest.java index 240feda8423..5acd8221a9b 100644 --- a/jrt/tests/com/yahoo/jrt/EchoTest.java +++ b/jrt/tests/com/yahoo/jrt/EchoTest.java @@ -19,8 +19,8 @@ public class EchoTest { public void setUp() throws ListenFailedException { server = new Supervisor(new Transport()); client = new Supervisor(new Transport()); - acceptor = server.listen(new Spec(Test.PORT)); - target = client.connect(new Spec("localhost", Test.PORT)); + acceptor = server.listen(new Spec(0)); + target = client.connect(new Spec("localhost", acceptor.port())); server.addMethod(new Method("echo", "*", "*", this, "rpc_echo")); refValues = new Values(); byte[] dataValue = { 1, 2, 3, 4 }; diff --git a/jrt/tests/com/yahoo/jrt/InvokeAsyncTest.java b/jrt/tests/com/yahoo/jrt/InvokeAsyncTest.java index d744fcf8f29..cdc52e9441a 100644 --- a/jrt/tests/com/yahoo/jrt/InvokeAsyncTest.java +++ b/jrt/tests/com/yahoo/jrt/InvokeAsyncTest.java @@ -21,8 +21,8 @@ public class InvokeAsyncTest { public void setUp() throws ListenFailedException { server = new Supervisor(new Transport()); client = new Supervisor(new Transport()); - acceptor = server.listen(new Spec(Test.PORT)); - target = client.connect(new Spec("localhost", Test.PORT)); + acceptor = server.listen(new Spec(0)); + target = client.connect(new Spec("localhost", acceptor.port())); server.addMethod(new Method("concat", "ss", "s", this, "rpc_concat") .methodDesc("Concatenate 2 strings") .paramDesc(0, "str1", "a string") diff --git a/jrt/tests/com/yahoo/jrt/InvokeErrorTest.java b/jrt/tests/com/yahoo/jrt/InvokeErrorTest.java index 6fe92be97e5..4e810b71fb6 100644 --- a/jrt/tests/com/yahoo/jrt/InvokeErrorTest.java +++ b/jrt/tests/com/yahoo/jrt/InvokeErrorTest.java @@ -21,8 +21,8 @@ public class InvokeErrorTest { public void setUp() throws ListenFailedException { server = new Supervisor(new Transport()); client = new Supervisor(new Transport()); - acceptor = server.listen(new Spec(Test.PORT)); - target = client.connect(new Spec("localhost", Test.PORT)); + acceptor = server.listen(new Spec(0)); + target = client.connect(new Spec("localhost", acceptor.port())); server.addMethod(new Method("test", "iib", "i", this, "rpc_test")); server.addMethod(new Method("test_barrier", "iib", "i", this, "rpc_test_barrier")); diff --git a/jrt/tests/com/yahoo/jrt/InvokeSyncTest.java b/jrt/tests/com/yahoo/jrt/InvokeSyncTest.java index 37856c0db81..931001804aa 100644 --- a/jrt/tests/com/yahoo/jrt/InvokeSyncTest.java +++ b/jrt/tests/com/yahoo/jrt/InvokeSyncTest.java @@ -26,8 +26,8 @@ public class InvokeSyncTest { public void setUp() throws ListenFailedException { server = new Supervisor(new Transport()); client = new Supervisor(new Transport()); - acceptor = server.listen(new Spec(Test.PORT)); - target = client.connect(new Spec("localhost", Test.PORT)); + acceptor = server.listen(new Spec(0)); + target = client.connect(new Spec("localhost", acceptor.port())); server.addMethod(new Method("concat", "ss", "s", this, "rpc_concat") .methodDesc("Concatenate 2 strings") .paramDesc(0, "str1", "a string") @@ -74,12 +74,12 @@ public class InvokeSyncTest { public void testRpcInvoker() throws IOException { ByteArrayOutputStream baos = new ByteArrayOutputStream(); System.setOut(new PrintStream(baos)); - RpcInvoker.main(new String[] {"-h", "localhost:"+Test.PORT, "concat", "s:foo", "s:bar"}); + RpcInvoker.main(new String[] {"-h", "localhost:"+acceptor.port(), "concat", "s:foo", "s:bar"}); baos.flush(); assertEquals(baos.toString(), "foobar\n"); baos.reset(); System.setOut(new PrintStream(baos)); - RpcInvoker.main(new String[] {"-h", "localhost:"+Test.PORT, "alltypes", "b:1", "h:2", "i:3", "l:4", "f:5.0", "d:6.0", "s:baz"}); + RpcInvoker.main(new String[] {"-h", "localhost:"+acceptor.port(), "alltypes", "b:1", "h:2", "i:3", "l:4", "f:5.0", "d:6.0", "s:baz"}); baos.flush(); assertEquals(baos.toString(), "This was alltypes. The string param was: baz\n"); } diff --git a/jrt/tests/com/yahoo/jrt/InvokeVoidTest.java b/jrt/tests/com/yahoo/jrt/InvokeVoidTest.java index d0c0d9c728e..25e86a16445 100644 --- a/jrt/tests/com/yahoo/jrt/InvokeVoidTest.java +++ b/jrt/tests/com/yahoo/jrt/InvokeVoidTest.java @@ -19,8 +19,8 @@ public class InvokeVoidTest { public void setUp() throws ListenFailedException { server = new Test.Orb(new Transport()); client = new Test.Orb(new Transport()); - acceptor = server.listen(new Spec(Test.PORT)); - target = client.connect(new Spec("localhost", Test.PORT)); + acceptor = server.listen(new Spec(0)); + target = client.connect(new Spec("localhost", acceptor.port())); server.addMethod(new Method("set", "i", "", this, "rpc_set") .methodDesc("Set the stored value") diff --git a/jrt/tests/com/yahoo/jrt/ListenTest.java b/jrt/tests/com/yahoo/jrt/ListenTest.java index 88fd6f22f24..5e3bde08a20 100644 --- a/jrt/tests/com/yahoo/jrt/ListenTest.java +++ b/jrt/tests/com/yahoo/jrt/ListenTest.java @@ -24,36 +24,39 @@ public class ListenTest { @org.junit.Test public void testListen() { try { - Acceptor a = server.listen(new Spec(Test.PORT)); - assertEquals(Test.PORT, a.port()); + Acceptor a = server.listen(new Spec(0)); + assertTrue(a.port() > 0); a.shutdown().join(); assertEquals(-1, a.port()); } catch (ListenFailedException e) { assertTrue(false); } try { - Acceptor a = server.listen(new Spec(null, Test.PORT)); - assertEquals(Test.PORT, a.port()); + Acceptor a = server.listen(new Spec(null, 0)); + assertTrue(a.port() > 0); a.shutdown().join(); assertEquals(-1, a.port()); } catch (ListenFailedException e) { assertTrue(false); } try { - Acceptor a = server.listen(new Spec("tcp/" + Test.PORT)); - assertEquals(Test.PORT, a.port()); + Acceptor a = server.listen(new Spec("tcp/" + 0)); + assertTrue(a.port() > 0); a.shutdown().join(); assertEquals(-1, a.port()); } catch (ListenFailedException e) { assertTrue(false); } try { - Acceptor a = server.listen(new Spec(Test.PORT_0)); - Acceptor b = server.listen(new Spec(Test.PORT_1)); - Acceptor c = server.listen(new Spec(Test.PORT_2)); - assertEquals(Test.PORT_0, a.port()); - assertEquals(Test.PORT_1, b.port()); - assertEquals(Test.PORT_2, c.port()); + Acceptor a = server.listen(new Spec(0)); + Acceptor b = server.listen(new Spec(0)); + Acceptor c = server.listen(new Spec(0)); + assertTrue(a.port() > 0); + assertTrue(b.port() > 0); + assertTrue(c.port() > 0); + assertTrue(a.port() != b.port()); + assertTrue(a.port() != c.port()); + assertTrue(b.port() != c.port()); a.shutdown().join(); assertEquals(-1, a.port()); b.shutdown().join(); @@ -71,35 +74,21 @@ public class ListenTest { Acceptor a = server.listen(new Spec("bogus")); assertTrue(false); } catch (ListenFailedException e) {} - - try { - Acceptor a = server.listen(new Spec(Test.PORT)); - assertEquals(Test.PORT, a.port()); - // try { - // Acceptor b = server.listen(new Spec(Test.PORT)); - // assertTrue(false); - // } catch (ListenFailedException e) {} - a.shutdown().join(); - assertEquals(-1, a.port()); - } catch (ListenFailedException e) { - assertTrue(false); - } } @org.junit.Test - public void testListenAnyPort() { + public void testListenSamePort() { try { Acceptor a = server.listen(new Spec("tcp/0")); assertTrue(a.port() > 0); - // try { - // Acceptor b = server.listen(new Spec(a.port())); - // assertTrue(false); - // } catch (ListenFailedException e) {} + try { + Acceptor b = server.listen(new Spec(a.port())); + assertTrue(false); + } catch (ListenFailedException e) {} a.shutdown().join(); assertEquals(-1, a.port()); } catch (ListenFailedException e) { assertTrue(false); } } - } diff --git a/jrt/tests/com/yahoo/jrt/MandatoryMethodsTest.java b/jrt/tests/com/yahoo/jrt/MandatoryMethodsTest.java index c61cae18c09..0bf83240338 100644 --- a/jrt/tests/com/yahoo/jrt/MandatoryMethodsTest.java +++ b/jrt/tests/com/yahoo/jrt/MandatoryMethodsTest.java @@ -23,8 +23,8 @@ public class MandatoryMethodsTest { public void setUp() throws ListenFailedException { server = new Supervisor(new Transport()); client = new Supervisor(new Transport()); - acceptor = server.listen(new Spec(Test.PORT)); - target = client.connect(new Spec("localhost", Test.PORT)); + acceptor = server.listen(new Spec(0)); + target = client.connect(new Spec("localhost", acceptor.port())); } @After diff --git a/jrt/tests/com/yahoo/jrt/SessionTest.java b/jrt/tests/com/yahoo/jrt/SessionTest.java index afe14bfb908..1a788c4038e 100644 --- a/jrt/tests/com/yahoo/jrt/SessionTest.java +++ b/jrt/tests/com/yahoo/jrt/SessionTest.java @@ -115,8 +115,8 @@ public class SessionTest implements SessionHandler { server.setSessionHandler(this); client = new Test.Orb(new Transport()); client.setSessionHandler(this); - acceptor = server.listen(new Spec(Test.PORT)); - target = client.connect(new Spec("localhost", Test.PORT), + acceptor = server.listen(new Spec(0)); + target = client.connect(new Spec("localhost", acceptor.port()), new Session()); server.addMethod(new Method("set", "i", "", this, @@ -425,7 +425,7 @@ public class SessionTest implements SessionHandler { assertEquals(1, client.downCount); assertEquals(1, client.finiCount); - target = client.connect(new Spec("localhost", Test.PORT), + target = client.connect(new Spec("localhost", acceptor.port()), new Session()); waitState(0, 2, 1, 2, 2, 2, -1, 2, 2); @@ -441,7 +441,7 @@ public class SessionTest implements SessionHandler { client.transport().shutdown().join(); - target = client.connect(new Spec("localhost", Test.PORT), + target = client.connect(new Spec("localhost", acceptor.port()), new Session()); waitState(0, 2, 1, 2, 2, 3, oldClientLive, 3, 3); diff --git a/jrt/tests/com/yahoo/jrt/Test.java b/jrt/tests/com/yahoo/jrt/Test.java index b538d5729e5..efde0c57912 100644 --- a/jrt/tests/com/yahoo/jrt/Test.java +++ b/jrt/tests/com/yahoo/jrt/Test.java @@ -8,19 +8,6 @@ public class Test { @org.junit.Test public void testNothing() {} - // www.random.org [2000, 9999] - public static final int PORT = 9741; - public static final int PORT_0 = 5069; - public static final int PORT_1 = 4935; - public static final int PORT_2 = 8862; - public static final int PORT_3 = 4695; - public static final int PORT_4 = 6975; - public static final int PORT_5 = 7186; - public static final int PORT_6 = 7694; - public static final int PORT_7 = 3518; - public static final int PORT_8 = 3542; - public static final int PORT_9 = 4954; - /** * Supervisor extension with some extra statistics used for * testing. diff --git a/jrt/tests/com/yahoo/jrt/TimeoutTest.java b/jrt/tests/com/yahoo/jrt/TimeoutTest.java index d1eb2a9895a..61822554a65 100644 --- a/jrt/tests/com/yahoo/jrt/TimeoutTest.java +++ b/jrt/tests/com/yahoo/jrt/TimeoutTest.java @@ -20,8 +20,8 @@ public class TimeoutTest { public void setUp() throws ListenFailedException { server = new Supervisor(new Transport()); client = new Supervisor(new Transport()); - acceptor = server.listen(new Spec(Test.PORT)); - target = client.connect(new Spec("localhost", Test.PORT)); + acceptor = server.listen(new Spec(0)); + target = client.connect(new Spec("localhost", acceptor.port())); server.addMethod(new Method("concat", "ss", "s", this, "rpc_concat") .methodDesc("Concatenate 2 strings") .paramDesc(0, "str1", "a string") diff --git a/jrt/tests/com/yahoo/jrt/WatcherTest.java b/jrt/tests/com/yahoo/jrt/WatcherTest.java index 591a55662f0..3926b859875 100644 --- a/jrt/tests/com/yahoo/jrt/WatcherTest.java +++ b/jrt/tests/com/yahoo/jrt/WatcherTest.java @@ -36,8 +36,8 @@ public class WatcherTest { public void setUp() throws ListenFailedException { server = new Supervisor(new Transport()); client = new Supervisor(new Transport()); - acceptor = server.listen(new Spec(Test.PORT)); - target = client.connect(new Spec("localhost", Test.PORT)); + acceptor = server.listen(new Spec(0)); + target = client.connect(new Spec("localhost", acceptor.port())); } @After diff --git a/logforwarder/src/apps/vespa-logforwarder-start/child-handler.cpp b/logforwarder/src/apps/vespa-logforwarder-start/child-handler.cpp index 6ed209cbe5b..0588ce5d2e7 100644 --- a/logforwarder/src/apps/vespa-logforwarder-start/child-handler.cpp +++ b/logforwarder/src/apps/vespa-logforwarder-start/child-handler.cpp @@ -6,6 +6,8 @@ #include <unistd.h> #include <sys/types.h> #include <sys/wait.h> +#include <vector> +#include <string> #include <vespa/log/log.h> LOG_SETUP(".child-handler"); @@ -15,13 +17,18 @@ ChildHandler::ChildHandler() : _childRunning(false) {} namespace { void -runSplunk(const vespalib::string &prefix, const char *a1, const char *a2 = 0) +runSplunk(const vespalib::string &prefix, std::vector<const char *> args) { - const char *argv[] = { 0, a1, a2, 0 }; vespalib::string path = prefix + "/bin/splunk"; - argv[0] = path.c_str(); - LOG(debug, "starting splunk forwarder with command: '%s' '%s' '%s'", - argv[0], argv[1], argv[2]); + args.insert(args.begin(), path.c_str()); + std::string dbg = ""; + for (const char *arg : args) { + dbg.append(" '"); + dbg.append(arg); + dbg.append("'"); + } + LOG(debug, "starting splunk forwarder with command: %s", dbg.c_str()); + args.push_back(nullptr); fflush(stdout); pid_t child = fork(); if (child == -1) { @@ -33,10 +40,10 @@ runSplunk(const vespalib::string &prefix, const char *a1, const char *a2 = 0) char *cenv = const_cast<char *>(env.c_str()); // safe cast putenv(cenv); LOG(debug, "added to environment: '%s'", cenv); - char **cargv = const_cast<char **>(argv); // safe cast - execv(argv[0], cargv); + char **cargv = const_cast<char **>(args.data()); // safe cast + execv(cargv[0], cargv); // if execv fails: - perror(argv[0]); + perror(cargv[0]); exit(1); } LOG(debug, "child running with pid %d", (int)child); @@ -69,19 +76,19 @@ void ChildHandler::startChild(const vespalib::string &prefix) { if (! _childRunning) { - runSplunk(prefix, "start", "--accept-license"); + runSplunk(prefix, {"start", "--answer-yes", "--no-prompt", "--accept-license"}); _childRunning = true; // it is possible that splunk was already running, and // then the above won't do anything, so we need to // *also* do the restart below, after a small delay. sleep(1); } - runSplunk(prefix, "restart"); + runSplunk(prefix, {"restart"}); } void ChildHandler::stopChild(const vespalib::string &prefix) { - runSplunk(prefix, "stop"); + runSplunk(prefix, {"stop"}); _childRunning = false; } diff --git a/node-admin/src/main/application/services.xml b/node-admin/src/main/application/services.xml index 96fe82a5b94..284b356d2ca 100644 --- a/node-admin/src/main/application/services.xml +++ b/node-admin/src/main/application/services.xml @@ -1,7 +1,7 @@ <?xml version="1.0" encoding="utf-8" ?> <!-- Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> <services version="1.0" xmlns:preprocess="properties"> - <jdisc id="node-admin" jetty="true" version="1.0"> + <container id="node-admin" version="1.0"> <!-- Please update container test when changing this file --> <accesslog type="vespa" fileNamePattern="logs/vespa/node-admin/access.log.%Y%m%d%H%M%S" rotationScheme="date" symlinkName="access.log" /> <component id="docker-api" class="com.yahoo.vespa.hosted.dockerapi.DockerImpl" bundle="docker-api"/> @@ -13,5 +13,5 @@ </config> <preprocess:include file="variant.xml" required="false"/> - </jdisc> + </container> </services> diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/ConfigServerInfo.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/ConfigServerInfo.java index 93243f8b8ed..9e94f6ed7e4 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/ConfigServerInfo.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/ConfigServerInfo.java @@ -1,20 +1,14 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.node.admin.component; -import com.google.common.base.Strings; -import com.yahoo.vespa.athenz.api.AthenzIdentity; import com.yahoo.vespa.athenz.api.AthenzService; import com.yahoo.vespa.athenz.utils.AthenzIdentities; import com.yahoo.vespa.hosted.node.admin.config.ConfigServerConfig; -import com.yahoo.vespa.hosted.node.admin.util.KeyStoreOptions; import java.net.URI; -import java.nio.file.Paths; import java.util.ArrayList; -import java.util.Arrays; import java.util.List; import java.util.Map; -import java.util.Optional; import java.util.function.Function; import static java.util.stream.Collectors.toMap; diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/TaskContext.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/TaskContext.java index 0c49e478d6a..3c44186f78d 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/TaskContext.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/TaskContext.java @@ -21,8 +21,8 @@ public interface TaskContext { * to bob". */ void recordSystemModification(Logger logger, String message); - default void recordSystemModification(Logger logger, String messageFormat, String... args) { - recordSystemModification(logger, String.format(messageFormat, (Object[]) args)); + default void recordSystemModification(Logger logger, String messageFormat, Object... args) { + recordSystemModification(logger, String.format(messageFormat, args)); } /** @@ -35,8 +35,8 @@ public interface TaskContext { * Do not log a message that is also recorded with recordSystemModification. */ default void log(Logger logger, String message) {} - default void log(Logger logger, String messageFormat, String... args) { - log(logger, String.format(messageFormat, (Object[]) args)); + default void log(Logger logger, String messageFormat, Object... args) { + log(logger, String.format(messageFormat, args)); } /** diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/ConfigServerApiImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/ConfigServerApiImpl.java index 12ba777f018..aea44e728ad 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/ConfigServerApiImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/ConfigServerApiImpl.java @@ -110,41 +110,26 @@ public class ConfigServerApiImpl implements ConfigServerApi { private <T> T tryAllConfigServers(CreateRequest requestFactory, Class<T> wantedReturnType) { Exception lastException = null; for (URI configServer : configServers) { - final CloseableHttpResponse response; - try { - response = client.execute(requestFactory.createRequest(configServer)); - } catch (Exception e) { - // Failure to communicate with a config server is not abnormal, as they are - // upgraded at the same time as Docker hosts. - if (e.getMessage().indexOf("(Connection refused)") > 0) { - NODE_ADMIN_LOGGER.info("Connection refused to " + configServer + " (upgrading?), will try next"); - } else { - NODE_ADMIN_LOGGER.warning("Failed to communicate with " + configServer + ", will try next: " + e.getMessage()); - } - lastException = e; - continue; - } - - try { - Optional<HttpException> retryableException = HttpException.handleStatusCode( - response.getStatusLine().getStatusCode(), - "Config server " + configServer); - if (retryableException.isPresent()) { - lastException = retryableException.get(); - continue; - } + try (CloseableHttpResponse response = client.execute(requestFactory.createRequest(configServer))) { + HttpException.handleStatusCode( + response.getStatusLine().getStatusCode(), "Config server " + configServer); try { return mapper.readValue(response.getEntity().getContent(), wantedReturnType); } catch (IOException e) { - throw new RuntimeException("Response didn't contain nodes element, failed parsing?", e); + throw new RuntimeException("Failed parse response from config server", e); } - } finally { - try { - response.close(); - } catch (IOException e) { - NODE_ADMIN_LOGGER.warning("Ignoring exception from closing response", e); + } catch (HttpException e) { + if (!e.isRetryable()) throw e; + lastException = e; + } catch (Exception e) { + // Failure to communicate with a config server is not abnormal during upgrades + if (e.getMessage().contains("(Connection refused)")) { + NODE_ADMIN_LOGGER.info("Connection refused to " + configServer + " (upgrading?), will try next"); + } else { + NODE_ADMIN_LOGGER.warning("Failed to communicate with " + configServer + ", will try next: " + e.getMessage()); } + lastException = e; } } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/HttpException.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/HttpException.java index d0f436d16b6..256fe38ec68 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/HttpException.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/HttpException.java @@ -2,7 +2,6 @@ package com.yahoo.vespa.hosted.node.admin.configserver; import javax.ws.rs.core.Response; -import java.util.Optional; /** * @author hakonhall @@ -10,35 +9,34 @@ import java.util.Optional; @SuppressWarnings("serial") public class HttpException extends RuntimeException { - public static class NotFoundException extends HttpException { - - public NotFoundException(String message) { - super(Response.Status.NOT_FOUND, message); - } + private final boolean isRetryable; + private HttpException(int statusCode, String message, boolean isRetryable) { + super("HTTP status code " + statusCode + ": " + message); + this.isRetryable = isRetryable; } - public static class ForbiddenException extends HttpException { - - public ForbiddenException(String message) { - super(Response.Status.FORBIDDEN, message); - } + private HttpException(Response.Status status, String message, boolean isRetryable) { + super(status.toString() + " (" + status.getStatusCode() + "): " + message); + this.isRetryable = isRetryable; + } + public boolean isRetryable() { + return isRetryable; } /** - * Returns empty on success. - * Returns an exception if the error is retriable. - * Throws an exception on a non-retriable error, like 404 Not Found. + * Returns on success. + * @throws HttpException for all non-expected status codes. */ - static Optional<HttpException> handleStatusCode(int statusCode, String message) { + static void handleStatusCode(int statusCode, String message) { Response.Status status = Response.Status.fromStatusCode(statusCode); if (status == null) { - return Optional.of(new HttpException(statusCode, message)); + throw new HttpException(statusCode, message, true); } switch (status.getFamily()) { - case SUCCESSFUL: return Optional.empty(); + case SUCCESSFUL: return; case CLIENT_ERROR: switch (status) { case FORBIDDEN: @@ -48,20 +46,24 @@ public class HttpException extends RuntimeException { case CONFLICT: // A response body is assumed to be present, and // will later be interpreted as an error. - return Optional.empty(); + return; } - throw new HttpException(statusCode, message); + throw new HttpException(status, message, false); } // Other errors like server-side errors are assumed to be retryable. - return Optional.of(new HttpException(status, message)); + throw new HttpException(status, message, true); } - private HttpException(int statusCode, String message) { - super("HTTP status code " + statusCode + ": " + message); + public static class NotFoundException extends HttpException { + public NotFoundException(String message) { + super(Response.Status.NOT_FOUND, message, false); + } } - private HttpException(Response.Status status, String message) { - super(status.toString() + " (" + status.getStatusCode() + "): " + message); + public static class ForbiddenException extends HttpException { + public ForbiddenException(String message) { + super(Response.Status.FORBIDDEN, message, false); + } } } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java index ea92ca9b56f..383c025e2cb 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java @@ -89,6 +89,13 @@ public class StorageMaintainer { "129600", "--crit", "1", "--coredir", environment.pathInNodeUnderVespaHome("var/crash/processing").toString()); configs.add(annotatedCheck(node, coredumpSchedule)); + // athenz certificate check + Path athenzCertExpiryCheckPath = environment.pathInNodeUnderVespaHome("libexec64/yms/yms_check_athenz_certs"); + SecretAgentCheckConfig athenzCertExpirySchedule = new SecretAgentCheckConfig("athenz-certificate-expiry", 60, + athenzCertExpiryCheckPath, "--threshold", "20") + .withRunAsUser("root"); + configs.add(annotatedCheck(node, athenzCertExpirySchedule)); + if (node.getNodeType() != NodeType.config) { // vespa-health Path vespaHealthCheckPath = environment.pathInNodeUnderVespaHome("libexec/yms/yms_check_vespa_health"); diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/StoredInteger.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/StoredInteger.java new file mode 100644 index 00000000000..a815515ac83 --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/StoredInteger.java @@ -0,0 +1,58 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.task.util.file; + +import com.yahoo.vespa.hosted.node.admin.component.TaskContext; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.file.Files; +import java.nio.file.NoSuchFileException; +import java.nio.file.Path; +import java.util.OptionalInt; +import java.util.function.Supplier; +import java.util.logging.Logger; + +/** + * Class wrapping an integer stored on disk + * + * @author freva + */ +public class StoredInteger implements Supplier<OptionalInt> { + + private static final Logger logger = Logger.getLogger(StoredInteger.class.getName()); + + private final Path path; + private OptionalInt value; + private boolean hasBeenRead = false; + + public StoredInteger(Path path) { + this.path = path; + } + + @Override + public OptionalInt get() { + if (!hasBeenRead) { + try { + String value = new String(Files.readAllBytes(path)); + this.value = OptionalInt.of(Integer.valueOf(value)); + } catch (NoSuchFileException e) { + this.value = OptionalInt.empty(); + } catch (IOException e) { + throw new UncheckedIOException("Failed to read integer in " + path, e); + } + hasBeenRead = true; + } + return value; + } + + public void write(TaskContext taskContext, int value) { + try { + Files.write(path, Integer.toString(value).getBytes()); + this.value = OptionalInt.of(value); + this.hasBeenRead = true; + taskContext.log(logger, "Stored new integer in %s: %d", path, value); + } catch (IOException e) { + throw new UncheckedIOException("Failed to store integer in " + path, e); + } + } +} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ChildProcessException.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ChildProcessException.java index 9f7aaab2060..cbc8ffbf1b7 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ChildProcessException.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ChildProcessException.java @@ -65,7 +65,7 @@ public abstract class ChildProcessException extends RuntimeException { if (possiblyHugeOutput.length() <= maxOutputPrefix + maxOutputSuffix + maxOutputSlack) { stringBuilder.append(possiblyHugeOutput); } else { - stringBuilder.append(possiblyHugeOutput.substring(0, maxOutputPrefix)) + stringBuilder.append(possiblyHugeOutput, 0, maxOutputPrefix) .append("... [") .append(possiblyHugeOutput.length() - maxOutputPrefix - maxOutputSuffix) .append(" chars omitted] ...") diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/Yum.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/Yum.java index d88c6f4ab33..5d60823d1c5 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/Yum.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/Yum.java @@ -36,6 +36,10 @@ public class Yum { return newYumCommand("install", packages, INSTALL_NOOP_PATTERN); } + /** + * @param packages A list of packages, each package being of the form name-1.2.3-1.el7.noarch, + * if no packages are given, will upgrade all installed packages + */ public GenericYumCommand upgrade(String... packages) { return newYumCommand("upgrade", packages, UPGRADE_NOOP_PATTERN); } @@ -70,7 +74,7 @@ public class Yum { this.packages = packages; this.commandOutputNoopPattern = commandOutputNoopPattern; - if (packages.isEmpty()) { + if (packages.isEmpty() && ! "upgrade".equals(yumCommand)) { throw new IllegalArgumentException("No packages specified"); } } diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/RebootTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/RebootTest.java index db14efdd5d2..a1af36f9c21 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/RebootTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/RebootTest.java @@ -11,10 +11,6 @@ import com.yahoo.vespa.hosted.provision.Node; import org.junit.Ignore; import org.junit.Test; -import java.util.Optional; - -import static org.hamcrest.core.Is.is; -import static org.hamcrest.junit.MatcherAssert.assertThat; import static org.junit.Assert.assertTrue; /** diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImplTest.java index 91c61623ee7..c348dc4c8b5 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImplTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImplTest.java @@ -25,6 +25,7 @@ import java.util.stream.Collectors; import static java.util.Arrays.asList; import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; @@ -152,7 +153,7 @@ public class NodeAdminImplTest { assertTrue(nodeAdmin.isFrozen()); assertTrue(nodeAdmin.subsystemFreezeDuration().isZero()); clock.advance(Duration.ofSeconds(1)); - assertTrue(nodeAdmin.subsystemFreezeDuration().equals(Duration.ofSeconds(1))); + assertEquals(Duration.ofSeconds(1), nodeAdmin.subsystemFreezeDuration()); // Unfreezing floors freeze duration assertTrue(nodeAdmin.setFrozen(false)); // Unfreeze everything @@ -164,7 +165,7 @@ public class NodeAdminImplTest { assertTrue(nodeAdmin.setFrozen(true)); assertTrue(nodeAdmin.subsystemFreezeDuration().isZero()); clock.advance(Duration.ofSeconds(1)); - assertTrue(nodeAdmin.subsystemFreezeDuration().equals(Duration.ofSeconds(1))); + assertEquals(Duration.ofSeconds(1), nodeAdmin.subsystemFreezeDuration()); } @Test diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/MakeDirectoryTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/MakeDirectoryTest.java index a83f3bbe7d4..b714ab539f6 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/MakeDirectoryTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/MakeDirectoryTest.java @@ -25,7 +25,7 @@ public class MakeDirectoryTest { private final FileSystem fileSystem = TestFileSystem.create(); private final TestTaskContext context = new TestTaskContext(); - private String path = "/parent/dir"; + private final String path = "/parent/dir"; private String permissions = "rwxr----x"; private String owner = "test-owner"; private String group = "test-group"; diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/process/CommandLineTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/process/CommandLineTest.java index 5bc45d7540e..a5eb0ab059b 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/process/CommandLineTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/process/CommandLineTest.java @@ -104,7 +104,6 @@ public class CommandLineTest { @Test public void programFails() { - TestChildProcess2 child = new TestChildProcess2(0, ""); terminal.expectCommand("foo 2>&1", 1, ""); try { commandLine.add("foo").execute(); diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumTest.java index d29d8741438..7f37336db70 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumTest.java @@ -9,6 +9,7 @@ import org.junit.Test; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; import static org.mockito.Mockito.mock; @@ -16,6 +17,7 @@ import static org.mockito.Mockito.mock; public class YumTest { private final TaskContext taskContext = mock(TaskContext.class); private final TestTerminal terminal = new TestTerminal(); + private final Yum yum = new Yum(terminal); @Before public void tearDown() { @@ -29,7 +31,6 @@ public class YumTest { 0, "foobar\nNothing to do\n"); - Yum yum = new Yum(terminal); assertFalse(yum .install("package-1", "package-2") .enableRepo("repo-name") @@ -43,7 +44,7 @@ public class YumTest { 0, "foobar\nNo packages marked for update\n"); - assertFalse(new Yum(terminal) + assertFalse(yum .upgrade("package-1", "package-2") .converge(taskContext)); } @@ -55,7 +56,7 @@ public class YumTest { 0, "foobar\nNo Packages marked for removal\n"); - assertFalse(new Yum(terminal) + assertFalse(yum .remove("package-1", "package-2") .converge(taskContext)); } @@ -67,7 +68,6 @@ public class YumTest { 0, "installing, installing"); - Yum yum = new Yum(terminal); assertTrue(yum .install("package-1", "package-2") .converge(taskContext)); @@ -80,7 +80,6 @@ public class YumTest { 0, "installing, installing"); - Yum yum = new Yum(terminal); assertTrue(yum .install("package-1", "package-2") .enableRepo("repo-name") @@ -94,7 +93,6 @@ public class YumTest { 1, "error"); - Yum yum = new Yum(terminal); yum.install("package-1", "package-2") .enableRepo("repo-name") .converge(taskContext); @@ -112,15 +110,26 @@ public class YumTest { "No package package-2 available.\n" + "Nothing to do\n"); - Yum yum = new Yum(terminal); Yum.GenericYumCommand install = yum.install("package-1", "package-2", "package-3"); try { install.converge(taskContext); fail(); } catch (Exception e) { - assertTrue(e.getCause() != null); + assertNotNull(e.getCause()); assertEquals("Unknown package: package-1", e.getCause().getMessage()); } } + + @Test(expected = IllegalArgumentException.class) + public void throwIfNoPackagesSpecified() { + yum.install(); + } + + @Test + public void allowToCallUpgradeWithNoPackages() { + terminal.expectCommand("yum upgrade --assumeyes 2>&1", 0, "OK"); + + yum.upgrade().converge(taskContext); + } }
\ No newline at end of file diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ApplicationMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ApplicationMaintainer.java index 3b7c4857f48..e3df6199d06 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ApplicationMaintainer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ApplicationMaintainer.java @@ -11,12 +11,15 @@ import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeRepository; import java.time.Duration; +import java.time.Instant; import java.util.LinkedHashSet; import java.util.List; import java.util.Optional; import java.util.Set; -import java.util.concurrent.Executor; -import java.util.concurrent.Executors; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; /** @@ -26,11 +29,15 @@ import java.util.stream.Collectors; public abstract class ApplicationMaintainer extends Maintainer { private final Deployer deployer; + private final List<ApplicationId> pendingDeployments = new CopyOnWriteArrayList<>(); // Use a fixed thread pool to avoid overload on config servers. Resource usage when deploying varies // a lot between applications, so doing one by one avoids issues where one or more resource-demanding // deployments happen simultaneously - private final Executor deploymentExecutor = Executors.newSingleThreadExecutor(new DaemonThreadFactory("node repo application maintainer")); + private final ThreadPoolExecutor deploymentExecutor = new ThreadPoolExecutor(1, 1, + 0L, TimeUnit.MILLISECONDS, + new LinkedBlockingQueue<>(), + new DaemonThreadFactory("node repo application maintainer")); protected ApplicationMaintainer(Deployer deployer, NodeRepository nodeRepository, Duration interval, JobControl jobControl) { super(nodeRepository, interval, jobControl); @@ -39,12 +46,15 @@ public abstract class ApplicationMaintainer extends Maintainer { @Override protected final void maintain() { - Set<ApplicationId> applications = applicationsNeedingMaintenance(); - for (ApplicationId application : applications) { - deploy(application); - } + applicationsNeedingMaintenance().forEach(this::deploy); + } + + /** Returns the number of deployments that are pending execution */ + public int pendingDeployments() { + return pendingDeployments.size(); } + /** Returns whether given application should be deployed at this moment in time */ protected boolean canDeployNow(ApplicationId application) { return true; } @@ -56,16 +66,21 @@ public abstract class ApplicationMaintainer extends Maintainer { * even when deployments are slow. */ protected void deploy(ApplicationId application) { + if (pendingDeployments.contains(application)) { + return;// Avoid queuing multiple deployments for same application + } + log.log(LogLevel.INFO, application + " will be deployed, last deploy time " + + getLastDeployTime(application)); + pendingDeployments.add(application); deploymentExecutor.execute(() -> deployWithLock(application)); } protected Deployer deployer() { return deployer; } - protected Set<ApplicationId> applicationsNeedingMaintenance() { return nodesNeedingMaintenance().stream() - .map(node -> node.allocation().get().owner()) - .collect(Collectors.toCollection(LinkedHashSet::new)); + .map(node -> node.allocation().get().owner()) + .collect(Collectors.toCollection(LinkedHashSet::new)); } /** @@ -75,7 +90,7 @@ public abstract class ApplicationMaintainer extends Maintainer { protected abstract List<Node> nodesNeedingMaintenance(); /** Redeploy this application. A lock will be taken for the duration of the deployment activation */ - final void deployWithLock(ApplicationId application) { + protected final void deployWithLock(ApplicationId application) { // An application might change its state between the time the set of applications is retrieved and the // time deployment happens. Lock the application and check if it's still active. // @@ -89,12 +104,31 @@ public abstract class ApplicationMaintainer extends Maintainer { deployment.get().activate(); } catch (RuntimeException e) { log.log(LogLevel.WARNING, "Exception on maintenance redeploy", e); + } finally { + pendingDeployments.remove(application); } } + /** Returns the last time application was deployed. Epoch is returned if the application has never been deployed. */ + protected final Instant getLastDeployTime(ApplicationId application) { + return deployer.lastDeployTime(application).orElse(Instant.EPOCH); + } + /** Returns true when application has at least one active node */ private boolean isActive(ApplicationId application) { return ! nodeRepository().getNodes(application, Node.State.active).isEmpty(); } + @Override + public void deconstruct() { + super.deconstruct(); + this.deploymentExecutor.shutdownNow(); + try { + // Give deployments in progress some time to complete + this.deploymentExecutor.awaitTermination(1, TimeUnit.MINUTES); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + } + } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/PeriodicApplicationMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/PeriodicApplicationMaintainer.java index ee5d6a04ddc..8b2d0a55cd8 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/PeriodicApplicationMaintainer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/PeriodicApplicationMaintainer.java @@ -3,17 +3,18 @@ package com.yahoo.vespa.hosted.provision.maintenance; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Deployer; -import com.yahoo.log.LogLevel; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeRepository; +import java.time.Clock; import java.time.Duration; import java.time.Instant; import java.util.Collections; import java.util.Comparator; +import java.util.LinkedHashSet; import java.util.List; -import java.util.Optional; import java.util.Set; +import java.util.stream.Collectors; /** * The application maintainer regularly redeploys all applications to make sure the node repo and application @@ -23,14 +24,17 @@ import java.util.Set; * @author bratseth */ public class PeriodicApplicationMaintainer extends ApplicationMaintainer { + private final Duration minTimeBetweenRedeployments; + private final Clock clock; private final Instant start; public PeriodicApplicationMaintainer(Deployer deployer, NodeRepository nodeRepository, Duration interval, Duration minTimeBetweenRedeployments, JobControl jobControl) { super(deployer, nodeRepository, interval, jobControl); this.minTimeBetweenRedeployments = minTimeBetweenRedeployments; - this.start = Instant.now(); + this.clock = nodeRepository.clock(); + this.start = clock.instant(); } @Override @@ -39,24 +43,17 @@ public class PeriodicApplicationMaintainer extends ApplicationMaintainer { return getLastDeployTime(application).isBefore(nodeRepository().clock().instant().minus(minTimeBetweenRedeployments)); } - // Returns the app that was deployed the longest time ago + // Returns the applications that need to be redeployed by this config server at this point in time. @Override protected Set<ApplicationId> applicationsNeedingMaintenance() { if (waitInitially()) return Collections.emptySet(); - Optional<ApplicationId> app = (nodesNeedingMaintenance().stream() - .map(node -> node.allocation().get().owner()) - .distinct() - .filter(this::shouldBeDeployedOnThisServer) - .min(Comparator.comparing(this::getLastDeployTime))) - .filter(this::canDeployNow); - app.ifPresent(applicationId -> log.log(LogLevel.INFO, applicationId + " will be deployed, last deploy time " + - getLastDeployTime(applicationId))); - return app.map(Collections::singleton).orElseGet(Collections::emptySet); - } - - private Instant getLastDeployTime(ApplicationId application) { - return deployer().lastDeployTime(application).orElse(Instant.EPOCH); + return nodesNeedingMaintenance().stream() + .map(node -> node.allocation().get().owner()) + .filter(this::shouldBeDeployedOnThisServer) + .filter(this::canDeployNow) + .sorted(Comparator.comparing(this::getLastDeployTime)) + .collect(Collectors.toCollection(LinkedHashSet::new)); } // We only know last deploy time for applications that were deployed on this config server, @@ -66,8 +63,8 @@ public class PeriodicApplicationMaintainer extends ApplicationMaintainer { } // TODO: Do not start deploying until some time has gone (ideally only until bootstrap of config server is finished) - protected boolean waitInitially() { - return Instant.now().isBefore(start.plus(minTimeBetweenRedeployments)); + private boolean waitInitially() { + return clock.instant().isBefore(start.plus(minTimeBetweenRedeployments)); } @Override diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockDeployer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockDeployer.java index 99beab50e16..299dc66c547 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockDeployer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockDeployer.java @@ -20,6 +20,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.concurrent.locks.ReentrantLock; import java.util.stream.Collectors; /** @@ -35,6 +36,7 @@ public class MockDeployer implements Deployer { public int redeployments = 0; private final Clock clock; + private final ReentrantLock lock = new ReentrantLock(); @Inject @SuppressWarnings("unused") @@ -54,6 +56,10 @@ public class MockDeployer implements Deployer { this.applications = applications; } + public ReentrantLock lock() { + return lock; + } + @Override public Optional<Deployment> deployFromLocalActive(ApplicationId id, boolean bootstrap) { return deployFromLocalActive(id, Duration.ofSeconds(60)); @@ -61,8 +67,17 @@ public class MockDeployer implements Deployer { @Override public Optional<Deployment> deployFromLocalActive(ApplicationId id, Duration timeout) { - lastDeployTimes.put(id, clock.instant()); - return Optional.of(new MockDeployment(provisioner, applications.get(id))); + try { + lock.lockInterruptibly(); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + try { + lastDeployTimes.put(id, clock.instant()); + return Optional.of(new MockDeployment(provisioner, applications.get(id))); + } finally { + lock.unlock(); + } } @Override diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/PeriodicApplicationMaintainerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/PeriodicApplicationMaintainerTest.java index 2ddb2e0d004..7e48edfc805 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/PeriodicApplicationMaintainerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/PeriodicApplicationMaintainerTest.java @@ -29,6 +29,7 @@ import com.yahoo.vespa.hosted.provision.provisioning.FlavorConfigBuilder; import com.yahoo.vespa.hosted.provision.provisioning.NodeRepositoryProvisioner; import com.yahoo.vespa.hosted.provision.testutils.MockDeployer; import com.yahoo.vespa.hosted.provision.testutils.MockNameResolver; +import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -41,6 +42,7 @@ import java.util.Map; import java.util.Optional; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; /** * @author bratseth @@ -51,26 +53,36 @@ public class PeriodicApplicationMaintainerTest { private NodeRepository nodeRepository; private Fixture fixture; + private ManualClock clock; @Before public void before() { Curator curator = new MockCurator(); Zone zone = new Zone(Environment.prod, RegionName.from("us-east")); - this.nodeRepository = new NodeRepository(nodeFlavors, curator, new ManualClock(), zone, + this.clock = new ManualClock(); + this.nodeRepository = new NodeRepository(nodeFlavors, curator, clock, zone, new MockNameResolver().mockAnyLookup(), new DockerImage("docker-registry.domain.tld:8080/dist/vespa"), true); this.fixture = new Fixture(zone, nodeRepository, nodeFlavors, curator); - } - @Test - public void test_application_maintenance() { createReadyNodes(15, nodeRepository, nodeFlavors); createHostNodes(2, nodeRepository, nodeFlavors); + } + + @After + public void after() { + this.fixture.maintainer.deconstruct(); + } + @Test(timeout = 60_000) + public void test_application_maintenance() { // Create applications fixture.activate(); + // Exhaust initial wait period + clock.advance(Duration.ofMinutes(30).plus(Duration.ofSeconds(1))); + // Fail and park some nodes nodeRepository.fail(nodeRepository.getNodes(fixture.app1).get(3).hostname(), Agent.system, "Failing to unit test"); nodeRepository.fail(nodeRepository.getNodes(fixture.app2).get(0).hostname(), Agent.system, "Failing to unit test"); @@ -102,7 +114,7 @@ public class PeriodicApplicationMaintainerTest { 0, fixture.getNodes(Node.State.active).retired().size()); // Cause maintenance deployment which will update the applications with the re-activated nodes - ((ManualClock)nodeRepository.clock()).advance(Duration.ofMinutes(35)); // Otherwise redeploys are inhibited + clock.advance(Duration.ofMinutes(35)); // Otherwise redeploys are inhibited fixture.runApplicationMaintainer(); assertEquals("Superflous content nodes are retired", reactivatedInApp2, fixture.getNodes(Node.State.active).retired().size()); @@ -110,11 +122,8 @@ public class PeriodicApplicationMaintainerTest { reactivatedInApp1, fixture.getNodes(Node.State.inactive).size()); } - @Test + @Test(timeout = 60_000) public void deleted_application_is_not_reactivated() { - createReadyNodes(15, nodeRepository, nodeFlavors); - createHostNodes(2, nodeRepository, nodeFlavors); - // Create applications fixture.activate(); @@ -126,36 +135,81 @@ public class PeriodicApplicationMaintainerTest { assertEquals(fixture.wantedNodesApp2, nodeRepository.getNodes(fixture.app2, Node.State.inactive).size()); // Nodes belonging to app2 are inactive after maintenance - fixture.runApplicationMaintainer(Optional.of(frozenActiveNodes)); + fixture.maintainer.setOverriddenNodesNeedingMaintenance(frozenActiveNodes); + fixture.runApplicationMaintainer(); assertEquals("Inactive nodes were incorrectly activated after maintenance", fixture.wantedNodesApp2, nodeRepository.getNodes(fixture.app2, Node.State.inactive).size()); } - @Test + @Test(timeout = 60_000) public void application_deploy_inhibits_redeploy_for_a_while() { - ManualClock clock = (ManualClock)nodeRepository.clock(); - createReadyNodes(15, nodeRepository, nodeFlavors); - createHostNodes(2, nodeRepository, nodeFlavors); - - // Create applications fixture.activate(); + + // Holds off on deployments a while after starting + fixture.runApplicationMaintainer(); + assertFalse("No deployment expected", fixture.deployer.lastDeployTime(fixture.app1).isPresent()); + assertFalse("No deployment expected", fixture.deployer.lastDeployTime(fixture.app2).isPresent()); + // Exhaust initial wait period + clock.advance(Duration.ofMinutes(30).plus(Duration.ofSeconds(1))); + + // First deployment of applications fixture.runApplicationMaintainer(); Instant firstDeployTime = clock.instant(); assertEquals(firstDeployTime, fixture.deployer.lastDeployTime(fixture.app1).get()); assertEquals(firstDeployTime, fixture.deployer.lastDeployTime(fixture.app2).get()); - ((ManualClock) nodeRepository.clock()).advance(Duration.ofMinutes(5)); + clock.advance(Duration.ofMinutes(5)); fixture.runApplicationMaintainer(); // Too soon: Not redeployed: assertEquals(firstDeployTime, fixture.deployer.lastDeployTime(fixture.app1).get()); assertEquals(firstDeployTime, fixture.deployer.lastDeployTime(fixture.app2).get()); - ((ManualClock) nodeRepository.clock()).advance(Duration.ofMinutes(30)); + clock.advance(Duration.ofMinutes(30)); fixture.runApplicationMaintainer(); // Redeployed: assertEquals(clock.instant(), fixture.deployer.lastDeployTime(fixture.app1).get()); assertEquals(clock.instant(), fixture.deployer.lastDeployTime(fixture.app2).get()); } + @Test(timeout = 60_000) + public void queues_all_eligible_applications_for_deployment() throws Exception { + fixture.activate(); + + // Exhaust initial wait period + clock.advance(Duration.ofMinutes(30).plus(Duration.ofSeconds(1))); + + // Lock deployer to simulate slow deployments + fixture.deployer.lock().lockInterruptibly(); + + try { + // Queues all eligible applications + assertEquals(2, fixture.maintainer.applicationsNeedingMaintenance().size()); + fixture.runApplicationMaintainer(false); + assertEquals(2, fixture.maintainer.pendingDeployments()); + + // Enough time passes to make applications eligible for another periodic deployment + clock.advance(Duration.ofMinutes(30).plus(Duration.ofSeconds(1))); + fixture.runApplicationMaintainer(false); + + // Deployments are not re-queued as previous deployments are still pending + assertEquals(2, fixture.maintainer.pendingDeployments()); + + // Slow deployments complete + fixture.deployer.lock().unlock(); + fixture.runApplicationMaintainer(); + Instant deployTime = clock.instant(); + assertEquals(deployTime, fixture.deployer.lastDeployTime(fixture.app1).get()); + assertEquals(deployTime, fixture.deployer.lastDeployTime(fixture.app2).get()); + + // Too soon: Already deployed recently + clock.advance(Duration.ofMinutes(5)); + assertEquals(0, fixture.maintainer.applicationsNeedingMaintenance().size()); + } finally { + if (fixture.deployer.lock().isHeldByCurrentThread()) { + fixture.deployer.lock().unlock(); + } + } + } + private void createReadyNodes(int count, NodeRepository nodeRepository, NodeFlavors nodeFlavors) { List<Node> nodes = new ArrayList<>(count); for (int i = 0; i < count; i++) @@ -179,7 +233,7 @@ public class PeriodicApplicationMaintainerTest { final NodeRepository nodeRepository; final NodeRepositoryProvisioner provisioner; final Curator curator; - final Deployer deployer; + final MockDeployer deployer; final ApplicationId app1 = ApplicationId.from(TenantName.from("foo1"), ApplicationName.from("bar"), InstanceName.from("fuz")); final ApplicationId app2 = ApplicationId.from(TenantName.from("foo2"), ApplicationName.from("bar"), InstanceName.from("fuz")); @@ -188,6 +242,8 @@ public class PeriodicApplicationMaintainerTest { final int wantedNodesApp1 = 5; final int wantedNodesApp2 = 7; + private final TestablePeriodicApplicationMaintainer maintainer; + Fixture(Zone zone, NodeRepository nodeRepository, NodeFlavors flavors, Curator curator) { this.nodeRepository = nodeRepository; this.curator = curator; @@ -199,6 +255,8 @@ public class PeriodicApplicationMaintainerTest { apps.put(app2, new MockDeployer.ApplicationContext(app2, clusterApp2, Capacity.fromNodeCount(wantedNodesApp2, Optional.of("default"), false, true), 1)); this.deployer = new MockDeployer(provisioner, nodeRepository.clock(), apps); + this.maintainer = new TestablePeriodicApplicationMaintainer(deployer, nodeRepository, Duration.ofDays(1), // Long duration to prevent scheduled runs during test + Duration.ofMinutes(30)); } void activate() { @@ -222,16 +280,12 @@ public class PeriodicApplicationMaintainerTest { } void runApplicationMaintainer() { - runApplicationMaintainer(Optional.empty()); + runApplicationMaintainer(true); } - void runApplicationMaintainer(Optional<List<Node>> overriddenNodesNeedingMaintenance) { - TestablePeriodicApplicationMaintainer maintainer = - new TestablePeriodicApplicationMaintainer(deployer, nodeRepository, Duration.ofMinutes(1), - Duration.ofMinutes(30), overriddenNodesNeedingMaintenance); - // Need to run twice, as only one app is deployed per run - maintainer.run(); + void runApplicationMaintainer(boolean waitForDeployments) { maintainer.run(); + while (waitForDeployments && fixture.maintainer.pendingDeployments() != 0); } NodeList getNodes(Node.State ... states) { @@ -240,36 +294,32 @@ public class PeriodicApplicationMaintainerTest { } - public static class TestablePeriodicApplicationMaintainer extends PeriodicApplicationMaintainer { + private static class TestablePeriodicApplicationMaintainer extends PeriodicApplicationMaintainer { - private Optional<List<Node>> overriddenNodesNeedingMaintenance; - - TestablePeriodicApplicationMaintainer(Deployer deployer, NodeRepository nodeRepository, Duration interval, - Duration minTimeBetweenRedeployments, Optional<List<Node>> overriddenNodesNeedingMaintenance) { - super(deployer, nodeRepository, interval, minTimeBetweenRedeployments, new JobControl(nodeRepository.database())); + private List<Node> overriddenNodesNeedingMaintenance; + + TestablePeriodicApplicationMaintainer setOverriddenNodesNeedingMaintenance(List<Node> overriddenNodesNeedingMaintenance) { this.overriddenNodesNeedingMaintenance = overriddenNodesNeedingMaintenance; + return this; } - @Override - protected void deploy(ApplicationId application) { - deployWithLock(application); + TestablePeriodicApplicationMaintainer(Deployer deployer, NodeRepository nodeRepository, Duration interval, + Duration minTimeBetweenRedeployments) { + super(deployer, nodeRepository, interval, minTimeBetweenRedeployments, new JobControl(nodeRepository.database())); } @Override protected List<Node> nodesNeedingMaintenance() { - if (overriddenNodesNeedingMaintenance.isPresent()) - return overriddenNodesNeedingMaintenance.get(); - return super.nodesNeedingMaintenance(); + return overriddenNodesNeedingMaintenance != null + ? overriddenNodesNeedingMaintenance + : super.nodesNeedingMaintenance(); } + @Override protected boolean shouldBeDeployedOnThisServer(ApplicationId application) { return true; } - protected boolean waitInitially() { - return false; - } - } } diff --git a/searchcore/CMakeLists.txt b/searchcore/CMakeLists.txt index 89e6493cfbc..9d996d96dc7 100644 --- a/searchcore/CMakeLists.txt +++ b/searchcore/CMakeLists.txt @@ -129,6 +129,7 @@ vespa_define_module( src/tests/proton/proton src/tests/proton/proton_config_fetcher src/tests/proton/proton_configurer + src/tests/proton/proton_disk_layout src/tests/proton/reference/gid_to_lid_change_handler src/tests/proton/reference/gid_to_lid_change_listener src/tests/proton/reference/gid_to_lid_change_registrator diff --git a/searchcore/src/tests/proton/attribute/attribute_directory/attribute_directory_test.cpp b/searchcore/src/tests/proton/attribute/attribute_directory/attribute_directory_test.cpp index 37cf201e354..f95ea478ce1 100644 --- a/searchcore/src/tests/proton/attribute/attribute_directory/attribute_directory_test.cpp +++ b/searchcore/src/tests/proton/attribute/attribute_directory/attribute_directory_test.cpp @@ -137,6 +137,7 @@ struct Fixture : public DirectoryHandler EXPECT_TRUE(hasAttributeDir(dir)); auto writer = dir->getWriter(); writer->createInvalidSnapshot(serialNum); + vespalib::mkdir(writer->getSnapshotDir(serialNum), false); writer->markValidSnapshot(serialNum); TEST_DO(assertAttributeDiskDir("foo")); } @@ -162,6 +163,7 @@ struct Fixture : public DirectoryHandler auto dir = createFooAttrDir(); auto writer = dir->getWriter(); writer->createInvalidSnapshot(serialNum); + vespalib::mkdir(writer->getSnapshotDir(serialNum), false); writer->markValidSnapshot(serialNum); } @@ -208,8 +210,10 @@ TEST_F("Test that we can prune attribute snapshots", Fixture) TEST_DO(f.assertNotAttributeDiskDir("foo")); auto writer = dir->getWriter(); writer->createInvalidSnapshot(2); + vespalib::mkdir(writer->getSnapshotDir(2), false); writer->markValidSnapshot(2); writer->createInvalidSnapshot(4); + vespalib::mkdir(writer->getSnapshotDir(4), false); writer->markValidSnapshot(4); writer.reset(); TEST_DO(f.assertAttributeDiskDir("foo")); diff --git a/searchcore/src/tests/proton/proton_configurer/proton_configurer_test.cpp b/searchcore/src/tests/proton/proton_configurer/proton_configurer_test.cpp index a89f2e3f0ff..dfb1268aaa6 100644 --- a/searchcore/src/tests/proton/proton_configurer/proton_configurer_test.cpp +++ b/searchcore/src/tests/proton/proton_configurer/proton_configurer_test.cpp @@ -16,6 +16,7 @@ #include <vespa/searchcore/proton/server/proton_config_snapshot.h> #include <vespa/searchcore/proton/server/proton_configurer.h> #include <vespa/searchcore/proton/server/i_proton_configurer_owner.h> +#include <vespa/searchcore/proton/server/i_proton_disk_layout.h> #include <vespa/searchsummary/config/config-juniperrc.h> #include <vespa/searchcore/config/config-ranking-constants.h> #include <vespa/vespalib/testkit/testapp.h> @@ -224,18 +225,33 @@ struct MyDocumentDBConfigOwner : public DocumentDBConfigOwner void reconfigure(const DocumentDBConfig::SP & config) override; }; -struct MyProtonConfigurerOwner : public IProtonConfigurerOwner +struct MyLog +{ + std::vector<vespalib::string> _log; + + MyLog() + : _log() + { + } + + void appendLog(vespalib::string logEntry) + { + _log.emplace_back(logEntry); + } +}; + +struct MyProtonConfigurerOwner : public IProtonConfigurerOwner, + public MyLog { using InitializeThreads = std::shared_ptr<vespalib::ThreadStackExecutorBase>; vespalib::ThreadStackExecutor _executor; std::map<DocTypeName, std::shared_ptr<MyDocumentDBConfigOwner>> _dbs; - std::vector<vespalib::string> _log; MyProtonConfigurerOwner() : IProtonConfigurerOwner(), + MyLog(), _executor(1, 128 * 1024), - _dbs(), - _log() + _dbs() { } virtual ~MyProtonConfigurerOwner() { } @@ -286,17 +302,48 @@ MyDocumentDBConfigOwner::reconfigure(const DocumentDBConfig::SP & config) _owner.reconfigureDocumentDB(_name, config); } +struct MyProtonDiskLayout : public IProtonDiskLayout +{ + MyLog &_log; + + MyProtonDiskLayout(MyLog &myLog) + : _log(myLog) + { + } + void remove(const DocTypeName &docTypeName) override { + std::ostringstream os; + os << "remove dbdir " << docTypeName.getName(); + _log.appendLog(os.str()); + } + void initAndPruneUnused(const std::set<DocTypeName> &docTypeNames) override { + std::ostringstream os; + os << "initial dbs "; + bool first = true; + for (const auto &docTypeName : docTypeNames) { + if (!first) { + os << ","; + } + first = false; + os << docTypeName.getName(); + } + _log.appendLog(os.str()); + } +}; + struct Fixture { MyProtonConfigurerOwner _owner; ConfigFixture _config; + std::unique_ptr<IProtonDiskLayout> _diskLayout; ProtonConfigurer _configurer; Fixture() : _owner(), _config("test"), - _configurer(_owner._executor, _owner) + _diskLayout(), + _configurer(_owner._executor, _owner, _diskLayout) { + _diskLayout = std::make_unique<MyProtonDiskLayout>(_owner); } ~Fixture() { } @@ -338,14 +385,14 @@ TEST_F("require that nothing is applied before initial config", Fixture()) TEST_F("require that initial config is applied", Fixture()) { f.applyInitialConfig(); - TEST_DO(f1.assertLog({"apply config 2", "add db _alwaysthere_ 2"})); + TEST_DO(f1.assertLog({"initial dbs _alwaysthere_", "apply config 2", "add db _alwaysthere_ 2"})); } TEST_F("require that new config is blocked", Fixture()) { f.applyInitialConfig(); f.reconfigure(); - TEST_DO(f1.assertLog({"apply config 2", "add db _alwaysthere_ 2"})); + TEST_DO(f1.assertLog({"initial dbs _alwaysthere_", "apply config 2", "add db _alwaysthere_ 2"})); } TEST_F("require that new config can be unblocked", Fixture()) @@ -353,14 +400,14 @@ TEST_F("require that new config can be unblocked", Fixture()) f.applyInitialConfig(); f.reconfigure(); f.allowReconfig(); - TEST_DO(f1.assertLog({"apply config 2", "add db _alwaysthere_ 2", "apply config 3", "reconf db _alwaysthere_ 3"})); + TEST_DO(f1.assertLog({"initial dbs _alwaysthere_", "apply config 2", "add db _alwaysthere_ 2", "apply config 3", "reconf db _alwaysthere_ 3"})); } TEST_F("require that initial config is not reapplied due to config unblock", Fixture()) { f.applyInitialConfig(); f.allowReconfig(); - TEST_DO(f1.assertLog({"apply config 2", "add db _alwaysthere_ 2"})); + TEST_DO(f1.assertLog({"initial dbs _alwaysthere_", "apply config 2", "add db _alwaysthere_ 2"})); } TEST_F("require that we can add document db", Fixture()) @@ -369,7 +416,7 @@ TEST_F("require that we can add document db", Fixture()) f.allowReconfig(); f.addDocType("foobar"); f.reconfigure(); - TEST_DO(f1.assertLog({"apply config 2", "add db _alwaysthere_ 2", "apply config 3","reconf db _alwaysthere_ 3", "add db foobar 3"})); + TEST_DO(f1.assertLog({"initial dbs _alwaysthere_", "apply config 2", "add db _alwaysthere_ 2", "apply config 3","reconf db _alwaysthere_ 3", "add db foobar 3"})); } TEST_F("require that we can remove document db", Fixture()) @@ -379,7 +426,7 @@ TEST_F("require that we can remove document db", Fixture()) f.allowReconfig(); f.removeDocType("foobar"); f.reconfigure(); - TEST_DO(f1.assertLog({"apply config 2", "add db _alwaysthere_ 2", "add db foobar 2", "apply config 3","reconf db _alwaysthere_ 3", "remove db foobar"})); + TEST_DO(f1.assertLog({"initial dbs _alwaysthere_,foobar", "apply config 2", "add db _alwaysthere_ 2", "add db foobar 2", "apply config 3","reconf db _alwaysthere_ 3", "remove db foobar", "remove dbdir foobar"})); } TEST_F("require that document db adds and reconfigs are intermingled", Fixture()) @@ -392,7 +439,7 @@ TEST_F("require that document db adds and reconfigs are intermingled", Fixture() f.addDocType("foobar"); f.addDocType("zbar"); f.reconfigure(); - TEST_DO(f1.assertLog({"apply config 2", "add db _alwaysthere_ 2", "add db foobar 2", "apply config 3","reconf db _alwaysthere_ 3", "add db abar 3", "reconf db foobar 3", "add db zbar 3"})); + TEST_DO(f1.assertLog({"initial dbs _alwaysthere_,foobar", "apply config 2", "add db _alwaysthere_ 2", "add db foobar 2", "apply config 3","reconf db _alwaysthere_ 3", "add db abar 3", "reconf db foobar 3", "add db zbar 3"})); } TEST_F("require that document db removes are applied at end", Fixture()) @@ -403,7 +450,7 @@ TEST_F("require that document db removes are applied at end", Fixture()) f.allowReconfig(); f.removeDocType("abar"); f.reconfigure(); - TEST_DO(f1.assertLog({"apply config 2", "add db _alwaysthere_ 2", "add db abar 2", "add db foobar 2", "apply config 3","reconf db _alwaysthere_ 3", "reconf db foobar 3", "remove db abar"})); + TEST_DO(f1.assertLog({"initial dbs _alwaysthere_,abar,foobar", "apply config 2", "add db _alwaysthere_ 2", "add db abar 2", "add db foobar 2", "apply config 3","reconf db _alwaysthere_ 3", "reconf db foobar 3", "remove db abar", "remove dbdir abar"})); } TEST_F("require that new configs can be blocked again", Fixture()) @@ -413,7 +460,7 @@ TEST_F("require that new configs can be blocked again", Fixture()) f.allowReconfig(); f.disableReconfig(); f.reconfigure(); - TEST_DO(f1.assertLog({"apply config 2", "add db _alwaysthere_ 2", "apply config 3", "reconf db _alwaysthere_ 3"})); + TEST_DO(f1.assertLog({"initial dbs _alwaysthere_", "apply config 2", "add db _alwaysthere_ 2", "apply config 3", "reconf db _alwaysthere_ 3"})); } TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchcore/src/tests/proton/proton_disk_layout/CMakeLists.txt b/searchcore/src/tests/proton/proton_disk_layout/CMakeLists.txt new file mode 100644 index 00000000000..f63fa21a954 --- /dev/null +++ b/searchcore/src/tests/proton/proton_disk_layout/CMakeLists.txt @@ -0,0 +1,9 @@ +# Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(searchcore_proton_disk_layout_test_app TEST + SOURCES + proton_disk_layout_test.cpp + DEPENDS + searchcore_server + searchcore_fconfig +) +vespa_add_test(NAME searchcore_proton_disk_layout_test_app COMMAND searchcore_proton_disk_layout_test_app) diff --git a/searchcore/src/tests/proton/proton_disk_layout/proton_disk_layout_test.cpp b/searchcore/src/tests/proton/proton_disk_layout/proton_disk_layout_test.cpp new file mode 100644 index 00000000000..edb4250ce76 --- /dev/null +++ b/searchcore/src/tests/proton/proton_disk_layout/proton_disk_layout_test.cpp @@ -0,0 +1,178 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/searchcore/proton/server/proton_disk_layout.h> +#include <vespa/searchcore/proton/common/doctypename.h> +#include <vespa/searchlib/index/dummyfileheadercontext.h> +#include <vespa/searchlib/transactionlog/translogserver.h> +#include <vespa/searchlib/transactionlog/translogclient.h> +#include <vespa/vespalib/io/fileutil.h> +#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/test/insertion_operators.h> +#include <vespa/vespalib/util/stringfmt.h> + +using search::index::DummyFileHeaderContext; +using search::transactionlog::TransLogClient; +using search::transactionlog::TransLogServer; +using proton::DocTypeName; +using proton::ProtonDiskLayout; + +static constexpr unsigned int tlsPort = 9018; + +static const vespalib::string baseDir("testdb"); +static const vespalib::string documentsDir(baseDir + "/documents"); + +struct FixtureBase +{ + FixtureBase() { vespalib::rmdir(baseDir, true); } + ~FixtureBase() { vespalib::rmdir(baseDir, true); } +}; + +struct DiskLayoutFixture { + DummyFileHeaderContext _fileHeaderContext; + TransLogServer _tls; + vespalib::string _tlsSpec; + ProtonDiskLayout _diskLayout; + + DiskLayoutFixture(); + ~DiskLayoutFixture(); + + void createDirs(const std::set<vespalib::string> &dirs) { + for (const auto &dir : dirs) { + vespalib::mkdir(documentsDir + "/" + dir, false); + } + } + void createDomains(const std::set<vespalib::string> &domains) { + TransLogClient tlc(_tlsSpec); + for (const auto &domain : domains) { + ASSERT_TRUE(tlc.create(domain)); + } + } + + std::set<vespalib::string> listDomains() { + std::vector<vespalib::string> domainVector; + TransLogClient tlc(_tlsSpec); + ASSERT_TRUE(tlc.listDomains(domainVector)); + std::set<vespalib::string> domains; + for (const auto &domain : domainVector) { + domains.emplace(domain); + } + return domains; + } + + std::set<vespalib::string> listDirs() { + std::set<vespalib::string> dirs; + auto names = vespalib::listDirectory(documentsDir); + for (const auto &name : names) { + if (vespalib::isDirectory(documentsDir + "/" + name)) { + dirs.emplace(name); + } + } + return dirs; + } + + void initAndPruneUnused(const std::set<vespalib::string> names) + { + std::set<DocTypeName> docTypeNames; + for (const auto &name: names) { + docTypeNames.emplace(name); + } + _diskLayout.initAndPruneUnused(docTypeNames); + } + + void assertDirs(const std::set<vespalib::string> &expDirs) { + EXPECT_EQUAL(expDirs, listDirs()); + } + + void assertDomains(const std::set<vespalib::string> &expDomains) + { + EXPECT_EQUAL(expDomains, listDomains()); + } +}; + +DiskLayoutFixture::DiskLayoutFixture() + : _fileHeaderContext(), + _tls("tls", tlsPort, baseDir, _fileHeaderContext), + _tlsSpec(vespalib::make_string("tcp/localhost:%u", tlsPort)), + _diskLayout(baseDir, _tlsSpec) +{ +} + +DiskLayoutFixture::~DiskLayoutFixture() = default; + +struct Fixture : public FixtureBase, public DiskLayoutFixture +{ + Fixture() + : FixtureBase(), + DiskLayoutFixture() + { + } +}; + +TEST_F("require that empty config is ok", Fixture) { + TEST_DO(f.assertDirs({})); + TEST_DO(f.assertDomains({})); +} + +TEST_F("require that disk layout is preserved", FixtureBase) +{ + { + DiskLayoutFixture diskLayout; + diskLayout.createDirs({"foo", "bar"}); + diskLayout.createDomains({"bar", "baz"}); + } + { + DiskLayoutFixture diskLayout; + TEST_DO(diskLayout.assertDirs({"foo", "bar"})); + TEST_DO(diskLayout.assertDomains({"bar", "baz"})); + } +} + +TEST_F("require that used dir is preserved", Fixture) +{ + f.createDirs({"foo"}); + f.createDomains({"foo"}); + f.initAndPruneUnused({"foo"}); + TEST_DO(f.assertDirs({"foo"})); + TEST_DO(f.assertDomains({"foo"})); +} + +TEST_F("require that unused dir is removed", Fixture) +{ + f.createDirs({"foo"}); + f.createDomains({"foo"}); + f.initAndPruneUnused({"bar"}); + TEST_DO(f.assertDirs({})); + TEST_DO(f.assertDomains({})); +} + +TEST_F("require that interrupted remove is completed", Fixture) +{ + f.createDirs({"foo.removed"}); + f.createDomains({"foo"}); + f.initAndPruneUnused({"foo"}); + TEST_DO(f.assertDirs({})); + TEST_DO(f.assertDomains({})); +} + +TEST_F("require that early interrupted remove is completed", Fixture) +{ + f.createDirs({"foo", "foo.removed"}); + f.createDomains({"foo"}); + f.initAndPruneUnused({"foo"}); + TEST_DO(f.assertDirs({})); + TEST_DO(f.assertDomains({})); +} + +TEST_F("require that live document db dir remove works", Fixture) +{ + f.createDirs({"foo"}); + f.createDomains({"foo"}); + f.initAndPruneUnused({"foo"}); + TEST_DO(f.assertDirs({"foo"})); + TEST_DO(f.assertDomains({"foo"})); + f._diskLayout.remove(DocTypeName("foo")); + TEST_DO(f.assertDirs({})); + TEST_DO(f.assertDomains({})); +} + +TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchcore/src/tests/proton/reprocessing/attribute_reprocessing_initializer/attribute_reprocessing_initializer_test.cpp b/searchcore/src/tests/proton/reprocessing/attribute_reprocessing_initializer/attribute_reprocessing_initializer_test.cpp index 7be774f7291..72e558fd25f 100644 --- a/searchcore/src/tests/proton/reprocessing/attribute_reprocessing_initializer/attribute_reprocessing_initializer_test.cpp +++ b/searchcore/src/tests/proton/reprocessing/attribute_reprocessing_initializer/attribute_reprocessing_initializer_test.cpp @@ -294,9 +294,9 @@ TEST("require that added attribute aspect with flushed attribute after interrupt auto dir = diskLayout->createAttributeDir("a"); auto writer = dir->getWriter(); writer->createInvalidSnapshot(INIT_SERIAL_NUM); - writer->markValidSnapshot(INIT_SERIAL_NUM); auto snapshotdir = writer->getSnapshotDir(INIT_SERIAL_NUM); vespalib::mkdir(snapshotdir); + writer->markValidSnapshot(INIT_SERIAL_NUM); auto av = AttributeFactory::createAttribute(snapshotdir + "/a", Config(BasicType::STRING)); av->save(); diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_directory.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attribute_directory.cpp index f2e4ac4905d..2bbc6c99dc0 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_directory.cpp +++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_directory.cpp @@ -112,6 +112,7 @@ AttributeDirectory::createInvalidSnapshot(SerialNum serialNum) if (empty()) { vespalib::string dirName(getDirName()); vespalib::mkdir(dirName, false); + vespalib::File::sync(vespalib::dirname(dirName)); } { std::lock_guard<std::mutex> guard(_mutex); @@ -130,6 +131,9 @@ AttributeDirectory::markValidSnapshot(SerialNum serialNum) assert(snap.syncToken == serialNum); _snapInfo.validateSnapshot(serialNum); } + vespalib::string snapshotDir(getSnapshotDir(serialNum)); + vespalib::File::sync(snapshotDir); + vespalib::File::sync(dirname(snapshotDir)); saveSnapInfo(); } @@ -178,6 +182,7 @@ AttributeDirectory::removeInvalidSnapshots() vespalib::rmdir(subDir, true); } if (!toRemove.empty()) { + vespalib::File::sync(getDirName()); { std::lock_guard<std::mutex> guard(_mutex); for (const auto &serialNum : toRemove) { @@ -194,6 +199,7 @@ AttributeDirectory::removeDiskDir() if (empty()) { vespalib::string dirName(getDirName()); vespalib::rmdir(dirName, true); + vespalib::File::sync(vespalib::dirname(dirName)); return true; } return false; diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attributedisklayout.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attributedisklayout.cpp index bb2f99d077b..a675927b85f 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attributedisklayout.cpp +++ b/searchcore/src/vespa/searchcore/proton/attribute/attributedisklayout.cpp @@ -14,6 +14,7 @@ AttributeDiskLayout::AttributeDiskLayout(const vespalib::string &baseDir, Privat _dirs() { vespalib::mkdir(_baseDir, false); + vespalib::File::sync(vespalib::dirname(_baseDir)); } AttributeDiskLayout::~AttributeDiskLayout() diff --git a/searchcore/src/vespa/searchcore/proton/index/index_manager_initializer.cpp b/searchcore/src/vespa/searchcore/proton/index/index_manager_initializer.cpp index b8f2947c9b4..ad233a66d1f 100644 --- a/searchcore/src/vespa/searchcore/proton/index/index_manager_initializer.cpp +++ b/searchcore/src/vespa/searchcore/proton/index/index_manager_initializer.cpp @@ -45,6 +45,7 @@ IndexManagerInitializer::run() LOG(debug, "About to create proton::IndexManager with %u index field(s)", _schema.getNumIndexFields()); vespalib::mkdir(_baseDir, false); + vespalib::File::sync(vespalib::dirname(_baseDir)); *_indexManager = std::make_shared<proton::IndexManager> (_baseDir, _warmupCfg, diff --git a/searchcore/src/vespa/searchcore/proton/metrics/trans_log_server_metrics.cpp b/searchcore/src/vespa/searchcore/proton/metrics/trans_log_server_metrics.cpp index 00a7e9b9140..c2624719d81 100644 --- a/searchcore/src/vespa/searchcore/proton/metrics/trans_log_server_metrics.cpp +++ b/searchcore/src/vespa/searchcore/proton/metrics/trans_log_server_metrics.cpp @@ -44,6 +44,7 @@ TransLogServerMetrics::considerRemoveDomains(const DomainStats &stats) for (auto itr = _domainMetrics.begin(); itr != _domainMetrics.end(); ) { const vespalib::string &documentType = itr->first; if (stats.find(documentType) == stats.end()) { + _parent->unregisterMetric(*itr->second); itr = _domainMetrics.erase(itr); } else { ++itr; diff --git a/searchcore/src/vespa/searchcore/proton/server/CMakeLists.txt b/searchcore/src/vespa/searchcore/proton/server/CMakeLists.txt index 87aa19fb1b2..2df34312b52 100644 --- a/searchcore/src/vespa/searchcore/proton/server/CMakeLists.txt +++ b/searchcore/src/vespa/searchcore/proton/server/CMakeLists.txt @@ -75,6 +75,7 @@ vespa_add_library(searchcore_server STATIC proton_config_fetcher.cpp proton_config_snapshot.cpp proton_configurer.cpp + proton_disk_layout.cpp prune_session_cache_job.cpp pruneremoveddocumentsjob.cpp putdonecontext.cpp diff --git a/searchcore/src/vespa/searchcore/proton/server/fast_access_feed_view.cpp b/searchcore/src/vespa/searchcore/proton/server/fast_access_feed_view.cpp index 1c2406b2acf..dfe98b44adc 100644 --- a/searchcore/src/vespa/searchcore/proton/server/fast_access_feed_view.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/fast_access_feed_view.cpp @@ -37,11 +37,14 @@ FastAccessFeedView::updateAttributes(SerialNum serialNum, search::DocumentIdT li } void -FastAccessFeedView::updateAttributes(SerialNum serialNum, Lid lid, FutureDoc doc, +FastAccessFeedView::updateAttributes(SerialNum serialNum, Lid lid, FutureDoc futureDoc, bool immediateCommit, OnOperationDoneType onWriteDone) { if (_attributeWriter->hasStructFieldAttribute()) { - _attributeWriter->update(serialNum, *doc.get(), lid, immediateCommit, onWriteDone); + const std::unique_ptr<const Document> & doc = futureDoc.get(); + if (doc) { + _attributeWriter->update(serialNum, *doc, lid, immediateCommit, onWriteDone); + } } } diff --git a/searchcore/src/vespa/searchcore/proton/server/fileconfigmanager.cpp b/searchcore/src/vespa/searchcore/proton/server/fileconfigmanager.cpp index 76f2ffce93a..72f48df3295 100644 --- a/searchcore/src/vespa/searchcore/proton/server/fileconfigmanager.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/fileconfigmanager.cpp @@ -229,6 +229,7 @@ FileConfigManager::FileConfigManager(const vespalib::string &baseDir, _protonConfig() { vespalib::mkdir(baseDir, false); + vespalib::File::sync(vespalib::dirname(baseDir)); if (!_info.load()) _info.save(); removeInvalid(); @@ -297,6 +298,8 @@ FileConfigManager::saveConfig(const DocumentDBConfig &snapshot, bool saveHistorySchemaRes = historySchema.saveToFile(snapDir + "/historyschema.txt"); assert(saveHistorySchemaRes); (void) saveHistorySchemaRes; + vespalib::File::sync(snapDir); + vespalib::File::sync(_baseDir); _info.validateSnapshot(serialNum); @@ -402,6 +405,7 @@ FileConfigManager::removeInvalid() LOG(warning, "Removing obsolete config directory '%s' failed due to %s", snapDir.c_str(), e.what()); } } + vespalib::File::sync(_baseDir); for (const auto &serial : toRem) { _info.removeSnapshot(serial); } diff --git a/searchcore/src/vespa/searchcore/proton/server/i_proton_disk_layout.h b/searchcore/src/vespa/searchcore/proton/server/i_proton_disk_layout.h new file mode 100644 index 00000000000..1ee521379eb --- /dev/null +++ b/searchcore/src/vespa/searchcore/proton/server/i_proton_disk_layout.h @@ -0,0 +1,23 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <set> + +namespace proton { + +class DocTypeName; + +/** + * Interface class with utility functions for handling the disk + * directory layout for proton instance. + */ +class IProtonDiskLayout +{ +public: + virtual ~IProtonDiskLayout() = default; + virtual void remove(const DocTypeName &docTypeName) = 0; + virtual void initAndPruneUnused(const std::set<DocTypeName> &docTypeNames) = 0; +}; + +} // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/server/proton.cpp b/searchcore/src/vespa/searchcore/proton/server/proton.cpp index 9a7d5a5141e..ab0abde6ccd 100644 --- a/searchcore/src/vespa/searchcore/proton/server/proton.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/proton.cpp @@ -9,6 +9,7 @@ #include "prepare_restart_handler.h" #include "proton.h" #include "proton_config_snapshot.h" +#include "proton_disk_layout.h" #include "resource_usage_explorer.h" #include "searchhandlerproxy.h" #include "simpleflush.h" @@ -193,7 +194,8 @@ Proton::Proton(const config::ConfigUri & configUri, // This executor can only have 1 thread as it is used for // serializing startup. _executor(1, 128 * 1024), - _protonConfigurer(_executor, *this), + _protonDiskLayout(), + _protonConfigurer(_executor, *this, _protonDiskLayout), _protonConfigFetcher(configUri, _protonConfigurer, subscribeTimeout), _warmupExecutor(), _summaryExecutor(), @@ -269,7 +271,7 @@ Proton::init(const BootstrapConfig::SP & configSnapshot) strategy = std::make_shared<SimpleFlush>(); break; } - vespalib::mkdir(protonConfig.basedir + "/documents", true); + _protonDiskLayout = std::make_unique<ProtonDiskLayout>(protonConfig.basedir, protonConfig.tlsspec); vespalib::chdir(protonConfig.basedir); _tls->start(); _flushEngine = std::make_unique<FlushEngine>(std::make_shared<flushengine::TlsStatsFactory>(_tls->getTransLogServer()), diff --git a/searchcore/src/vespa/searchcore/proton/server/proton.h b/searchcore/src/vespa/searchcore/proton/server/proton.h index a3c596d7b0b..6e07ddcecdb 100644 --- a/searchcore/src/vespa/searchcore/proton/server/proton.h +++ b/searchcore/src/vespa/searchcore/proton/server/proton.h @@ -37,6 +37,7 @@ namespace proton { class DiskMemUsageSampler; class IDocumentDBReferenceRegistry; +class IProtonDiskLayout; class PrepareRestartHandler; class SummaryEngine; class DocsumBySlime; @@ -111,6 +112,7 @@ private: std::unique_ptr<vespalib::StateServer> _stateServer; std::unique_ptr<TransportServer> _fs4Server; vespalib::ThreadStackExecutor _executor; + std::unique_ptr<IProtonDiskLayout> _protonDiskLayout; ProtonConfigurer _protonConfigurer; ProtonConfigFetcher _protonConfigFetcher; std::unique_ptr<vespalib::ThreadStackExecutorBase> _warmupExecutor; diff --git a/searchcore/src/vespa/searchcore/proton/server/proton_configurer.cpp b/searchcore/src/vespa/searchcore/proton/server/proton_configurer.cpp index f871f36b042..0b9293a4aab 100644 --- a/searchcore/src/vespa/searchcore/proton/server/proton_configurer.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/proton_configurer.cpp @@ -6,6 +6,7 @@ #include "i_proton_configurer_owner.h" #include "document_db_config_owner.h" #include "document_db_directory_holder.h" +#include "i_proton_disk_layout.h" #include <vespa/vespalib/util/lambdatask.h> #include <vespa/vespalib/util/threadstackexecutorbase.h> #include <vespa/document/bucket/fixed_bucket_spaces.h> @@ -39,7 +40,8 @@ getBucketSpace(const BootstrapConfig &bootstrapConfig, const DocTypeName &name) ProtonConfigurer::ProtonConfigurer(vespalib::ThreadStackExecutorBase &executor, - IProtonConfigurerOwner &owner) + IProtonConfigurerOwner &owner, + const std::unique_ptr<IProtonDiskLayout> &diskLayout) : IProtonConfigurer(), _executor(executor), _owner(owner), @@ -48,7 +50,8 @@ ProtonConfigurer::ProtonConfigurer(vespalib::ThreadStackExecutorBase &executor, _activeConfigSnapshot(), _mutex(), _allowReconfig(false), - _componentConfig() + _componentConfig(), + _diskLayout(diskLayout) { } @@ -135,6 +138,9 @@ ProtonConfigurer::applyConfig(std::shared_ptr<ProtonConfigSnapshot> configSnapsh } const auto &bootstrapConfig = configSnapshot->getBootstrapConfig(); const ProtonConfig &protonConfig = bootstrapConfig->getProtonConfig(); + if (initialConfig) { + pruneInitialDocumentDBDirs(*configSnapshot); + } _owner.applyConfig(bootstrapConfig); for (const auto &ddbConfig : protonConfig.documentdb) { DocTypeName docTypeName(ddbConfig.inputdoctypename); @@ -176,6 +182,18 @@ ProtonConfigurer::configureDocumentDB(const ProtonConfigSnapshot &configSnapshot } void +ProtonConfigurer::pruneInitialDocumentDBDirs(const ProtonConfigSnapshot &configSnapshot) +{ + std::set<DocTypeName> docTypeNames; + const auto &bootstrapConfig = configSnapshot.getBootstrapConfig(); + const ProtonConfig &protonConfig = bootstrapConfig->getProtonConfig(); + for (const auto &ddbConfig : protonConfig.documentdb) { + docTypeNames.emplace(ddbConfig.inputdoctypename); + } + _diskLayout->initAndPruneUnused(docTypeNames); +} + +void ProtonConfigurer::pruneDocumentDBs(const ProtonConfigSnapshot &configSnapshot) { // called by proton executor thread @@ -193,6 +211,7 @@ ProtonConfigurer::pruneDocumentDBs(const ProtonConfigSnapshot &configSnapshot) if (found == newDocTypes.end()) { _owner.removeDocumentDB(dbitr->first); DocumentDBDirectoryHolder::waitUntilDestroyed(dbitr->second.second); + _diskLayout->remove(dbitr->first); dbitr = _documentDBs.erase(dbitr); } else { ++dbitr; diff --git a/searchcore/src/vespa/searchcore/proton/server/proton_configurer.h b/searchcore/src/vespa/searchcore/proton/server/proton_configurer.h index ac0f6197fd7..c896f12bd4f 100644 --- a/searchcore/src/vespa/searchcore/proton/server/proton_configurer.h +++ b/searchcore/src/vespa/searchcore/proton/server/proton_configurer.h @@ -16,6 +16,7 @@ class DocumentDBDirectoryHolder; class IDocumentDBConfigOwner; class IProtonConfigurerOwner; class BootstrapConfig; +class IProtonDiskLayout; /* * Class to handle config changes to proton using config snapshots spanning @@ -34,6 +35,7 @@ class ProtonConfigurer : public IProtonConfigurer mutable std::mutex _mutex; bool _allowReconfig; vespalib::SimpleComponentConfigProducer _componentConfig; + const std::unique_ptr<IProtonDiskLayout> &_diskLayout; void performReconfigure(); bool skipConfig(const ProtonConfigSnapshot *configSnapshot, bool initialConfig); @@ -43,10 +45,12 @@ class ProtonConfigurer : public IProtonConfigurer const DocTypeName &docTypeName, document::BucketSpace bucketSpace, const vespalib::string &configId, const InitializeThreads &initializeThreads); void pruneDocumentDBs(const ProtonConfigSnapshot &configSnapshot); + void pruneInitialDocumentDBDirs(const ProtonConfigSnapshot &configSnapshot); public: ProtonConfigurer(vespalib::ThreadStackExecutorBase &executor, - IProtonConfigurerOwner &owner); + IProtonConfigurerOwner &owner, + const std::unique_ptr<IProtonDiskLayout> &diskLayout); ~ProtonConfigurer(); diff --git a/searchcore/src/vespa/searchcore/proton/server/proton_disk_layout.cpp b/searchcore/src/vespa/searchcore/proton/server/proton_disk_layout.cpp new file mode 100644 index 00000000000..31fd44eec5e --- /dev/null +++ b/searchcore/src/vespa/searchcore/proton/server/proton_disk_layout.cpp @@ -0,0 +1,120 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "proton_disk_layout.h" +#include <vespa/vespalib/io/fileutil.h> +#include <vespa/fastos/file.h> +#include <vespa/searchcore/proton/common/doctypename.h> +#include <vespa/searchlib/transactionlog/translogclient.h> +#include <cassert> + +#include <vespa/log/log.h> +LOG_SETUP(".proton.server.proton_disk_layout"); + +using search::transactionlog::TransLogClient; + +namespace proton { + +namespace { + +struct DocumentDBDirMeta +{ + bool normal; + bool removed; + + DocumentDBDirMeta() + : normal(false), + removed(false) + { + } +}; + +using DocumentDBDirScan = std::map<DocTypeName, DocumentDBDirMeta>; + +vespalib::string getDocumentsDir(const vespalib::string &baseDir) +{ + return baseDir + "/documents"; +} + +vespalib::string removedSuffix(".removed"); + +vespalib::string getNormalName(const vespalib::string removedName) { + return removedName.substr(0, removedName.size() - removedSuffix.size()); +} + +vespalib::string getRemovedName(const vespalib::string &normalName) +{ + return normalName + removedSuffix; +} + +bool isRemovedName(const vespalib::string &dirName) +{ + return dirName.size() > removedSuffix.size() && dirName.substr(dirName.size() - removedSuffix.size()) == removedSuffix; +} + +void scanDir(const vespalib::string documentsDir, DocumentDBDirScan &dirs) +{ + auto names = vespalib::listDirectory(documentsDir); + for (const auto &name : names) { + if (vespalib::isDirectory(documentsDir + "/" + name)) { + if (isRemovedName(name)) { + dirs[DocTypeName(getNormalName(name))].removed = true; + } else { + dirs[DocTypeName(name)].normal = true; + } + } + } +} + +} + +ProtonDiskLayout::ProtonDiskLayout(const vespalib::string &baseDir, const vespalib::string &tlsSpec) + : _baseDir(baseDir), + _tlsSpec(tlsSpec) +{ + vespalib::mkdir(getDocumentsDir(_baseDir), true); +} + +ProtonDiskLayout::~ProtonDiskLayout() = default; + +void +ProtonDiskLayout::remove(const DocTypeName &docTypeName) +{ + vespalib::string documentsDir(getDocumentsDir(_baseDir)); + vespalib::string name(docTypeName.toString()); + vespalib::string normalDir(documentsDir + "/" + name); + vespalib::string removedDir(documentsDir + "/" + getRemovedName(name)); + vespalib::rename(normalDir, removedDir, false, false); + vespalib::File::sync(documentsDir); + TransLogClient tlc(_tlsSpec); + if (!tlc.remove(name)) { + LOG(fatal, "Failed to remove tls domain %s", name.c_str()); + LOG_ABORT("Failed to remove tls domain"); + } + vespalib::rmdir(removedDir, true); + vespalib::File::sync(documentsDir); +} + +void +ProtonDiskLayout::initAndPruneUnused(const std::set<DocTypeName> &docTypeNames) +{ + vespalib::string documentsDir(getDocumentsDir(_baseDir)); + DocumentDBDirScan dirs; + scanDir(documentsDir, dirs); + for (const auto &dir : dirs) { + if (dir.second.removed) { + // Complete interrupted removal + if (dir.second.normal) { + vespalib::string name(dir.first.toString()); + vespalib::string normalDir(documentsDir + "/" + name); + vespalib::rmdir(normalDir, true); + } + remove(dir.first); + } else if (docTypeNames.count(dir.first) == 0) { + // Remove unused directory + remove(dir.first); + } + } +} + +} // namespace proton + diff --git a/searchcore/src/vespa/searchcore/proton/server/proton_disk_layout.h b/searchcore/src/vespa/searchcore/proton/server/proton_disk_layout.h new file mode 100644 index 00000000000..167a1900b99 --- /dev/null +++ b/searchcore/src/vespa/searchcore/proton/server/proton_disk_layout.h @@ -0,0 +1,27 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "i_proton_disk_layout.h" +#include <vespa/vespalib/stllike/string.h> + +namespace proton { + +/** + * Class with utility functions for handling the disk directory layout + * for proton instance. + */ +class ProtonDiskLayout : public IProtonDiskLayout +{ +private: + const vespalib::string _baseDir; + const vespalib::string _tlsSpec; + +public: + ProtonDiskLayout(const vespalib::string &baseDir, const vespalib::string &tlsSpec); + ~ProtonDiskLayout() override; + void remove(const DocTypeName &docTypeName) override; + void initAndPruneUnused(const std::set<DocTypeName> &docTypeNames) override; +}; + +} // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.cpp b/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.cpp index ad06525a5a3..2d3c204d259 100644 --- a/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.cpp @@ -139,6 +139,7 @@ StoreOnlyDocSubDB::StoreOnlyDocSubDB(const Config &cfg, const Context &ctx) _gidToLidChangeHandler(std::make_shared<DummyGidToLidChangeHandler>()) { vespalib::mkdir(_baseDir, false); // Assume parent is created. + vespalib::File::sync(vespalib::dirname(_baseDir)); } StoreOnlyDocSubDB::~StoreOnlyDocSubDB() diff --git a/searchcorespi/src/vespa/searchcorespi/index/diskindexcleaner.cpp b/searchcorespi/src/vespa/searchcorespi/index/diskindexcleaner.cpp index c363a93815b..e0cb8548f0d 100644 --- a/searchcorespi/src/vespa/searchcorespi/index/diskindexcleaner.cpp +++ b/searchcorespi/src/vespa/searchcorespi/index/diskindexcleaner.cpp @@ -3,6 +3,7 @@ #include "diskindexcleaner.h" #include "activediskindexes.h" #include <vespa/fastos/file.h> +#include <vespa/vespalib/io/fileutil.h> #include <sstream> #include <vector> @@ -34,6 +35,11 @@ bool isValidIndex(const string &index_dir) { return serial_file.OpenReadOnlyExisting(); } +void invalidateIndex(const string &index_dir) { + vespalib::unlink(index_dir + "/serial.dat"); + vespalib::File::sync(index_dir); +} + uint32_t findLastFusionId(const string &base_dir, const vector<string> &indexes) { uint32_t fusion_id = 0; @@ -56,7 +62,8 @@ uint32_t findLastFusionId(const string &base_dir, void removeDir(const string &dir) { LOG(debug, "Removing index dir '%s'", dir.c_str()); - FastOS_FileInterface::EmptyAndRemoveDirectory(dir.c_str()); + invalidateIndex(dir); + vespalib::rmdir(dir, true); } bool isOldIndex(const string &index, uint32_t last_fusion_id) { @@ -73,14 +80,18 @@ bool isOldIndex(const string &index, uint32_t last_fusion_id) { } void removeOld(const string &base_dir, const vector<string> &indexes, - const ActiveDiskIndexes &active_indexes) { + const ActiveDiskIndexes &active_indexes, bool remove) { uint32_t last_fusion_id = findLastFusionId(base_dir, indexes); for (size_t i = 0; i < indexes.size(); ++i) { const string index_dir = base_dir + "/" + indexes[i]; if (isOldIndex(indexes[i], last_fusion_id) && !active_indexes.isActive(index_dir)) { - removeDir(index_dir); + if (remove) { + removeDir(index_dir); + } else { + invalidateIndex(index_dir); + } } } } @@ -99,14 +110,14 @@ void removeInvalid(const string &base_dir, const vector<string> &indexes) { void DiskIndexCleaner::clean(const string &base_dir, const ActiveDiskIndexes &active_indexes) { vector<string> indexes = readIndexes(base_dir); - removeOld(base_dir, indexes, active_indexes); + removeOld(base_dir, indexes, active_indexes, false); removeInvalid(base_dir, indexes); } void DiskIndexCleaner::removeOldIndexes( const string &base_dir, const ActiveDiskIndexes &active_indexes) { vector<string> indexes = readIndexes(base_dir); - removeOld(base_dir, indexes, active_indexes); + removeOld(base_dir, indexes, active_indexes, true); } } diff --git a/searchcorespi/src/vespa/searchcorespi/index/indexmaintainer.cpp b/searchcorespi/src/vespa/searchcorespi/index/indexmaintainer.cpp index a18d24931cb..1e63741084d 100644 --- a/searchcorespi/src/vespa/searchcorespi/index/indexmaintainer.cpp +++ b/searchcorespi/src/vespa/searchcorespi/index/indexmaintainer.cpp @@ -16,6 +16,7 @@ #include <vespa/vespalib/util/lambdatask.h> #include <sstream> #include <vespa/searchcorespi/flush/closureflushtask.h> +#include <vespa/vespalib/io/fileutil.h> #include <vespa/vespalib/util/exceptions.h> #include <vespa/vespalib/util/array.hpp> #include <vespa/fastos/file.h> @@ -999,6 +1000,7 @@ IndexMaintainer::runFusion(const FusionSpec &fusion_spec) _activeFusionSchema.reset(); _activeFusionPrunedSchema.reset(); } + vespalib::File::sync(vespalib::dirname(fail_dir)); return fusion_spec.last_fusion_id; } diff --git a/searchcorespi/src/vespa/searchcorespi/index/indexwriteutilities.cpp b/searchcorespi/src/vespa/searchcorespi/index/indexwriteutilities.cpp index c8b2e81a9c0..ebb316610e1 100644 --- a/searchcorespi/src/vespa/searchcorespi/index/indexwriteutilities.cpp +++ b/searchcorespi/src/vespa/searchcorespi/index/indexwriteutilities.cpp @@ -5,6 +5,7 @@ #include <vespa/searchlib/common/serialnumfileheadercontext.h> #include <vespa/searchlib/index/schemautil.h> #include <vespa/fastlib/io/bufferedfile.h> +#include <vespa/vespalib/io/fileutil.h> #include <vespa/vespalib/util/exceptions.h> #include <sstream> #include <unistd.h> @@ -57,6 +58,7 @@ IndexWriteUtilities::writeSerialNum(SerialNum serialNum, tmpFileName.c_str()); } file.Close(); + vespalib::File::sync(dir); if (ok) { FastOS_File renameFile(tmpFileName.c_str()); @@ -67,6 +69,7 @@ IndexWriteUtilities::writeSerialNum(SerialNum serialNum, msg << "Unable to write serial number to '" << dir << "'."; throw IllegalStateException(msg.str()); } + vespalib::File::sync(dir); } bool @@ -94,12 +97,14 @@ IndexWriteUtilities::copySerialNumFile(const vespalib::string &sourceDir, return false; } file.Close(); + vespalib::File::sync(destDir); if (!file.Rename(dest.c_str())) { LOG(error, "Unable to rename file '%s' to '%s'", tmpDest.c_str(), dest.c_str()); return false; } + vespalib::File::sync(destDir); return true; } @@ -151,6 +156,7 @@ IndexWriteUtilities::updateDiskIndexSchema(const vespalib::string &indexDir, } vespalib::string schemaTmpName = schemaName + ".tmp"; vespalib::string schemaOrigName = schemaName + ".orig"; + vespalib::unlink(schemaTmpName); if (!newSchema->saveToFile(schemaTmpName)) { LOG(error, "Could not save schema to '%s'", schemaTmpName.c_str()); @@ -172,6 +178,7 @@ IndexWriteUtilities::updateDiskIndexSchema(const vespalib::string &indexDir, schemaName.c_str(), FastOS_File::getLastErrorString().c_str()); } + vespalib::File::sync(indexDir); } // XXX: FastOS layer violation int renameres = ::rename(schemaTmpName.c_str(), schemaName.c_str()); @@ -183,6 +190,7 @@ IndexWriteUtilities::updateDiskIndexSchema(const vespalib::string &indexDir, schemaName.c_str(), errString.c_str()); } + vespalib::File::sync(indexDir); } } // namespace index diff --git a/searchlib/src/vespa/searchlib/common/indexmetainfo.cpp b/searchlib/src/vespa/searchlib/common/indexmetainfo.cpp index 10015bb658a..837c38eb340 100644 --- a/searchlib/src/vespa/searchlib/common/indexmetainfo.cpp +++ b/searchlib/src/vespa/searchlib/common/indexmetainfo.cpp @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "indexmetainfo.h" +#include <vespa/vespalib/io/fileutil.h> #include <vespa/vespalib/util/stringfmt.h> #include <vespa/vespalib/util/guard.h> #include <cassert> @@ -310,6 +311,7 @@ IndexMetaInfo::save(const vespalib::string &baseName) { vespalib::string fileName = makeFileName(baseName); vespalib::string newName = fileName + ".new"; + vespalib::unlink(newName); vespalib::FilePointer f(fopen(newName.c_str(), "w")); if (!f.valid()) { LOG(warning, "could not open file for writing: %s", newName.c_str()); @@ -350,6 +352,7 @@ IndexMetaInfo::save(const vespalib::string &baseName) newName.c_str(), fileName.c_str()); return false; } + vespalib::File::sync(vespalib::dirname(fileName)); return true; } diff --git a/searchlib/src/vespa/searchlib/diskindex/fusion.cpp b/searchlib/src/vespa/searchlib/diskindex/fusion.cpp index e61fe7bab17..2db6025d257 100644 --- a/searchlib/src/vespa/searchlib/diskindex/fusion.cpp +++ b/searchlib/src/vespa/searchlib/diskindex/fusion.cpp @@ -218,6 +218,7 @@ Fusion::mergeField(uint32_t id) } if (!FileKit::createStamp(indexDir + "/.mergeocc_done")) return false; + vespalib::File::sync(indexDir); if (!CleanTmpDirs()) return false; diff --git a/searchlib/src/vespa/searchlib/diskindex/indexbuilder.cpp b/searchlib/src/vespa/searchlib/diskindex/indexbuilder.cpp index b879e1f65a6..b744a68932d 100644 --- a/searchlib/src/vespa/searchlib/diskindex/indexbuilder.cpp +++ b/searchlib/src/vespa/searchlib/diskindex/indexbuilder.cpp @@ -700,6 +700,7 @@ IndexBuilder::close() for (FieldHandle & fh : _fields) { if (fh.getValid()) { fh.close(); + vespalib::File::sync(fh.getDir()); } } if (!docsummary::DocumentSummary::writeDocIdLimit(_prefix, _docIdLimit)) { diff --git a/searchlib/src/vespa/searchlib/transactionlog/domain.cpp b/searchlib/src/vespa/searchlib/transactionlog/domain.cpp index 88c2dd9ecc3..1caff132779 100644 --- a/searchlib/src/vespa/searchlib/transactionlog/domain.cpp +++ b/searchlib/src/vespa/searchlib/transactionlog/domain.cpp @@ -3,6 +3,7 @@ #include "domain.h" #include <vespa/vespalib/util/stringfmt.h> #include <vespa/vespalib/util/closuretask.h> +#include <vespa/vespalib/io/fileutil.h> #include <vespa/fastos/file.h> #include <algorithm> #include <thread> @@ -60,6 +61,7 @@ Domain::Domain(const string &domainName, const string & baseDir, Executor & comm _sessionExecutor.sync(); if (_parts.empty() || _parts.crbegin()->second->isClosed()) { _parts[lastPart].reset(new DomainPart(_name, dir(), lastPart, _defaultCrcType, _fileHeaderContext, false)); + vespalib::File::sync(dir()); } } @@ -294,6 +296,7 @@ void Domain::commit(const Packet & packet) _parts[entry.serial()] = dp; } dp = _parts.rbegin()->second; + vespalib::File::sync(dir()); } dp->commit(entry.serial(), packet); cleanSessions(); @@ -310,6 +313,7 @@ bool Domain::erase(SerialNum to) _parts.erase(it); } retval = retval && dp->erase(to); + vespalib::File::sync(dir()); } if (_parts.begin()->second->range().to() >= to) { _parts.begin()->second->erase(to); diff --git a/searchlib/src/vespa/searchlib/transactionlog/translogserver.cpp b/searchlib/src/vespa/searchlib/transactionlog/translogserver.cpp index 4c3c5609a93..bfca137ba06 100644 --- a/searchlib/src/vespa/searchlib/transactionlog/translogserver.cpp +++ b/searchlib/src/vespa/searchlib/transactionlog/translogserver.cpp @@ -336,6 +336,29 @@ void TransLogServer::exportRPC(FRT_Supervisor & supervisor) rb.ReturnDesc("syncedto", "Entry synced to"); } +namespace { + +void +writeDomainDir(std::lock_guard<std::mutex> &guard, + vespalib::string dir, + vespalib::string domainList, + std::map<vespalib::string, std::shared_ptr<Domain>> &domains) +{ + (void) guard; + vespalib::string domainListTmp(domainList + ".tmp"); + vespalib::unlink(domainListTmp); + std::ofstream domainDir(domainListTmp.c_str(), std::ios::trunc); + for (const auto &domainEntry : domains) { + domainDir << domainEntry.first << std::endl; + } + domainDir.close(); + vespalib::File::sync(domainListTmp); + vespalib::rename(domainListTmp, domainList, false, false); + vespalib::File::sync(dir); +} + +} + void TransLogServer::createDomain(FRT_RPCRequest *req) { uint32_t retval(0); @@ -351,12 +374,9 @@ void TransLogServer::createDomain(FRT_RPCRequest *req) try { domain = std::make_shared<Domain>(domainName, dir(), _commitExecutor, _sessionExecutor, _domainPartSize, _defaultCrcType, _fileHeaderContext); - { - Guard domainGuard(_lock); - _domains[domain->name()] = domain; - } - std::ofstream domainDir(domainList().c_str(), std::ios::app); - domainDir << domain->name() << std::endl; + Guard domainGuard(_lock); + _domains[domain->name()] = domain; + writeDomainDir(domainGuard, dir(), domainList(), _domains); } catch (const std::exception & e) { LOG(warning, "Failed creating %s domain. Exception = %s", domainName, e.what()); retval = uint32_t(-1); @@ -385,12 +405,10 @@ void TransLogServer::deleteDomain(FRT_RPCRequest *req) Guard domainGuard(_lock); _domains.erase(domainName); } - vespalib::rmdir(Domain::getDir(dir(), domainName).c_str(), true); - std::ofstream domainDir(domainList().c_str(), std::ios::trunc); + vespalib::rmdir(Domain::getDir(dir(), domainName), true); + vespalib::File::sync(dir()); Guard domainGuard(_lock); - for (DomainList::const_iterator it(_domains.begin()), mt(_domains.end()); it != mt; it++) { - domainDir << it->first << std::endl; - } + writeDomainDir(domainGuard, dir(), domainList(), _domains); } catch (const std::exception & e) { msg = make_string("Failed deleting %s domain. Exception = %s", domainName, e.what()); retval = -1; diff --git a/storage/src/vespa/storage/common/content_bucket_space.cpp b/storage/src/vespa/storage/common/content_bucket_space.cpp index 2bae118e364..0827c721100 100644 --- a/storage/src/vespa/storage/common/content_bucket_space.cpp +++ b/storage/src/vespa/storage/common/content_bucket_space.cpp @@ -9,7 +9,8 @@ ContentBucketSpace::ContentBucketSpace(document::BucketSpace bucketSpace) _bucketDatabase(), _lock(), _clusterState(), - _distribution() + _distribution(), + _nodeUpInLastNodeStateSeenByProvider(false) { } @@ -41,4 +42,18 @@ ContentBucketSpace::getDistribution() const return _distribution; } +bool +ContentBucketSpace::getNodeUpInLastNodeStateSeenByProvider() const +{ + std::lock_guard guard(_lock); + return _nodeUpInLastNodeStateSeenByProvider; +} + +void +ContentBucketSpace::setNodeUpInLastNodeStateSeenByProvider(bool nodeUpInLastNodeStateSeenByProvider) +{ + std::lock_guard guard(_lock); + _nodeUpInLastNodeStateSeenByProvider = nodeUpInLastNodeStateSeenByProvider; +} + } diff --git a/storage/src/vespa/storage/common/content_bucket_space.h b/storage/src/vespa/storage/common/content_bucket_space.h index 6ccc82bc4fb..81ce6234879 100644 --- a/storage/src/vespa/storage/common/content_bucket_space.h +++ b/storage/src/vespa/storage/common/content_bucket_space.h @@ -22,6 +22,7 @@ private: mutable std::mutex _lock; std::shared_ptr<const lib::ClusterState> _clusterState; std::shared_ptr<const lib::Distribution> _distribution; + bool _nodeUpInLastNodeStateSeenByProvider; public: using UP = std::unique_ptr<ContentBucketSpace>; @@ -33,6 +34,8 @@ public: std::shared_ptr<const lib::ClusterState> getClusterState() const; void setDistribution(std::shared_ptr<const lib::Distribution> distribution); std::shared_ptr<const lib::Distribution> getDistribution() const; + bool getNodeUpInLastNodeStateSeenByProvider() const; + void setNodeUpInLastNodeStateSeenByProvider(bool nodeUpInLastNodeStateSeenByProvider); }; } diff --git a/storage/src/vespa/storage/persistence/filestorage/filestormanager.cpp b/storage/src/vespa/storage/persistence/filestorage/filestormanager.cpp index 961af1532a1..d1f0a24178a 100644 --- a/storage/src/vespa/storage/persistence/filestorage/filestormanager.cpp +++ b/storage/src/vespa/storage/persistence/filestorage/filestormanager.cpp @@ -36,7 +36,6 @@ FileStorManager(const config::ConfigUri & configUri, const spi::PartitionStateLi _partitions(partitions), _providerCore(provider), _providerErrorWrapper(_providerCore), - _nodeUpInLastNodeStateSeenByProvider(false), _providerMetric(new spi::MetricPersistenceProvider(_providerErrorWrapper)), _provider(_providerMetric.get()), _bucketIdFactory(_component.getBucketIdFactory()), @@ -886,21 +885,23 @@ FileStorManager::updateState() auto clusterStateBundle = _component.getStateUpdater().getClusterStateBundle(); lib::ClusterState::CSP state(clusterStateBundle->getBaselineClusterState()); lib::Node node(_component.getNodeType(), _component.getIndex()); - bool nodeUp = state->getNodeState(node).getState().oneOf("uir"); LOG(debug, "FileStorManager received cluster state '%s'", state->toString().c_str()); - // If edge where we go down - if (_nodeUpInLastNodeStateSeenByProvider && !nodeUp) { - LOG(debug, "Received cluster state where this node is down; de-activating all buckets in database"); - Deactivator deactivator; - _component.getBucketSpaceRepo().forEachBucket(deactivator, "FileStorManager::updateState"); - } for (const auto &elem : _component.getBucketSpaceRepo()) { BucketSpace bucketSpace(elem.first); - spi::ClusterState spiState(*elem.second->getClusterState(), _component.getIndex(), *elem.second->getDistribution()); + ContentBucketSpace &contentBucketSpace = *elem.second; + auto derivedClusterState = contentBucketSpace.getClusterState(); + bool nodeUp = derivedClusterState->getNodeState(node).getState().oneOf("uir"); + // If edge where we go down + if (contentBucketSpace.getNodeUpInLastNodeStateSeenByProvider() && !nodeUp) { + LOG(debug, "Received cluster state where this node is down; de-activating all buckets in database for bucket space %s", bucketSpace.toString().c_str()); + Deactivator deactivator; + contentBucketSpace.bucketDatabase().all(deactivator, "FileStorManager::updateState"); + } + contentBucketSpace.setNodeUpInLastNodeStateSeenByProvider(nodeUp); + spi::ClusterState spiState(*derivedClusterState, _component.getIndex(), *contentBucketSpace.getDistribution()); _provider->setClusterState(bucketSpace, spiState); } - _nodeUpInLastNodeStateSeenByProvider = nodeUp; } void diff --git a/storage/src/vespa/storage/persistence/filestorage/filestormanager.h b/storage/src/vespa/storage/persistence/filestorage/filestormanager.h index 0bd2ffa5910..4bf2c1049cf 100644 --- a/storage/src/vespa/storage/persistence/filestorage/filestormanager.h +++ b/storage/src/vespa/storage/persistence/filestorage/filestormanager.h @@ -56,7 +56,6 @@ class FileStorManager : public StorageLinkQueued, const spi::PartitionStateList& _partitions; spi::PersistenceProvider& _providerCore; ProviderErrorWrapper _providerErrorWrapper; - bool _nodeUpInLastNodeStateSeenByProvider; spi::MetricPersistenceProvider::UP _providerMetric; spi::PersistenceProvider* _provider; diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/bindings/SignedIdentityDocumentEntity.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/bindings/SignedIdentityDocumentEntity.java index 3d4872549d6..6f773d51df3 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/bindings/SignedIdentityDocumentEntity.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/bindings/SignedIdentityDocumentEntity.java @@ -54,7 +54,7 @@ public class SignedIdentityDocumentEntity { @JsonProperty("ip-addresses") Set<String> ipAddresses, @JsonProperty("identity-type") String identityType) { this.rawIdentityDocument = rawIdentityDocument; - this.identityDocument = parseIdentityDocument(rawIdentityDocument); + this.identityDocument = rawIdentityDocument != null ? parseIdentityDocument(rawIdentityDocument) : null; this.signature = signature; this.signingKeyVersion = signingKeyVersion; this.providerUniqueId = providerUniqueId; diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/utils/SiaUtils.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/utils/SiaUtils.java index 55e9103b040..05459e5488b 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/utils/SiaUtils.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/utils/SiaUtils.java @@ -7,13 +7,18 @@ import com.yahoo.vespa.athenz.tls.X509CertificateUtils; import java.io.IOException; import java.io.UncheckedIOException; +import java.nio.file.DirectoryStream; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.nio.file.StandardCopyOption; import java.security.PrivateKey; import java.security.cert.X509Certificate; +import java.util.List; import java.util.Optional; +import java.util.stream.StreamSupport; + +import static java.util.stream.Collectors.toList; /** * Misc utility methods for SIA provided credentials @@ -105,6 +110,25 @@ public class SiaUtils { } } + public static List<AthenzService> findSiaServices() { + return findSiaServices(DEFAULT_SIA_DIRECTORY); + } + + public static List<AthenzService> findSiaServices(Path root) { + String keyFileSuffix = ".key.pem"; + Path keysDirectory = root.resolve("keys"); + try (DirectoryStream<Path> directoryStream = Files.newDirectoryStream(keysDirectory)) { + return StreamSupport.stream(directoryStream.spliterator(), false) + .map(path -> path.getFileName().toString()) + .filter(fileName -> fileName.endsWith(keyFileSuffix)) + .map(fileName -> fileName.substring(0, fileName.length() - keyFileSuffix.length())) + .map(AthenzService::new) + .collect(toList()); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + private static Path toTempFile(Path file) { return Paths.get(file.toAbsolutePath().toString() + ".tmp"); } diff --git a/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/utils/SiaUtilsTest.java b/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/utils/SiaUtilsTest.java new file mode 100644 index 00000000000..0282373cdaf --- /dev/null +++ b/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/utils/SiaUtilsTest.java @@ -0,0 +1,40 @@ +package com.yahoo.vespa.athenz.utils; + +import com.yahoo.vespa.athenz.api.AthenzService; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; + +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.CoreMatchers.hasItem; +import static org.junit.Assert.assertThat; + +/** + * @author bjorncs + */ +public class SiaUtilsTest { + + @Rule + public TemporaryFolder tempDirectory = new TemporaryFolder(); + + @Test + public void it_finds_all_identity_names_from_files_in_sia_keys_directory() throws IOException { + Path siaRoot = tempDirectory.getRoot().toPath(); + Files.createDirectory(siaRoot.resolve("keys")); + AthenzService fooService = new AthenzService("my.domain.foo"); + Files.createFile(SiaUtils.getPrivateKeyFile(siaRoot, fooService)); + AthenzService barService = new AthenzService("my.domain.bar"); + Files.createFile(SiaUtils.getPrivateKeyFile(siaRoot, barService)); + + List<AthenzService> siaIdentities = SiaUtils.findSiaServices(siaRoot); + assertThat(siaIdentities.size(), equalTo(2)); + assertThat(siaIdentities, hasItem(fooService)); + assertThat(siaIdentities, hasItem(barService)); + } + +}
\ No newline at end of file diff --git a/vespabase/src/common-env.sh b/vespabase/src/common-env.sh index 8dfcf8d2c4c..018630b0622 100755 --- a/vespabase/src/common-env.sh +++ b/vespabase/src/common-env.sh @@ -107,6 +107,13 @@ populate_environment () { fi } +add_valgrind_suppressions_file() { + if [ -f "$1" ] + then + VESPA_VALGRIND_SUPPREESSIONS_OPT="$VESPA_VALGRIND_SUPPREESSIONS_OPT --suppressions=$1" + fi +} + populate_environment PATH=$VESPA_HOME/bin64:$VESPA_HOME/bin:/usr/local/bin:/usr/X11R6/bin:/sbin:/bin:/usr/sbin:/usr/bin @@ -119,13 +126,17 @@ if [ "$JAVA_HOME" ] && [ -f "${JAVA_HOME}/bin/java" ]; then PATH="${PATH}:${JAVA_HOME}/bin" fi +VESPA_VALGRIND_SUPPREESSIONS_OPT="" +add_valgrind_suppressions_file ${VESPA_HOME}/etc/vespa/valgrind-suppressions.txt +add_valgrind_suppressions_file ${VESPA_HOME}/etc/vespa/suppressions.txt + consider_fallback VESPA_VALGRIND_OPT "--num-callers=32 \ --run-libc-freeres=yes \ --track-origins=yes \ --freelist-vol=1000000000 \ --leak-check=full \ --show-reachable=yes \ ---suppressions=${VESPA_HOME}/etc/vespa/suppressions.txt" +${VESPA_VALGRIND_SUPPREESSIONS_OPT}" consider_fallback VESPA_USE_HUGEPAGES_LIST $(get_var "hugepages_list") consider_fallback VESPA_USE_HUGEPAGES_LIST "all" diff --git a/vespalib/src/vespa/vespalib/io/fileutil.cpp b/vespalib/src/vespa/vespalib/io/fileutil.cpp index e360c84f569..5ab5fb99a0d 100644 --- a/vespalib/src/vespa/vespalib/io/fileutil.cpp +++ b/vespalib/src/vespa/vespalib/io/fileutil.cpp @@ -408,6 +408,15 @@ File::sync() } } +void +File::sync(vespalib::stringref path) +{ + File file(path); + file.open(READONLY); + file.sync(); + file.close(); +} + bool File::close() { diff --git a/vespalib/src/vespa/vespalib/io/fileutil.h b/vespalib/src/vespa/vespalib/io/fileutil.h index 187939ed412..1b493a9ebf1 100644 --- a/vespalib/src/vespa/vespalib/io/fileutil.h +++ b/vespalib/src/vespa/vespalib/io/fileutil.h @@ -190,6 +190,16 @@ public: */ static vespalib::string readAll(vespalib::stringref path); + /** + * Sync file or directory. + * + * This is a convenience function for the member functions open() and + * sync(), see there for more details. + * + * @throw IoException If we failed to sync the file. + */ + static void sync(vespalib::stringref path); + virtual void sync(); virtual bool close(); virtual bool unlink(); |