diff options
author | Harald Musum <musum@verizonmedia.com> | 2019-08-26 20:37:04 +0200 |
---|---|---|
committer | Harald Musum <musum@verizonmedia.com> | 2019-08-26 20:37:04 +0200 |
commit | a018fd1ebf965f9d8a71f8c0166c7d547cc27ae0 (patch) | |
tree | 4cf4c641db10d0eeccc1f006e26b654865366951 | |
parent | f4894027188f4221e92f770c98e6078c8dfd08b5 (diff) | |
parent | cb7cb4c365899b4f360785e5905aa132bdb4f9d3 (diff) |
Merge branch 'master' into revert-10388-revert-10370-revert-10346-revert-10336-hmusum/upgrade-to-zookeeper-3.4.14
51 files changed, 379 insertions, 207 deletions
diff --git a/athenz-identity-provider-service/pom.xml b/athenz-identity-provider-service/pom.xml index f4358394808..202d92d25a8 100644 --- a/athenz-identity-provider-service/pom.xml +++ b/athenz-identity-provider-service/pom.xml @@ -16,12 +16,45 @@ <dependencies> <!-- PROVIDED --> <dependency> + <groupId>com.google.inject</groupId> + <artifactId>guice</artifactId> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>component</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> + <dependency> <groupId>com.yahoo.vespa</groupId> <artifactId>container-dev</artifactId> <version>${project.version}</version> <scope>provided</scope> </dependency> <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>jdisc_core</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>vespalog</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>com.fasterxml.jackson.core</groupId> + <artifactId>jackson-core</artifactId> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>com.fasterxml.jackson.core</groupId> + <artifactId>jackson-databind</artifactId> + <scope>provided</scope> + </dependency> + <dependency> <groupId>org.eclipse.jetty</groupId> <artifactId>jetty-server</artifactId> <scope>provided</scope> @@ -55,6 +88,12 @@ <version>${project.version}</version> <scope>provided</scope> </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>security-utils</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> <!-- COMPILE --> <dependency> @@ -68,20 +107,19 @@ <!-- TEST --> <dependency> - <groupId>junit</groupId> - <artifactId>junit</artifactId> + <groupId>com.yahoo.vespa</groupId> + <artifactId>zkfacade</artifactId> + <version>${project.version}</version> <scope>test</scope> </dependency> <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>container-test</artifactId> - <version>${project.version}</version> + <groupId>org.hamcrest</groupId> + <artifactId>hamcrest-core</artifactId> <scope>test</scope> </dependency> <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>testutil</artifactId> - <version>${project.version}</version> + <groupId>junit</groupId> + <artifactId>junit</artifactId> <scope>test</scope> </dependency> <dependency> diff --git a/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/identitydocument/IdentityDocumentGeneratorTest.java b/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/identitydocument/IdentityDocumentGeneratorTest.java index 92ba8f0edfb..66ab5255b6f 100644 --- a/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/identitydocument/IdentityDocumentGeneratorTest.java +++ b/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/identitydocument/IdentityDocumentGeneratorTest.java @@ -24,10 +24,8 @@ import com.yahoo.vespa.hosted.provision.node.Allocation; import com.yahoo.vespa.hosted.provision.node.Generation; import com.yahoo.vespa.hosted.provision.node.IP; import com.yahoo.vespa.hosted.provision.testutils.MockNodeFlavors; -import org.hamcrest.Matchers; import org.junit.Test; -import java.util.HashSet; import java.util.Optional; import java.util.Set; @@ -35,6 +33,7 @@ import static com.yahoo.vespa.hosted.athenz.instanceproviderservice.TestUtils.ge import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; +import static org.hamcrest.core.IsCollectionContaining.hasItem; import static org.mockito.Matchers.eq; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -92,7 +91,7 @@ public class IdentityDocumentGeneratorTest { assertEquals(expectedProviderUniqueId, signedIdentityDocument.providerUniqueId()); // Validate that container ips are present - assertThat(signedIdentityDocument.ipAddresses(), Matchers.containsInAnyOrder("::1")); + assertThat(signedIdentityDocument.ipAddresses(), hasItem("::1")); IdentityDocumentSigner signer = new IdentityDocumentSigner(); diff --git a/container-search/src/main/java/com/yahoo/prelude/Index.java b/container-search/src/main/java/com/yahoo/prelude/Index.java index 8433939090b..65d5879b004 100644 --- a/container-search/src/main/java/com/yahoo/prelude/Index.java +++ b/container-search/src/main/java/com/yahoo/prelude/Index.java @@ -66,10 +66,16 @@ public class Index { private boolean numerical = false; private long predicateUpperBound = Long.MAX_VALUE; private long predicateLowerBound = Long.MIN_VALUE; + /** True if this is an <i>exact</i> index - which should match tokens containing any characters */ private boolean exact = false; + private boolean isNGram = false; - private int gramSize=2; + private int gramSize = 2; + + /** Whether implicit phrases should lead to a phrase item or an and item */ + private boolean phraseSegmenting = true; + /** The string terminating an exact token in this index, or null to use the default (space) */ private String exactTerminator = null; @@ -178,6 +184,10 @@ public class Index { setNumerical(true); } else if (commandString.startsWith("predicate-bounds ")) { setPredicateBounds(commandString.substring(17)); + } else if (commandString.equals("phrase-segmenting")) { + setPhraseSegmenting(true); + } else if (commandString.startsWith("phrase-segmenting ")) { + setPhraseSegmenting(Boolean.parseBoolean(commandString.substring("phrase-segmenting ".length()))); } else { commands.add(commandString); } @@ -307,6 +317,10 @@ public class Index { public long getPredicateLowerBound() { return predicateLowerBound; } + public boolean getPhraseSegmenting() { return phraseSegmenting; } + + public boolean setPhraseSegmenting(boolean phraseSegmenting) { return this.phraseSegmenting = phraseSegmenting; } + /** Returns all the literal command strings given as arguments to addCommand in this instance */ public List<String> allCommands() { return allCommands; } diff --git a/container-search/src/main/java/com/yahoo/prelude/IndexFacts.java b/container-search/src/main/java/com/yahoo/prelude/IndexFacts.java index 1fd8cce9889..76eef33d6c0 100644 --- a/container-search/src/main/java/com/yahoo/prelude/IndexFacts.java +++ b/container-search/src/main/java/com/yahoo/prelude/IndexFacts.java @@ -273,17 +273,13 @@ public class IndexFacts { return false; } - /** - * @return whether it is permissible to update this object - */ + /** Returns whether it is permissible to update this object */ public boolean isFrozen() { return frozen; } private void ensureNotFrozen() { - if (frozen) { - throw new IllegalStateException("Tried to modify frozen IndexFacts instance."); - } + if (frozen) throw new IllegalStateException("Tried to modify frozen IndexFacts instance."); } public String getDefaultPosition(String sdName) { @@ -307,7 +303,8 @@ public class IndexFacts { return new Session(sources, restrict); } - public Session newSession(Collection<String> sources, Collection<String> restrict, + public Session newSession(Collection<String> sources, + Collection<String> restrict, Set<String> candidateDocumentTypes) { return new Session(sources, restrict, candidateDocumentTypes); } diff --git a/container-search/src/main/java/com/yahoo/prelude/IndexModel.java b/container-search/src/main/java/com/yahoo/prelude/IndexModel.java index c83294efed7..062a514056b 100644 --- a/container-search/src/main/java/com/yahoo/prelude/IndexModel.java +++ b/container-search/src/main/java/com/yahoo/prelude/IndexModel.java @@ -24,7 +24,6 @@ public final class IndexModel { private static final Logger log = Logger.getLogger(IndexModel.class.getName()); - // Copied from MasterClustersInfoUpdater. It's a temporary workaround for IndexFacts private Map<String, List<String>> masterClusters; private Map<String, SearchDefinition> searchDefinitions; private SearchDefinition unionSearchDefinition; @@ -34,9 +33,6 @@ public final class IndexModel { this(Collections.emptyMap(), Collections.singleton(searchDefinition)); } - /** - * Create an index model. - */ public IndexModel(Map<String, List<String>> masterClusters, Collection<SearchDefinition> searchDefinitions) { this.masterClusters = masterClusters; this.searchDefinitions = searchDefinitions.stream().collect(Collectors.toMap(sd -> sd.getName(), sd -> sd)); diff --git a/container-search/src/main/java/com/yahoo/prelude/query/IndexedSegmentItem.java b/container-search/src/main/java/com/yahoo/prelude/query/IndexedSegmentItem.java index bf032ec03a8..b459f0aa902 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/IndexedSegmentItem.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/IndexedSegmentItem.java @@ -38,12 +38,14 @@ public abstract class IndexedSegmentItem extends TaggableSegmentItem implements } // encode index bytes + @Override protected void encodeThis(ByteBuffer buffer) { super.encodeThis(buffer); putString(index, buffer); } /** Sets the name of the index to search */ + @Override public void setIndexName(String index) { if (index == null) { index = ""; @@ -59,17 +61,17 @@ public abstract class IndexedSegmentItem extends TaggableSegmentItem implements } } + @Override public boolean equals(Object object) { - if (!super.equals(object)) { - return false; - } + if ( ! super.equals(object)) return false; + IndexedItem other = (IndexedItem) object; // Ensured by superclass - if (!this.index.equals(other.getIndexName())) { - return false; - } + if ( ! this.index.equals(other.getIndexName())) return false; + return true; } + @Override public int hashCode() { return super.hashCode() + 31 * index.hashCode(); } diff --git a/container-search/src/main/java/com/yahoo/prelude/query/PhraseItem.java b/container-search/src/main/java/com/yahoo/prelude/query/PhraseItem.java index 4de0af1f408..2d648557d9c 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/PhraseItem.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/PhraseItem.java @@ -76,15 +76,15 @@ public class PhraseItem extends CompositeIndexedItem { public void addItem(Item item) { if (item instanceof WordItem || item instanceof PhraseSegmentItem || item instanceof WordAlternativesItem) { addIndexedItem((IndexedItem) item); - } else if (item instanceof IntItem) { + } + else if (item instanceof IntItem) { addIndexedItem(convertIntToWord(item)); - } else if (item instanceof PhraseItem) { - PhraseItem phrase = (PhraseItem) item; - - for (Iterator<Item> i = phrase.getItemIterator(); i.hasNext();) { + } + else if (item instanceof PhraseItem || item instanceof AndSegmentItem) { + for (Iterator<Item> i = ((CompositeItem) item).getItemIterator(); i.hasNext();) addIndexedItem((IndexedItem) i.next()); - } - } else { + } + else { throw new IllegalArgumentException("Can not add " + item + " to a phrase"); } } diff --git a/container-search/src/main/java/com/yahoo/prelude/query/PhraseSegmentItem.java b/container-search/src/main/java/com/yahoo/prelude/query/PhraseSegmentItem.java index 53a57a968f5..542f1393852 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/PhraseSegmentItem.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/PhraseSegmentItem.java @@ -52,7 +52,7 @@ public class PhraseSegmentItem extends IndexedSegmentItem { } public PhraseSegmentItem(String rawWord, String current, boolean isFromQuery, - boolean stemmed, Substring substring) { + boolean stemmed, Substring substring) { super(rawWord, current, isFromQuery, stemmed, substring); } @@ -146,16 +146,16 @@ public class PhraseSegmentItem extends IndexedSegmentItem { } - /** - * Returns false, no parenthezes for phrases - */ + /** Returns false, no parenthezes for phrases */ protected boolean shouldParenthize() { return false; } /** Segment phrase items uses a empty heading instead of "SPHRASE " */ + @Override protected void appendHeadingString(StringBuilder buffer) {} + @Override protected void appendBodyString(StringBuilder buffer) { appendIndexString(buffer); appendContentsString(buffer); @@ -175,14 +175,13 @@ public class PhraseSegmentItem extends IndexedSegmentItem { } // TODO: Must check all pertinent items + @Override public boolean equals(Object object) { - if (!super.equals(object)) { - return false; - } - // PhraseSegmentItem other = (PhraseSegmentItem) object; // Ensured by superclass + if ( ! super.equals(object)) return false; return true; } + @Override public String getIndexedString() { StringBuilder buf = new StringBuilder(); diff --git a/container-search/src/main/java/com/yahoo/prelude/query/SegmentItem.java b/container-search/src/main/java/com/yahoo/prelude/query/SegmentItem.java index 1227a7f80cf..1c3eb261f90 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/SegmentItem.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/SegmentItem.java @@ -126,6 +126,7 @@ public abstract class SegmentItem extends CompositeItem implements BlockItem { // TODO: Add a getItemIterator which is safe for immutability /** Return a deep copy of this object */ + @Override public SegmentItem clone() { SegmentItem copy; synchronized(this) { diff --git a/container-search/src/main/java/com/yahoo/prelude/query/SimpleIndexedItem.java b/container-search/src/main/java/com/yahoo/prelude/query/SimpleIndexedItem.java index f230c76c954..a8dc9db1928 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/SimpleIndexedItem.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/SimpleIndexedItem.java @@ -38,6 +38,7 @@ public abstract class SimpleIndexedItem extends SimpleTaggableItem implements In } /** Sets the name of the index to search */ + @Override public void setIndexName(String index) { if (index == null) { index = ""; diff --git a/container-search/src/main/java/com/yahoo/prelude/query/SimpleTaggableItem.java b/container-search/src/main/java/com/yahoo/prelude/query/SimpleTaggableItem.java index 6cbd2286a8f..6f82f340f4b 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/SimpleTaggableItem.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/SimpleTaggableItem.java @@ -25,7 +25,7 @@ public abstract class SimpleTaggableItem extends Item implements TaggableItem { uniqueID = id; } - /** See {@link TaggableItem#setConnectivity} */ + @Override public void setConnectivity(Item item, double connectivity) { setHasUniqueID(true); item.setHasUniqueID(true); diff --git a/container-search/src/main/java/com/yahoo/prelude/query/TaggableItem.java b/container-search/src/main/java/com/yahoo/prelude/query/TaggableItem.java index 03e85fa3260..8e2a472fd7c 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/TaggableItem.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/TaggableItem.java @@ -34,7 +34,6 @@ public interface TaggableItem { Item getConnectedItem(); double getConnectivity(); - /** * Used for setting explicit term significance (in the tf/idf sense) to a single term or phrase, * relative to the rest of the query. diff --git a/container-search/src/main/java/com/yahoo/prelude/query/TermItem.java b/container-search/src/main/java/com/yahoo/prelude/query/TermItem.java index 8fdf147ce0b..2c33e7a2630 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/TermItem.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/TermItem.java @@ -44,7 +44,7 @@ public abstract class TermItem extends SimpleIndexedItem implements BlockItem { this.origin = origin; } - final public int encode(ByteBuffer buffer) { + public final int encode(ByteBuffer buffer) { encodeThis(buffer); return 1; } diff --git a/container-search/src/main/java/com/yahoo/prelude/query/parser/AbstractParser.java b/container-search/src/main/java/com/yahoo/prelude/query/parser/AbstractParser.java index 8297a566a72..cd8579be7f0 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/parser/AbstractParser.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/parser/AbstractParser.java @@ -228,13 +228,13 @@ public abstract class AbstractParser implements CustomParser { protected abstract Item parseItems(); /** - * Assigns the default index to query terms having no default index The - * parser _should_ have done this, for some reason it doesn't + * Assigns the default index to query terms having no default index. The + * parser _should_ have done this, for some reason it doesn't. * - * @param defaultIndex The default index to assign. - * @param item The item to check. + * @param defaultIndex the default index to assign + * @param item the item to check */ - private static void assignDefaultIndex(final String defaultIndex, Item item) { + private static void assignDefaultIndex(String defaultIndex, Item item) { if (defaultIndex == null || item == null) return; if (item instanceof IndexedItem) { @@ -253,9 +253,6 @@ public abstract class AbstractParser implements CustomParser { /** * Unicode normalizes some piece of natural language text. The chosen form * is compatibility decomposition, canonical composition (NFKC). - * - * @param input The string to normalize. - * @return The normalized string. */ protected String normalize(String input) { if (input == null || input.length() == 0) return input; @@ -272,8 +269,8 @@ public abstract class AbstractParser implements CustomParser { /** * Tokenizes the given string and initializes tokens with the found tokens. * - * @param query the string to tokenize. - * @param defaultIndexName the name of the index to use as default. + * @param query the string to tokenize + * @param defaultIndexName the name of the index to use as default * @param indexFacts resolved information about the index we are searching * @param language the language set for this query, or null if none */ @@ -324,6 +321,13 @@ public abstract class AbstractParser implements CustomParser { } } + /** + * Segments a token + * + * @param indexName the index name which preceeded this token, or null if none + * @param token the token to segment + * @return the resulting item + */ // TODO: The segmenting stuff is a mess now, this will fix it: // - Make Segmenter a class which is instantiated per parsing // - Make the instance know the language, etc and do all dispatching internally @@ -331,13 +335,13 @@ public abstract class AbstractParser implements CustomParser { // TODO: Use segmenting for forced phrase searches? // // Language detection currently depends on tokenization (see generateLanguageDetectionTextFrom), but - // - the API's was originally not constructed for that, so a careful nd somewhat unsatisfactory dance - // most be carried out to make it work + // - the API's was originally not constructed for that, so a careful and somewhat unsatisfactory dance + // must be carried out to make it work // - it should really depend on parsing // This can be solved by making the segment method language independent by // always producing a query item containing the token text and resolve it to a WordItem or // SegmentItem after parsing and language detection. - protected Item segment(Token token) { + protected Item segment(String indexName, Token token) { String normalizedToken = normalize(token.toString()); if (token.isSpecial()) { @@ -361,13 +365,23 @@ public abstract class AbstractParser implements CustomParser { return new WordItem(segments.get(0), "", true, token.substring); } - CompositeItem composite = new PhraseSegmentItem(token.toString(), normalizedToken, true, false, token.substring); + CompositeItem composite; + if (indexFacts.getIndex(indexName).getPhraseSegmenting()) { + composite = new PhraseSegmentItem(token.toString(), normalizedToken, true, false, token.substring); + } + else { + composite = new AndSegmentItem(token.toString(), true, false); + } int n = 0; + WordItem previous = null; for (String segment : segments) { WordItem w = new WordItem(segment, "", true, token.substring); w.setFromSegmented(true); w.setSegmentIndex(n++); w.setStemmed(false); + if (previous != null) + previous.setConnectivity(w, 1.0); + previous = w; composite.addItem(w); } composite.lock(); diff --git a/container-search/src/main/java/com/yahoo/prelude/query/parser/PhraseParser.java b/container-search/src/main/java/com/yahoo/prelude/query/parser/PhraseParser.java index 2d7e55a10fc..6d4401aca04 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/parser/PhraseParser.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/parser/PhraseParser.java @@ -38,7 +38,7 @@ public class PhraseParser extends AbstractParser { } // Note, this depends on segment never creating AndItems when quoted // (the second argument) is true. - Item newWord = segment(token); + Item newWord = segment(null, token); if (firstWord == null) { // First pass firstWord = newWord; diff --git a/container-search/src/main/java/com/yahoo/prelude/query/parser/StructuredParser.java b/container-search/src/main/java/com/yahoo/prelude/query/parser/StructuredParser.java index 8c77b3d2130..ee4c0d4d9f0 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/parser/StructuredParser.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/parser/StructuredParser.java @@ -63,12 +63,12 @@ abstract class StructuredParser extends AbstractParser { item = number(); if (item == null) { - item = phrase(); + item = phrase(indexName); } if (item == null && indexName != null) { if (wordsAhead()) { - item = phrase(); + item = phrase(indexName); } } @@ -407,12 +407,12 @@ abstract class StructuredParser extends AbstractParser { } /** Words for phrases also permits numerals as words */ - private Item phraseWord(boolean insidePhrase) { + private Item phraseWord(String indexName, boolean insidePhrase) { int position = tokens.getPosition(); Item item = null; try { - item = word(); + item = word(indexName); if (item == null && tokens.currentIs(NUMBER)) { Token t = tokens.next(); @@ -437,7 +437,7 @@ abstract class StructuredParser extends AbstractParser { * a WordItem or PhraseSegmentItem if this is a CJK query, * null if the current item is not a word */ - private Item word() { + private Item word(String indexName) { int position = tokens.getPosition(); Item item = null; @@ -452,7 +452,7 @@ abstract class StructuredParser extends AbstractParser { if (submodes.url) { item = new WordItem(word, true); } else { - item = segment(word); + item = segment(indexName, word); } if (submodes.url || submodes.site) { @@ -499,15 +499,16 @@ abstract class StructuredParser extends AbstractParser { * An phrase or word, either marked by quotes or by non-spaces between * words or by a combination. * + * @param indexName the index name which preceeded this phrase, or null if none * @return a word if there's only one word, a phrase if there is * several quoted or non-space-separated words, or null otherwise */ - private Item phrase() { + private Item phrase(String indexName) { int position = tokens.getPosition(); Item item = null; try { - item = phraseBody(); + item = phraseBody(indexName); return item; } finally { if (item == null) { @@ -516,8 +517,8 @@ abstract class StructuredParser extends AbstractParser { } } - /** Returns a word, a phrase or another composite */ - private Item phraseBody() { + /** Returns a word, a phrase, or another composite */ + private Item phraseBody(String indexName) { boolean quoted = false; PhraseItem phrase = null; Item firstWord = null; @@ -538,7 +539,7 @@ abstract class StructuredParser extends AbstractParser { quoted = !quoted; } - Item word = phraseWord((firstWord != null) || (phrase != null)); + Item word = phraseWord(indexName, (firstWord != null) || (phrase != null)); if (word == null) { if (tokens.skipMultiple(QUOTE)) { diff --git a/container-search/src/main/java/com/yahoo/search/query/parser/ParserEnvironment.java b/container-search/src/main/java/com/yahoo/search/query/parser/ParserEnvironment.java index ca437fb9def..9e53f9d8ea9 100644 --- a/container-search/src/main/java/com/yahoo/search/query/parser/ParserEnvironment.java +++ b/container-search/src/main/java/com/yahoo/search/query/parser/ParserEnvironment.java @@ -50,19 +50,17 @@ public final class ParserEnvironment { public static ParserEnvironment fromExecutionContext(Execution.Context context) { ParserEnvironment env = new ParserEnvironment(); - if (context == null) { - return env; - } - if (context.getIndexFacts() != null) { + if (context == null) return env; + + if (context.getIndexFacts() != null) env.setIndexFacts(context.getIndexFacts()); - } - if (context.getLinguistics() != null) { + + if (context.getLinguistics() != null) env.setLinguistics(context.getLinguistics()); - } - SpecialTokenRegistry registry = context.getTokenRegistry(); - if (registry != null) { - env.setSpecialTokens(registry.getSpecialTokens("default")); - } + + if (context.getTokenRegistry() != null) + env.setSpecialTokens(context.getTokenRegistry().getSpecialTokens("default")); + return env; } diff --git a/container-search/src/main/java/com/yahoo/search/searchchain/Execution.java b/container-search/src/main/java/com/yahoo/search/searchchain/Execution.java index e2c79fa5a7e..5cc34ff5b28 100644 --- a/container-search/src/main/java/com/yahoo/search/searchchain/Execution.java +++ b/container-search/src/main/java/com/yahoo/search/searchchain/Execution.java @@ -59,15 +59,10 @@ public class Execution extends com.yahoo.processing.execution.Execution { */ public static final class Context { - /** - * Whether the search should perform detailed diagnostics. - */ + /** Whether the search should perform detailed diagnostics. */ private boolean detailedDiagnostics = false; - /** - * Whether the container was considered to be in a breakdown state when - * this query started. - */ + /** Whether the container was considered to be in a breakdown state when this query started. */ private boolean breakdown = false; /** @@ -79,19 +74,13 @@ public class Execution extends com.yahoo.processing.execution.Execution { private IndexFacts indexFacts = null; - /** - * The current set of special tokens. - */ + /** The current set of special tokens */ private SpecialTokenRegistry tokenRegistry = null; - /** - * The current template registry. - */ + /** The current template registry */ private RendererRegistry rendererRegistry = null; - /** - * The current linguistics. - */ + /** The current linguistics */ private Linguistics linguistics = null; /** Always set if this context belongs to an execution, never set if it does not. */ diff --git a/container-search/src/test/java/com/yahoo/prelude/query/parser/test/parseindexinfo.cfg b/container-search/src/test/java/com/yahoo/prelude/query/parser/test/parseindexinfo.cfg index 0d264e04799..4a4a5c3a04d 100644 --- a/container-search/src/test/java/com/yahoo/prelude/query/parser/test/parseindexinfo.cfg +++ b/container-search/src/test/java/com/yahoo/prelude/query/parser/test/parseindexinfo.cfg @@ -1,6 +1,6 @@ indexinfo[3] indexinfo[0].name one -indexinfo[0].command[44] +indexinfo[0].command[45] indexinfo[0].command[0].indexname url.all indexinfo[0].command[0].command fullurl indexinfo[0].command[1].indexname host.all @@ -89,6 +89,8 @@ indexinfo[0].command[42].indexname exactindex indexinfo[0].command[42].command index indexinfo[0].command[43].indexname exactindex indexinfo[0].command[43].command exact +indexinfo[0].command[44].indexname phraseSegment +indexinfo[0].command[44].command "phrase-segmenting false" indexinfo[1].name twoRanges indexinfo[1].command[2] @@ -109,3 +111,4 @@ indexinfo[2].command[3].indexname link indexinfo[2].command[3].command index indexinfo[2].command[4].indexname url indexinfo[2].command[4].command index + diff --git a/container-search/src/test/java/com/yahoo/search/test/QueryTestCase.java b/container-search/src/test/java/com/yahoo/search/test/QueryTestCase.java index 45f270b2e37..fd29f2c12fe 100644 --- a/container-search/src/test/java/com/yahoo/search/test/QueryTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/test/QueryTestCase.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.test; +import com.google.common.collect.ImmutableList; import com.yahoo.component.chain.Chain; import com.yahoo.language.Language; import com.yahoo.language.Linguistics; @@ -42,7 +43,9 @@ import org.junit.Test; import java.io.UnsupportedEncodingException; import java.net.URLEncoder; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; +import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -877,6 +880,57 @@ public class QueryTestCase { } } + @Test + public void testImplicitPhraseIsDefault() { + Query query = new Query(httpEncode("?query=it's fine")); + assertEquals("AND 'it s' fine", query.getModel().getQueryTree().toString()); + } + + @Test + public void testImplicitPhrase() { + Query query = new Query(httpEncode("?query=myfield:it's myfield:fine")); + + SearchDefinition test = new SearchDefinition("test"); + Index myField = new Index("myfield"); + myField.addCommand("phrase-segmenting true"); + assertTrue(myField.getPhraseSegmenting()); + test.addIndex(myField); + IndexModel indexModel = new IndexModel(test); + query.getModel().setExecution(new Execution(Execution.Context.createContextStub(new IndexFacts(indexModel)))); + + assertEquals("AND myfield:'it s' myfield:fine", query.getModel().getQueryTree().toString()); + } + + @Test + public void testImplicitAnd() { + Query query = new Query(httpEncode("?query=myfield:it's myfield:fine")); + + SearchDefinition test = new SearchDefinition("test"); + Index myField = new Index("myfield"); + myField.addCommand("phrase-segmenting false"); + assertFalse(myField.getPhraseSegmenting()); + test.addIndex(myField); + IndexModel indexModel = new IndexModel(test); + query.getModel().setExecution(new Execution(Execution.Context.createContextStub(new IndexFacts(indexModel)))); + + assertEquals("AND (SAND myfield:it myfield:s) myfield:fine", query.getModel().getQueryTree().toString()); + } + + @Test + public void testImplicitAndInPhrase() { + Query query = new Query(httpEncode("?query=myfield:\"it's fine\"")); + + SearchDefinition test = new SearchDefinition("test"); + Index myField = new Index("myfield"); + myField.addCommand("phrase-segmenting false"); + assertFalse(myField.getPhraseSegmenting()); + test.addIndex(myField); + IndexModel indexModel = new IndexModel(test); + query.getModel().setExecution(new Execution(Execution.Context.createContextStub(new IndexFacts(indexModel)))); + + assertEquals("myfield:\"it s fine\"", query.getModel().getQueryTree().toString()); + } + private void assertDetectionText(String expectedDetectionText, String queryString, String ... indexSpecs) { Query q = new Query(httpEncode("/?query=" + queryString)); SearchDefinition sd = new SearchDefinition("testSearchDefinition"); diff --git a/dist/vespa.spec b/dist/vespa.spec index 50216f6fdde..285c54287dc 100644 --- a/dist/vespa.spec +++ b/dist/vespa.spec @@ -37,6 +37,7 @@ BuildRequires: llvm5.0-devel BuildRequires: vespa-boost-devel >= 1.59.0-6 BuildRequires: vespa-gtest >= 1.8.1-1 BuildRequires: vespa-protobuf-devel >= 3.7.0-4 +BuildRequires: vespa-openssl-devel >= 1.1.1c-1 %endif %if 0%{?fedora} BuildRequires: cmake >= 3.9.1 @@ -108,6 +109,7 @@ Requires: gdb Requires: net-tools %if 0%{?centos} Requires: llvm5.0 +Requires: vespa-openssl >= 1.1.1c-1 Requires: vespa-protobuf >= 3.7.0-4 %define _vespa_llvm_version 5.0 %define _extra_link_directory /usr/lib64/llvm5.0/lib;%{_vespa_deps_prefix}/lib64 @@ -131,7 +133,6 @@ Requires: llvm-libs >= 8.0.0 %define _extra_include_directory %{_vespa_deps_prefix}/include %endif Requires: java-11-openjdk -Requires: openssl Requires(pre): shadow-utils # Ugly workaround because vespamalloc/src/vespamalloc/malloc/mmap.cpp uses the private diff --git a/searchcore/src/tests/proton/documentdb/document_subdbs/document_subdbs_test.cpp b/searchcore/src/tests/proton/documentdb/document_subdbs/document_subdbs_test.cpp index 68c388111d3..76b6bf55534 100644 --- a/searchcore/src/tests/proton/documentdb/document_subdbs/document_subdbs_test.cpp +++ b/searchcore/src/tests/proton/documentdb/document_subdbs/document_subdbs_test.cpp @@ -235,7 +235,7 @@ MySearchableContext::MySearchableContext(IThreadingService &writeService, IBucketDBHandlerInitializer & bucketDBHandlerInitializer) : _fastUpdCtx(writeService, bucketDB, bucketDBHandlerInitializer), _queryLimiter(), _clock(), - _ctx(_fastUpdCtx._ctx, _queryLimiter, _clock, writeService.shared()) + _ctx(_fastUpdCtx._ctx, _queryLimiter, _clock, dynamic_cast<vespalib::SyncableThreadExecutor &>(writeService.shared())) {} MySearchableContext::~MySearchableContext() = default; diff --git a/searchcore/src/vespa/searchcore/proton/docsummary/summarymanagerinitializer.h b/searchcore/src/vespa/searchcore/proton/docsummary/summarymanagerinitializer.h index e7cb37aea28..6c2d3e3d4cc 100644 --- a/searchcore/src/vespa/searchcore/proton/docsummary/summarymanagerinitializer.h +++ b/searchcore/src/vespa/searchcore/proton/docsummary/summarymanagerinitializer.h @@ -43,7 +43,7 @@ public: search::transactionlog::SyncProxy &tlSyncer, IBucketizerSP bucketizer, std::shared_ptr<SummaryManager::SP> result); - ~SummaryManagerInitializer(); + ~SummaryManagerInitializer() override; void run() override; }; diff --git a/searchcore/src/vespa/searchcore/proton/index/index_manager_initializer.cpp b/searchcore/src/vespa/searchcore/proton/index/index_manager_initializer.cpp index 9c680ec7768..e87f74e2258 100644 --- a/searchcore/src/vespa/searchcore/proton/index/index_manager_initializer.cpp +++ b/searchcore/src/vespa/searchcore/proton/index/index_manager_initializer.cpp @@ -15,7 +15,7 @@ IndexManagerInitializer(const vespalib::string &baseDir, search::SerialNum serialNum, searchcorespi::IIndexManager::Reconfigurer & reconfigurer, searchcorespi::index::IThreadingService & threadingService, - vespalib::ThreadExecutor & warmupExecutor, + vespalib::SyncableThreadExecutor & warmupExecutor, const search::TuneFileIndexManager & tuneFileIndexManager, const search::TuneFileAttributes &tuneFileAttributes, const search::common::FileHeaderContext & fileHeaderContext, diff --git a/searchcore/src/vespa/searchcore/proton/index/index_manager_initializer.h b/searchcore/src/vespa/searchcore/proton/index/index_manager_initializer.h index 1a14c8d2ea3..8e446e222b3 100644 --- a/searchcore/src/vespa/searchcore/proton/index/index_manager_initializer.h +++ b/searchcore/src/vespa/searchcore/proton/index/index_manager_initializer.h @@ -20,7 +20,7 @@ class IndexManagerInitializer : public initializer::InitializerTask search::SerialNum _serialNum; searchcorespi::IIndexManager::Reconfigurer &_reconfigurer; searchcorespi::index::IThreadingService &_threadingService; - vespalib::ThreadExecutor &_warmupExecutor; + vespalib::SyncableThreadExecutor &_warmupExecutor; const search::TuneFileIndexManager _tuneFileIndexManager; const search::TuneFileAttributes _tuneFileAttributes; const search::common::FileHeaderContext &_fileHeaderContext; @@ -33,7 +33,7 @@ public: search::SerialNum serialNum, searchcorespi::IIndexManager::Reconfigurer & reconfigurer, searchcorespi::index::IThreadingService & threadingService, - vespalib::ThreadExecutor & warmupExecutor, + vespalib::SyncableThreadExecutor & warmupExecutor, const search::TuneFileIndexManager & tuneFileIndexManager, const search::TuneFileAttributes & tuneFileAttributes, const search::common::FileHeaderContext & fileHeaderContext, diff --git a/searchcore/src/vespa/searchcore/proton/index/indexmanager.cpp b/searchcore/src/vespa/searchcore/proton/index/indexmanager.cpp index 8e838414015..6ffd3203019 100644 --- a/searchcore/src/vespa/searchcore/proton/index/indexmanager.cpp +++ b/searchcore/src/vespa/searchcore/proton/index/indexmanager.cpp @@ -78,7 +78,7 @@ IndexManager::IndexManager(const vespalib::string &baseDir, SerialNum serialNum, Reconfigurer &reconfigurer, IThreadingService &threadingService, - vespalib::ThreadExecutor & warmupExecutor, + vespalib::SyncableThreadExecutor & warmupExecutor, const search::TuneFileIndexManager &tuneFileIndexManager, const search::TuneFileAttributes &tuneFileAttributes, const FileHeaderContext &fileHeaderContext) : diff --git a/searchcore/src/vespa/searchcore/proton/index/indexmanager.h b/searchcore/src/vespa/searchcore/proton/index/indexmanager.h index b14912239a3..7e305681f78 100644 --- a/searchcore/src/vespa/searchcore/proton/index/indexmanager.h +++ b/searchcore/src/vespa/searchcore/proton/index/indexmanager.h @@ -72,7 +72,7 @@ public: SerialNum serialNum, Reconfigurer &reconfigurer, searchcorespi::index::IThreadingService &threadingService, - vespalib::ThreadExecutor & warmupExecutor, + vespalib::SyncableThreadExecutor & warmupExecutor, const search::TuneFileIndexManager &tuneFileIndexManager, const search::TuneFileAttributes &tuneFileAttributes, const search::common::FileHeaderContext &fileHeaderContext); diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp b/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp index 5aacfa2678c..45784fc2683 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp @@ -119,7 +119,7 @@ DocumentDB::DocumentDB(const vespalib::string &baseDir, document::BucketSpace bucketSpace, const ProtonConfig &protonCfg, IDocumentDBOwner &owner, - vespalib::ThreadExecutor &warmupExecutor, + vespalib::SyncableThreadExecutor &warmupExecutor, vespalib::ThreadStackExecutorBase &sharedExecutor, search::transactionlog::Writer &tlsDirectWriter, MetricsWireService &metricsWireService, diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdb.h b/searchcore/src/vespa/searchcore/proton/server/documentdb.h index b4e58d6f178..d6448c0b515 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentdb.h +++ b/searchcore/src/vespa/searchcore/proton/server/documentdb.h @@ -247,7 +247,7 @@ public: document::BucketSpace bucketSpace, const ProtonConfig &protonCfg, IDocumentDBOwner &owner, - vespalib::ThreadExecutor &warmupExecutor, + vespalib::SyncableThreadExecutor &warmupExecutor, vespalib::ThreadStackExecutorBase &sharedExecutor, search::transactionlog::Writer &tlsDirectWriter, MetricsWireService &metricsWireService, diff --git a/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.cpp b/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.cpp index 682719436f1..74010391fcb 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.cpp @@ -34,7 +34,7 @@ DocumentSubDBCollection::DocumentSubDBCollection( const IGetSerialNum &getSerialNum, const DocTypeName &docTypeName, searchcorespi::index::IThreadingService &writeService, - vespalib::ThreadExecutor &warmupExecutor, + vespalib::SyncableThreadExecutor &warmupExecutor, const search::common::FileHeaderContext &fileHeaderContext, MetricsWireService &metricsWireService, DocumentDBTaggedMetrics &metrics, diff --git a/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.h b/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.h index 05427f5e545..d09afb92cc8 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.h +++ b/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.h @@ -11,7 +11,7 @@ namespace vespalib { class Clock; - class ThreadExecutor; + class SyncableThreadExecutor; class ThreadStackExecutorBase; } @@ -100,7 +100,7 @@ public: const IGetSerialNum &getSerialNum, const DocTypeName &docTypeName, searchcorespi::index::IThreadingService &writeService, - vespalib::ThreadExecutor &warmupExecutor, + vespalib::SyncableThreadExecutor &warmupExecutor, const search::common::FileHeaderContext &fileHeaderContext, MetricsWireService &metricsWireService, DocumentDBTaggedMetrics &metrics, diff --git a/searchcore/src/vespa/searchcore/proton/server/maintenancecontroller.cpp b/searchcore/src/vespa/searchcore/proton/server/maintenancecontroller.cpp index e6afb8a19b2..a2672cc7972 100644 --- a/searchcore/src/vespa/searchcore/proton/server/maintenancecontroller.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/maintenancecontroller.cpp @@ -32,7 +32,7 @@ public: } MaintenanceController::MaintenanceController(IThreadService &masterThread, - vespalib::ThreadExecutor & defaultExecutor, + vespalib::SyncableThreadExecutor & defaultExecutor, const DocTypeName &docTypeName) : IBucketFreezeListener(), _masterThread(masterThread), diff --git a/searchcore/src/vespa/searchcore/proton/server/maintenancecontroller.h b/searchcore/src/vespa/searchcore/proton/server/maintenancecontroller.h index 69416ff63d7..24c1c18959e 100644 --- a/searchcore/src/vespa/searchcore/proton/server/maintenancecontroller.h +++ b/searchcore/src/vespa/searchcore/proton/server/maintenancecontroller.h @@ -11,7 +11,7 @@ namespace vespalib { class Timer; - class ThreadExecutor; + class SyncableThreadExecutor; class Executor; } namespace searchcorespi { namespace index { struct IThreadService; }} @@ -34,7 +34,7 @@ public: using JobList = std::vector<std::shared_ptr<MaintenanceJobRunner>>; using UP = std::unique_ptr<MaintenanceController>; - MaintenanceController(IThreadService &masterThread, vespalib::ThreadExecutor & defaultExecutor, const DocTypeName &docTypeName); + MaintenanceController(IThreadService &masterThread, vespalib::SyncableThreadExecutor & defaultExecutor, const DocTypeName &docTypeName); virtual ~MaintenanceController(); void registerJobInMasterThread(IMaintenanceJob::UP job); @@ -73,7 +73,7 @@ private: using Guard = std::lock_guard<Mutex>; IThreadService &_masterThread; - vespalib::ThreadExecutor &_defaultExecutor; + vespalib::SyncableThreadExecutor &_defaultExecutor; MaintenanceDocumentSubDB _readySubDB; MaintenanceDocumentSubDB _remSubDB; MaintenanceDocumentSubDB _notReadySubDB; diff --git a/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.h b/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.h index e39ada31673..f3cf95ebfa0 100644 --- a/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.h +++ b/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.h @@ -52,15 +52,15 @@ public: }; struct Context { - const FastAccessDocSubDB::Context _fastUpdCtx; - matching::QueryLimiter &_queryLimiter; - const vespalib::Clock &_clock; - vespalib::ThreadExecutor &_warmupExecutor; + const FastAccessDocSubDB::Context _fastUpdCtx; + matching::QueryLimiter &_queryLimiter; + const vespalib::Clock &_clock; + vespalib::SyncableThreadExecutor &_warmupExecutor; Context(const FastAccessDocSubDB::Context &fastUpdCtx, matching::QueryLimiter &queryLimiter, const vespalib::Clock &clock, - vespalib::ThreadExecutor &warmupExecutor) + vespalib::SyncableThreadExecutor &warmupExecutor) : _fastUpdCtx(fastUpdCtx), _queryLimiter(queryLimiter), _clock(clock), @@ -80,7 +80,7 @@ private: vespalib::eval::ConstantValueCache _constantValueCache; matching::ConstantValueRepo _constantValueRepo; SearchableDocSubDBConfigurer _configurer; - vespalib::ThreadExecutor &_warmupExecutor; + vespalib::SyncableThreadExecutor &_warmupExecutor; std::shared_ptr<GidToLidChangeHandler> _realGidToLidChangeHandler; DocumentDBFlushConfig _flushConfig; bool _nodeRetired; diff --git a/searchcorespi/src/vespa/searchcorespi/index/i_thread_service.h b/searchcorespi/src/vespa/searchcorespi/index/i_thread_service.h index 40d92010c9b..65f18ac2c79 100644 --- a/searchcorespi/src/vespa/searchcorespi/index/i_thread_service.h +++ b/searchcorespi/src/vespa/searchcorespi/index/i_thread_service.h @@ -9,7 +9,7 @@ namespace searchcorespi::index { /** * Interface for a single thread used for write tasks. */ -struct IThreadService : public vespalib::ThreadExecutor +struct IThreadService : public vespalib::SyncableThreadExecutor { IThreadService(const IThreadService &) = delete; IThreadService & operator = (const IThreadService &) = delete; diff --git a/searchcorespi/src/vespa/searchcorespi/index/indexmaintainercontext.cpp b/searchcorespi/src/vespa/searchcorespi/index/indexmaintainercontext.cpp index 213d2cb8705..78b9930c69a 100644 --- a/searchcorespi/src/vespa/searchcorespi/index/indexmaintainercontext.cpp +++ b/searchcorespi/src/vespa/searchcorespi/index/indexmaintainercontext.cpp @@ -11,7 +11,7 @@ namespace searchcorespi::index { IndexMaintainerContext::IndexMaintainerContext(IThreadingService &threadingService, IIndexManager::Reconfigurer &reconfigurer, const FileHeaderContext &fileHeaderContext, - vespalib::ThreadExecutor & warmupExecutor) + vespalib::SyncableThreadExecutor & warmupExecutor) : _threadingService(threadingService), _reconfigurer(reconfigurer), _fileHeaderContext(fileHeaderContext), diff --git a/searchcorespi/src/vespa/searchcorespi/index/indexmaintainercontext.h b/searchcorespi/src/vespa/searchcorespi/index/indexmaintainercontext.h index 972a16effc3..ab4a447168e 100644 --- a/searchcorespi/src/vespa/searchcorespi/index/indexmaintainercontext.h +++ b/searchcorespi/src/vespa/searchcorespi/index/indexmaintainercontext.h @@ -15,15 +15,15 @@ namespace searchcorespi::index { class IndexMaintainerContext { private: IThreadingService &_threadingService; - searchcorespi::IIndexManager::Reconfigurer &_reconfigurer; + IIndexManager::Reconfigurer &_reconfigurer; const search::common::FileHeaderContext &_fileHeaderContext; - vespalib::ThreadExecutor & _warmupExecutor; + vespalib::SyncableThreadExecutor & _warmupExecutor; public: IndexMaintainerContext(IThreadingService &threadingService, - searchcorespi::IIndexManager::Reconfigurer &reconfigurer, + IIndexManager::Reconfigurer &reconfigurer, const search::common::FileHeaderContext &fileHeaderContext, - vespalib::ThreadExecutor & warmupExecutor); + vespalib::SyncableThreadExecutor & warmupExecutor); /** * Returns the treading service that encapsulates the thread model used for writing. @@ -35,7 +35,7 @@ public: /** * Returns the reconfigurer used to signal when the index maintainer has changed. */ - searchcorespi::IIndexManager::Reconfigurer &getReconfigurer() const { + IIndexManager::Reconfigurer &getReconfigurer() const { return _reconfigurer; } @@ -49,7 +49,7 @@ public: /** * @return The executor that should be used for warmup. */ - vespalib::ThreadExecutor & getWarmupExecutor() const { return _warmupExecutor; } + vespalib::SyncableThreadExecutor & getWarmupExecutor() const { return _warmupExecutor; } }; } diff --git a/searchcorespi/src/vespa/searchcorespi/index/warmupindexcollection.cpp b/searchcorespi/src/vespa/searchcorespi/index/warmupindexcollection.cpp index 911ff5f9eba..1786665c248 100644 --- a/searchcorespi/src/vespa/searchcorespi/index/warmupindexcollection.cpp +++ b/searchcorespi/src/vespa/searchcorespi/index/warmupindexcollection.cpp @@ -34,7 +34,7 @@ WarmupIndexCollection::WarmupIndexCollection(const WarmupConfig & warmupConfig, ISearchableIndexCollection::SP prev, ISearchableIndexCollection::SP next, IndexSearchable & warmup, - vespalib::ThreadExecutor & executor, + vespalib::SyncableThreadExecutor & executor, IWarmupDone & warmupDone) : _warmupConfig(warmupConfig), _prev(prev), diff --git a/searchcorespi/src/vespa/searchcorespi/index/warmupindexcollection.h b/searchcorespi/src/vespa/searchcorespi/index/warmupindexcollection.h index 229d1a7c31d..992cedb1057 100644 --- a/searchcorespi/src/vespa/searchcorespi/index/warmupindexcollection.h +++ b/searchcorespi/src/vespa/searchcorespi/index/warmupindexcollection.h @@ -30,9 +30,9 @@ public: ISearchableIndexCollection::SP prev, ISearchableIndexCollection::SP next, IndexSearchable & warmup, - vespalib::ThreadExecutor & executor, + vespalib::SyncableThreadExecutor & executor, IWarmupDone & warmupDone); - ~WarmupIndexCollection(); + ~WarmupIndexCollection() override; // Implements IIndexCollection const ISourceSelector &getSourceSelector() const override; size_t getSourceCount() const override; @@ -95,15 +95,15 @@ private: void fireWarmup(Task::UP task); bool handledBefore(uint32_t fieldId, const Node &term); - const WarmupConfig _warmupConfig; - ISearchableIndexCollection::SP _prev; - ISearchableIndexCollection::SP _next; - IndexSearchable & _warmup; - vespalib::ThreadExecutor & _executor; - IWarmupDone & _warmupDone; - fastos::TimeStamp _warmupEndTime; - std::mutex _lock; - std::unique_ptr<FieldTermMap> _handledTerms; + const WarmupConfig _warmupConfig; + ISearchableIndexCollection::SP _prev; + ISearchableIndexCollection::SP _next; + IndexSearchable & _warmup; + vespalib::SyncableThreadExecutor & _executor; + IWarmupDone & _warmupDone; + fastos::TimeStamp _warmupEndTime; + std::mutex _lock; + std::unique_ptr<FieldTermMap> _handledTerms; }; } // namespace searchcorespi diff --git a/searchlib/src/vespa/searchlib/docstore/compacter.cpp b/searchlib/src/vespa/searchlib/docstore/compacter.cpp index 4d154da9907..b53281b4dbb 100644 --- a/searchlib/src/vespa/searchlib/docstore/compacter.cpp +++ b/searchlib/src/vespa/searchlib/docstore/compacter.cpp @@ -7,8 +7,7 @@ #include <vespa/log/log.h> LOG_SETUP(".searchlib.docstore.compacter"); -namespace search { -namespace docstore { +namespace search::docstore { using vespalib::alloc::Alloc; @@ -19,7 +18,7 @@ Compacter::write(LockGuard guard, uint32_t chunkId, uint32_t lid, const void *bu _ds.write(std::move(guard), fileId, lid, buffer, sz); } -BucketCompacter::BucketCompacter(size_t maxSignificantBucketBits, const CompressionConfig & compression, LogDataStore & ds, ThreadExecutor & executor, const IBucketizer & bucketizer, FileId source, FileId destination) : +BucketCompacter::BucketCompacter(size_t maxSignificantBucketBits, const CompressionConfig & compression, LogDataStore & ds, Executor & executor, const IBucketizer & bucketizer, FileId source, FileId destination) : _unSignificantBucketBits((maxSignificantBucketBits > 8) ? (maxSignificantBucketBits - 8) : 0), _sourceFileId(source), _destinationFileId(destination), @@ -97,4 +96,3 @@ BucketCompacter::write(BucketId bucketId, uint32_t chunkId, uint32_t lid, const } } -} diff --git a/searchlib/src/vespa/searchlib/docstore/compacter.h b/searchlib/src/vespa/searchlib/docstore/compacter.h index 362896487aa..666943ed629 100644 --- a/searchlib/src/vespa/searchlib/docstore/compacter.h +++ b/searchlib/src/vespa/searchlib/docstore/compacter.h @@ -6,11 +6,9 @@ #include "storebybucket.h" #include <vespa/vespalib/data/memorydatastore.h> -namespace search { +namespace search { class LogDataStore; } -class LogDataStore; - -namespace docstore { +namespace search::docstore { /** * A simple write through implementation of the IWriteData interface. @@ -34,11 +32,11 @@ private: class BucketCompacter : public IWriteData, public StoreByBucket::IWrite { using CompressionConfig = vespalib::compression::CompressionConfig; - using ThreadExecutor = vespalib::ThreadExecutor; + using Executor = vespalib::Executor; public: using FileId = FileChunk::FileId; BucketCompacter(size_t maxSignificantBucketBits, const CompressionConfig & compression, LogDataStore & ds, - ThreadExecutor & exeutor, const IBucketizer & bucketizer, FileId source, FileId destination); + Executor & exeutor, const IBucketizer & bucketizer, FileId source, FileId destination); void write(LockGuard guard, uint32_t chunkId, uint32_t lid, const void *buffer, size_t sz) override ; void write(BucketId bucketId, uint32_t chunkId, uint32_t lid, const void *buffer, size_t sz) override; void close() override; @@ -60,4 +58,3 @@ private: }; } -} diff --git a/searchlib/src/vespa/searchlib/docstore/logdatastore.cpp b/searchlib/src/vespa/searchlib/docstore/logdatastore.cpp index 46fcdafc585..e96628bdf4f 100644 --- a/searchlib/src/vespa/searchlib/docstore/logdatastore.cpp +++ b/searchlib/src/vespa/searchlib/docstore/logdatastore.cpp @@ -103,7 +103,6 @@ LogDataStore::~LogDataStore() { // Must be called before ending threads as there are sanity checks. _fileChunks.clear(); - _executor.sync(); _genHandler.updateFirstUsedGeneration(); _lidInfo.removeOldGenerations(_genHandler.getFirstUsedGeneration()); } diff --git a/searchlib/src/vespa/searchlib/docstore/storebybucket.cpp b/searchlib/src/vespa/searchlib/docstore/storebybucket.cpp index d0d2d1cdac2..13de5687ee0 100644 --- a/searchlib/src/vespa/searchlib/docstore/storebybucket.cpp +++ b/searchlib/src/vespa/searchlib/docstore/storebybucket.cpp @@ -12,20 +12,21 @@ using document::BucketId; using vespalib::makeTask; using vespalib::makeClosure; -StoreByBucket::StoreByBucket(MemoryDataStore & backingMemory, ThreadExecutor & executor, const CompressionConfig & compression) : - _chunkSerial(0), - _current(), - _where(), - _backingMemory(backingMemory), - _executor(executor), - _lock(), - _chunks(), - _compression(compression) +StoreByBucket::StoreByBucket(MemoryDataStore & backingMemory, Executor & executor, const CompressionConfig & compression) + : _chunkSerial(0), + _current(), + _where(), + _backingMemory(backingMemory), + _executor(executor), + _monitor(), + _numChunksPosted(0), + _chunks(), + _compression(compression) { createChunk().swap(_current); } -StoreByBucket::~StoreByBucket() { } +StoreByBucket::~StoreByBucket() = default; void StoreByBucket::add(BucketId bucketId, uint32_t chunkId, uint32_t lid, const void *buffer, size_t sz) @@ -33,6 +34,7 @@ StoreByBucket::add(BucketId bucketId, uint32_t chunkId, uint32_t lid, const void if ( ! _current->hasRoom(sz)) { Chunk::UP tmpChunk = createChunk(); _current.swap(tmpChunk); + incChunksPosted(); _executor.execute(makeTask(makeClosure(this, &StoreByBucket::closeChunk, std::move(tmpChunk)))); } Index idx(bucketId, _current->getId(), chunkId, lid); @@ -48,7 +50,7 @@ StoreByBucket::createChunk() size_t StoreByBucket::getChunkCount() const { - vespalib::LockGuard guard(_lock); + vespalib::LockGuard guard(_monitor); return _chunks.size(); } @@ -59,15 +61,33 @@ StoreByBucket::closeChunk(Chunk::UP chunk) chunk->pack(1, buffer, _compression); buffer.shrink(buffer.getDataLen()); ConstBufferRef bufferRef(_backingMemory.push_back(buffer.getData(), buffer.getDataLen()).data(), buffer.getDataLen()); - vespalib::LockGuard guard(_lock); + vespalib::MonitorGuard guard(_monitor); _chunks[chunk->getId()] = bufferRef; + if (_numChunksPosted == _chunks.size()) { + guard.signal(); + } +} + +void +StoreByBucket::incChunksPosted() { + vespalib::MonitorGuard guard(_monitor); + _numChunksPosted++; +} + +void +StoreByBucket::waitAllProcessed() { + vespalib::MonitorGuard guard(_monitor); + while (_numChunksPosted != _chunks.size()) { + guard.wait(); + } } void StoreByBucket::drain(IWrite & drainer) { + incChunksPosted(); _executor.execute(makeTask(makeClosure(this, &StoreByBucket::closeChunk, std::move(_current)))); - _executor.sync(); + waitAllProcessed(); std::vector<Chunk::UP> chunks; chunks.resize(_chunks.size()); for (const auto & it : _chunks) { diff --git a/searchlib/src/vespa/searchlib/docstore/storebybucket.h b/searchlib/src/vespa/searchlib/docstore/storebybucket.h index ac1f6fbe007..8be0610b588 100644 --- a/searchlib/src/vespa/searchlib/docstore/storebybucket.h +++ b/searchlib/src/vespa/searchlib/docstore/storebybucket.h @@ -5,13 +5,12 @@ #include "chunk.h" #include <vespa/document/bucket/bucketid.h> #include <vespa/vespalib/data/memorydatastore.h> -#include <vespa/vespalib/util/threadexecutor.h> +#include <vespa/vespalib/util/executor.h> #include <vespa/vespalib/util/sync.h> #include <vespa/vespalib/stllike/hash_map.h> #include <map> -namespace search { -namespace docstore { +namespace search::docstore { /** * StoreByBucket will organize the data you add to it by buckets. @@ -21,12 +20,12 @@ namespace docstore { class StoreByBucket { using MemoryDataStore = vespalib::MemoryDataStore; - using ThreadExecutor = vespalib::ThreadExecutor; + using Executor = vespalib::Executor; using ConstBufferRef = vespalib::ConstBufferRef; using CompressionConfig = vespalib::compression::CompressionConfig; public: StoreByBucket(vespalib::MemoryDataStore & backingMemory, const CompressionConfig & compression); - StoreByBucket(MemoryDataStore & backingMemory, ThreadExecutor & executor, const CompressionConfig & compression); + StoreByBucket(MemoryDataStore & backingMemory, Executor & executor, const CompressionConfig & compression); StoreByBucket(StoreByBucket &&) = default; ~StoreByBucket(); class IWrite { @@ -47,6 +46,8 @@ public: return lidCount; } private: + void incChunksPosted(); + void waitAllProcessed(); Chunk::UP createChunk(); void closeChunk(Chunk::UP chunk); struct Index { @@ -67,11 +68,11 @@ private: Chunk::UP _current; std::map<uint64_t, IndexVector> _where; MemoryDataStore & _backingMemory; - ThreadExecutor & _executor; - vespalib::Lock _lock; + Executor & _executor; + vespalib::Monitor _monitor; + size_t _numChunksPosted; vespalib::hash_map<uint64_t, ConstBufferRef> _chunks; CompressionConfig _compression; }; } -} diff --git a/searchlib/src/vespa/searchlib/transactionlog/domain.h b/searchlib/src/vespa/searchlib/transactionlog/domain.h index c0ee484926c..d6f964d5140 100644 --- a/searchlib/src/vespa/searchlib/transactionlog/domain.h +++ b/searchlib/src/vespa/searchlib/transactionlog/domain.h @@ -39,7 +39,7 @@ class Domain { public: using SP = std::shared_ptr<Domain>; - using Executor = vespalib::ThreadExecutor; + using Executor = vespalib::SyncableThreadExecutor; Domain(const vespalib::string &name, const vespalib::string &baseDir, Executor & commitExecutor, Executor & sessionExecutor, uint64_t domainPartSize, DomainPart::Crc defaultCrcType, const common::FileHeaderContext &fileHeaderContext); diff --git a/security-utils/src/main/java/com/yahoo/security/KeyFormat.java b/security-utils/src/main/java/com/yahoo/security/KeyFormat.java new file mode 100644 index 00000000000..a04e7951dfe --- /dev/null +++ b/security-utils/src/main/java/com/yahoo/security/KeyFormat.java @@ -0,0 +1,11 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.security; + +/** + * Key format + * + * @author bjorncs + */ +public enum KeyFormat { + PKCS1, PKCS8 +} diff --git a/security-utils/src/main/java/com/yahoo/security/KeyUtils.java b/security-utils/src/main/java/com/yahoo/security/KeyUtils.java index f847e78f3c5..ed3b41d6e2a 100644 --- a/security-utils/src/main/java/com/yahoo/security/KeyUtils.java +++ b/security-utils/src/main/java/com/yahoo/security/KeyUtils.java @@ -141,10 +141,36 @@ public class KeyUtils { } } + // Note: Encoding using PKCS#1 as default as this is to be read by tools only supporting PKCS#1 + // Should ideally be PKCS#8 public static String toPem(PrivateKey privateKey) { + return toPem(privateKey, KeyFormat.PKCS1); + } + + public static String toPem(PrivateKey privateKey, KeyFormat format) { + switch (format) { + case PKCS1: + return toPkcs1Pem(privateKey); + case PKCS8: + return toPkcs8Pem(privateKey); + default: + throw new IllegalArgumentException("Unknown format: " + format); + } + } + + public static String toPem(PublicKey publicKey) { + try (StringWriter stringWriter = new StringWriter(); JcaPEMWriter pemWriter = new JcaPEMWriter(stringWriter)) { + pemWriter.writeObject(publicKey); + pemWriter.flush(); + return stringWriter.toString(); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + private static String toPkcs1Pem(PrivateKey privateKey) { try (StringWriter stringWriter = new StringWriter(); JcaPEMWriter pemWriter = new JcaPEMWriter(stringWriter)) { String algorithm = privateKey.getAlgorithm(); - // Note: Encoding using PKCS#1 as this is to be read by tools only supporting PKCS#1 String type; if (algorithm.equals(RSA.getAlgorithmName())) { type = "RSA PRIVATE KEY"; @@ -161,9 +187,9 @@ public class KeyUtils { } } - public static String toPem(PublicKey publicKey) { + private static String toPkcs8Pem(PrivateKey privateKey) { try (StringWriter stringWriter = new StringWriter(); JcaPEMWriter pemWriter = new JcaPEMWriter(stringWriter)) { - pemWriter.writeObject(publicKey); + pemWriter.writeObject(new PemObject("PRIVATE KEY", privateKey.getEncoded())); pemWriter.flush(); return stringWriter.toString(); } catch (IOException e) { diff --git a/security-utils/src/test/java/com/yahoo/security/KeyUtilsTest.java b/security-utils/src/test/java/com/yahoo/security/KeyUtilsTest.java index dc0c0a126ea..8a1c72b1e59 100644 --- a/security-utils/src/test/java/com/yahoo/security/KeyUtilsTest.java +++ b/security-utils/src/test/java/com/yahoo/security/KeyUtilsTest.java @@ -32,25 +32,23 @@ public class KeyUtilsTest { } @Test - public void can_serialize_and_deserialize_rsa_privatekey_using_pem_format() { - KeyPair keyPair = KeyUtils.generateKeypair(KeyAlgorithm.RSA); - String pem = KeyUtils.toPem(keyPair.getPrivate()); - assertThat(pem, containsString("BEGIN RSA PRIVATE KEY")); - assertThat(pem, containsString("END RSA PRIVATE KEY")); - PrivateKey deserializedKey = KeyUtils.fromPemEncodedPrivateKey(pem); - assertEquals(keyPair.getPrivate(), deserializedKey); - assertEquals(KeyAlgorithm.RSA.getAlgorithmName(), deserializedKey.getAlgorithm()); + public void can_serialize_and_deserialize_rsa_privatekey_using_pkcs1_pem_format() { + testPrivateKeySerialization(KeyAlgorithm.RSA, KeyFormat.PKCS1, "RSA PRIVATE KEY"); } @Test - public void can_serialize_and_deserialize_ec_privatekey_using_pem_format() { - KeyPair keyPair = KeyUtils.generateKeypair(KeyAlgorithm.EC); - String pem = KeyUtils.toPem(keyPair.getPrivate()); - assertThat(pem, containsString("BEGIN EC PRIVATE KEY")); - assertThat(pem, containsString("END EC PRIVATE KEY")); - PrivateKey deserializedKey = KeyUtils.fromPemEncodedPrivateKey(pem); - assertEquals(keyPair.getPrivate(), deserializedKey); - assertEquals(KeyAlgorithm.EC.getAlgorithmName(), deserializedKey.getAlgorithm()); + public void can_serialize_and_deserialize_rsa_privatekey_using_pkcs8_pem_format() { + testPrivateKeySerialization(KeyAlgorithm.RSA, KeyFormat.PKCS8, "PRIVATE KEY"); + } + + @Test + public void can_serialize_and_deserialize_ec_privatekey_using_pkcs1_pem_format() { + testPrivateKeySerialization(KeyAlgorithm.EC, KeyFormat.PKCS1, "EC PRIVATE KEY"); + } + + @Test + public void can_serialize_and_deserialize_ec_privatekey_using_pkcs8_pem_format() { + testPrivateKeySerialization(KeyAlgorithm.EC, KeyFormat.PKCS8, "PRIVATE KEY"); } @Test @@ -75,4 +73,14 @@ public class KeyUtilsTest { assertEquals(KeyAlgorithm.EC.getAlgorithmName(), deserializedKey.getAlgorithm()); } + private static void testPrivateKeySerialization(KeyAlgorithm keyAlgorithm, KeyFormat keyFormat, String pemLabel) { + KeyPair keyPair = KeyUtils.generateKeypair(keyAlgorithm); + String pem = KeyUtils.toPem(keyPair.getPrivate(), keyFormat); + assertThat(pem, containsString("BEGIN " + pemLabel)); + assertThat(pem, containsString("END " + pemLabel)); + PrivateKey deserializedKey = KeyUtils.fromPemEncodedPrivateKey(pem); + assertEquals(keyPair.getPrivate(), deserializedKey); + assertEquals(keyAlgorithm.getAlgorithmName(), deserializedKey.getAlgorithm()); + } + } diff --git a/vespalib/src/vespa/vespalib/util/threadexecutor.h b/vespalib/src/vespa/vespalib/util/threadexecutor.h index 158805288e9..2dcbb595bb3 100644 --- a/vespalib/src/vespa/vespalib/util/threadexecutor.h +++ b/vespalib/src/vespa/vespalib/util/threadexecutor.h @@ -2,16 +2,12 @@ #pragma once -#include <vespa/vespalib/util/executor.h> -#include <vespa/vespalib/util/syncable.h> +#include "executor.h" +#include "syncable.h" namespace vespalib { -/** - * Can both execute and sync - **/ -class ThreadExecutor : public Executor, - public Syncable +class ThreadExecutor : public Executor { public: /** @@ -21,5 +17,13 @@ public: virtual size_t getNumThreads() const = 0; }; +/** + * Can both execute and sync + **/ +class SyncableThreadExecutor : public ThreadExecutor, public Syncable +{ +public: +}; + } // namespace vespalib diff --git a/vespalib/src/vespa/vespalib/util/threadstackexecutorbase.cpp b/vespalib/src/vespa/vespalib/util/threadstackexecutorbase.cpp index dfd835e0e8e..969b5e6f61e 100644 --- a/vespalib/src/vespa/vespalib/util/threadstackexecutorbase.cpp +++ b/vespalib/src/vespa/vespalib/util/threadstackexecutorbase.cpp @@ -135,7 +135,9 @@ ThreadStackExecutorBase::run() ThreadStackExecutorBase::ThreadStackExecutorBase(uint32_t stackSize, uint32_t taskLimit, init_fun_t init_fun) - : _pool(std::make_unique<FastOS_ThreadPool>(stackSize)), + : SyncableThreadExecutor(), + Runnable(), + _pool(std::make_unique<FastOS_ThreadPool>(stackSize)), _monitor(), _stats(), _executorCompletion(), diff --git a/vespalib/src/vespa/vespalib/util/threadstackexecutorbase.h b/vespalib/src/vespa/vespalib/util/threadstackexecutorbase.h index 8ee08ed3929..21a6e9cabe0 100644 --- a/vespalib/src/vespa/vespalib/util/threadstackexecutorbase.h +++ b/vespalib/src/vespa/vespalib/util/threadstackexecutorbase.h @@ -32,7 +32,7 @@ namespace thread { struct ThreadInit; } /** * An executor service that executes tasks in multiple threads. **/ -class ThreadStackExecutorBase : public ThreadExecutor, +class ThreadStackExecutorBase : public SyncableThreadExecutor, public Runnable { public: @@ -238,7 +238,7 @@ public: /** * Will invoke shutdown then sync. **/ - ~ThreadStackExecutorBase(); + ~ThreadStackExecutorBase() override; }; } // namespace vespalib |