aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorHarald Musum <musum@verizonmedia.com>2019-08-26 20:37:04 +0200
committerHarald Musum <musum@verizonmedia.com>2019-08-26 20:37:04 +0200
commita018fd1ebf965f9d8a71f8c0166c7d547cc27ae0 (patch)
tree4cf4c641db10d0eeccc1f006e26b654865366951
parentf4894027188f4221e92f770c98e6078c8dfd08b5 (diff)
parentcb7cb4c365899b4f360785e5905aa132bdb4f9d3 (diff)
Merge branch 'master' into revert-10388-revert-10370-revert-10346-revert-10336-hmusum/upgrade-to-zookeeper-3.4.14
-rw-r--r--athenz-identity-provider-service/pom.xml54
-rw-r--r--athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/identitydocument/IdentityDocumentGeneratorTest.java5
-rw-r--r--container-search/src/main/java/com/yahoo/prelude/Index.java16
-rw-r--r--container-search/src/main/java/com/yahoo/prelude/IndexFacts.java11
-rw-r--r--container-search/src/main/java/com/yahoo/prelude/IndexModel.java4
-rw-r--r--container-search/src/main/java/com/yahoo/prelude/query/IndexedSegmentItem.java14
-rw-r--r--container-search/src/main/java/com/yahoo/prelude/query/PhraseItem.java14
-rw-r--r--container-search/src/main/java/com/yahoo/prelude/query/PhraseSegmentItem.java15
-rw-r--r--container-search/src/main/java/com/yahoo/prelude/query/SegmentItem.java1
-rw-r--r--container-search/src/main/java/com/yahoo/prelude/query/SimpleIndexedItem.java1
-rw-r--r--container-search/src/main/java/com/yahoo/prelude/query/SimpleTaggableItem.java2
-rw-r--r--container-search/src/main/java/com/yahoo/prelude/query/TaggableItem.java1
-rw-r--r--container-search/src/main/java/com/yahoo/prelude/query/TermItem.java2
-rw-r--r--container-search/src/main/java/com/yahoo/prelude/query/parser/AbstractParser.java42
-rw-r--r--container-search/src/main/java/com/yahoo/prelude/query/parser/PhraseParser.java2
-rw-r--r--container-search/src/main/java/com/yahoo/prelude/query/parser/StructuredParser.java23
-rw-r--r--container-search/src/main/java/com/yahoo/search/query/parser/ParserEnvironment.java20
-rw-r--r--container-search/src/main/java/com/yahoo/search/searchchain/Execution.java21
-rw-r--r--container-search/src/test/java/com/yahoo/prelude/query/parser/test/parseindexinfo.cfg5
-rw-r--r--container-search/src/test/java/com/yahoo/search/test/QueryTestCase.java54
-rw-r--r--dist/vespa.spec3
-rw-r--r--searchcore/src/tests/proton/documentdb/document_subdbs/document_subdbs_test.cpp2
-rw-r--r--searchcore/src/vespa/searchcore/proton/docsummary/summarymanagerinitializer.h2
-rw-r--r--searchcore/src/vespa/searchcore/proton/index/index_manager_initializer.cpp2
-rw-r--r--searchcore/src/vespa/searchcore/proton/index/index_manager_initializer.h4
-rw-r--r--searchcore/src/vespa/searchcore/proton/index/indexmanager.cpp2
-rw-r--r--searchcore/src/vespa/searchcore/proton/index/indexmanager.h2
-rw-r--r--searchcore/src/vespa/searchcore/proton/server/documentdb.cpp2
-rw-r--r--searchcore/src/vespa/searchcore/proton/server/documentdb.h2
-rw-r--r--searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.cpp2
-rw-r--r--searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.h4
-rw-r--r--searchcore/src/vespa/searchcore/proton/server/maintenancecontroller.cpp2
-rw-r--r--searchcore/src/vespa/searchcore/proton/server/maintenancecontroller.h6
-rw-r--r--searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.h12
-rw-r--r--searchcorespi/src/vespa/searchcorespi/index/i_thread_service.h2
-rw-r--r--searchcorespi/src/vespa/searchcorespi/index/indexmaintainercontext.cpp2
-rw-r--r--searchcorespi/src/vespa/searchcorespi/index/indexmaintainercontext.h12
-rw-r--r--searchcorespi/src/vespa/searchcorespi/index/warmupindexcollection.cpp2
-rw-r--r--searchcorespi/src/vespa/searchcorespi/index/warmupindexcollection.h22
-rw-r--r--searchlib/src/vespa/searchlib/docstore/compacter.cpp6
-rw-r--r--searchlib/src/vespa/searchlib/docstore/compacter.h11
-rw-r--r--searchlib/src/vespa/searchlib/docstore/logdatastore.cpp1
-rw-r--r--searchlib/src/vespa/searchlib/docstore/storebybucket.cpp46
-rw-r--r--searchlib/src/vespa/searchlib/docstore/storebybucket.h17
-rw-r--r--searchlib/src/vespa/searchlib/transactionlog/domain.h2
-rw-r--r--security-utils/src/main/java/com/yahoo/security/KeyFormat.java11
-rw-r--r--security-utils/src/main/java/com/yahoo/security/KeyUtils.java32
-rw-r--r--security-utils/src/test/java/com/yahoo/security/KeyUtilsTest.java40
-rw-r--r--vespalib/src/vespa/vespalib/util/threadexecutor.h18
-rw-r--r--vespalib/src/vespa/vespalib/util/threadstackexecutorbase.cpp4
-rw-r--r--vespalib/src/vespa/vespalib/util/threadstackexecutorbase.h4
51 files changed, 379 insertions, 207 deletions
diff --git a/athenz-identity-provider-service/pom.xml b/athenz-identity-provider-service/pom.xml
index f4358394808..202d92d25a8 100644
--- a/athenz-identity-provider-service/pom.xml
+++ b/athenz-identity-provider-service/pom.xml
@@ -16,12 +16,45 @@
<dependencies>
<!-- PROVIDED -->
<dependency>
+ <groupId>com.google.inject</groupId>
+ <artifactId>guice</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>com.yahoo.vespa</groupId>
+ <artifactId>component</artifactId>
+ <version>${project.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
<groupId>com.yahoo.vespa</groupId>
<artifactId>container-dev</artifactId>
<version>${project.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
+ <groupId>com.yahoo.vespa</groupId>
+ <artifactId>jdisc_core</artifactId>
+ <version>${project.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>com.yahoo.vespa</groupId>
+ <artifactId>vespalog</artifactId>
+ <version>${project.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-core</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-databind</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
<groupId>org.eclipse.jetty</groupId>
<artifactId>jetty-server</artifactId>
<scope>provided</scope>
@@ -55,6 +88,12 @@
<version>${project.version}</version>
<scope>provided</scope>
</dependency>
+ <dependency>
+ <groupId>com.yahoo.vespa</groupId>
+ <artifactId>security-utils</artifactId>
+ <version>${project.version}</version>
+ <scope>provided</scope>
+ </dependency>
<!-- COMPILE -->
<dependency>
@@ -68,20 +107,19 @@
<!-- TEST -->
<dependency>
- <groupId>junit</groupId>
- <artifactId>junit</artifactId>
+ <groupId>com.yahoo.vespa</groupId>
+ <artifactId>zkfacade</artifactId>
+ <version>${project.version}</version>
<scope>test</scope>
</dependency>
<dependency>
- <groupId>com.yahoo.vespa</groupId>
- <artifactId>container-test</artifactId>
- <version>${project.version}</version>
+ <groupId>org.hamcrest</groupId>
+ <artifactId>hamcrest-core</artifactId>
<scope>test</scope>
</dependency>
<dependency>
- <groupId>com.yahoo.vespa</groupId>
- <artifactId>testutil</artifactId>
- <version>${project.version}</version>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
<scope>test</scope>
</dependency>
<dependency>
diff --git a/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/identitydocument/IdentityDocumentGeneratorTest.java b/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/identitydocument/IdentityDocumentGeneratorTest.java
index 92ba8f0edfb..66ab5255b6f 100644
--- a/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/identitydocument/IdentityDocumentGeneratorTest.java
+++ b/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/identitydocument/IdentityDocumentGeneratorTest.java
@@ -24,10 +24,8 @@ import com.yahoo.vespa.hosted.provision.node.Allocation;
import com.yahoo.vespa.hosted.provision.node.Generation;
import com.yahoo.vespa.hosted.provision.node.IP;
import com.yahoo.vespa.hosted.provision.testutils.MockNodeFlavors;
-import org.hamcrest.Matchers;
import org.junit.Test;
-import java.util.HashSet;
import java.util.Optional;
import java.util.Set;
@@ -35,6 +33,7 @@ import static com.yahoo.vespa.hosted.athenz.instanceproviderservice.TestUtils.ge
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertThat;
import static org.junit.Assert.assertTrue;
+import static org.hamcrest.core.IsCollectionContaining.hasItem;
import static org.mockito.Matchers.eq;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
@@ -92,7 +91,7 @@ public class IdentityDocumentGeneratorTest {
assertEquals(expectedProviderUniqueId, signedIdentityDocument.providerUniqueId());
// Validate that container ips are present
- assertThat(signedIdentityDocument.ipAddresses(), Matchers.containsInAnyOrder("::1"));
+ assertThat(signedIdentityDocument.ipAddresses(), hasItem("::1"));
IdentityDocumentSigner signer = new IdentityDocumentSigner();
diff --git a/container-search/src/main/java/com/yahoo/prelude/Index.java b/container-search/src/main/java/com/yahoo/prelude/Index.java
index 8433939090b..65d5879b004 100644
--- a/container-search/src/main/java/com/yahoo/prelude/Index.java
+++ b/container-search/src/main/java/com/yahoo/prelude/Index.java
@@ -66,10 +66,16 @@ public class Index {
private boolean numerical = false;
private long predicateUpperBound = Long.MAX_VALUE;
private long predicateLowerBound = Long.MIN_VALUE;
+
/** True if this is an <i>exact</i> index - which should match tokens containing any characters */
private boolean exact = false;
+
private boolean isNGram = false;
- private int gramSize=2;
+ private int gramSize = 2;
+
+ /** Whether implicit phrases should lead to a phrase item or an and item */
+ private boolean phraseSegmenting = true;
+
/** The string terminating an exact token in this index, or null to use the default (space) */
private String exactTerminator = null;
@@ -178,6 +184,10 @@ public class Index {
setNumerical(true);
} else if (commandString.startsWith("predicate-bounds ")) {
setPredicateBounds(commandString.substring(17));
+ } else if (commandString.equals("phrase-segmenting")) {
+ setPhraseSegmenting(true);
+ } else if (commandString.startsWith("phrase-segmenting ")) {
+ setPhraseSegmenting(Boolean.parseBoolean(commandString.substring("phrase-segmenting ".length())));
} else {
commands.add(commandString);
}
@@ -307,6 +317,10 @@ public class Index {
public long getPredicateLowerBound() { return predicateLowerBound; }
+ public boolean getPhraseSegmenting() { return phraseSegmenting; }
+
+ public boolean setPhraseSegmenting(boolean phraseSegmenting) { return this.phraseSegmenting = phraseSegmenting; }
+
/** Returns all the literal command strings given as arguments to addCommand in this instance */
public List<String> allCommands() { return allCommands; }
diff --git a/container-search/src/main/java/com/yahoo/prelude/IndexFacts.java b/container-search/src/main/java/com/yahoo/prelude/IndexFacts.java
index 1fd8cce9889..76eef33d6c0 100644
--- a/container-search/src/main/java/com/yahoo/prelude/IndexFacts.java
+++ b/container-search/src/main/java/com/yahoo/prelude/IndexFacts.java
@@ -273,17 +273,13 @@ public class IndexFacts {
return false;
}
- /**
- * @return whether it is permissible to update this object
- */
+ /** Returns whether it is permissible to update this object */
public boolean isFrozen() {
return frozen;
}
private void ensureNotFrozen() {
- if (frozen) {
- throw new IllegalStateException("Tried to modify frozen IndexFacts instance.");
- }
+ if (frozen) throw new IllegalStateException("Tried to modify frozen IndexFacts instance.");
}
public String getDefaultPosition(String sdName) {
@@ -307,7 +303,8 @@ public class IndexFacts {
return new Session(sources, restrict);
}
- public Session newSession(Collection<String> sources, Collection<String> restrict,
+ public Session newSession(Collection<String> sources,
+ Collection<String> restrict,
Set<String> candidateDocumentTypes) {
return new Session(sources, restrict, candidateDocumentTypes);
}
diff --git a/container-search/src/main/java/com/yahoo/prelude/IndexModel.java b/container-search/src/main/java/com/yahoo/prelude/IndexModel.java
index c83294efed7..062a514056b 100644
--- a/container-search/src/main/java/com/yahoo/prelude/IndexModel.java
+++ b/container-search/src/main/java/com/yahoo/prelude/IndexModel.java
@@ -24,7 +24,6 @@ public final class IndexModel {
private static final Logger log = Logger.getLogger(IndexModel.class.getName());
- // Copied from MasterClustersInfoUpdater. It's a temporary workaround for IndexFacts
private Map<String, List<String>> masterClusters;
private Map<String, SearchDefinition> searchDefinitions;
private SearchDefinition unionSearchDefinition;
@@ -34,9 +33,6 @@ public final class IndexModel {
this(Collections.emptyMap(), Collections.singleton(searchDefinition));
}
- /**
- * Create an index model.
- */
public IndexModel(Map<String, List<String>> masterClusters, Collection<SearchDefinition> searchDefinitions) {
this.masterClusters = masterClusters;
this.searchDefinitions = searchDefinitions.stream().collect(Collectors.toMap(sd -> sd.getName(), sd -> sd));
diff --git a/container-search/src/main/java/com/yahoo/prelude/query/IndexedSegmentItem.java b/container-search/src/main/java/com/yahoo/prelude/query/IndexedSegmentItem.java
index bf032ec03a8..b459f0aa902 100644
--- a/container-search/src/main/java/com/yahoo/prelude/query/IndexedSegmentItem.java
+++ b/container-search/src/main/java/com/yahoo/prelude/query/IndexedSegmentItem.java
@@ -38,12 +38,14 @@ public abstract class IndexedSegmentItem extends TaggableSegmentItem implements
}
// encode index bytes
+ @Override
protected void encodeThis(ByteBuffer buffer) {
super.encodeThis(buffer);
putString(index, buffer);
}
/** Sets the name of the index to search */
+ @Override
public void setIndexName(String index) {
if (index == null) {
index = "";
@@ -59,17 +61,17 @@ public abstract class IndexedSegmentItem extends TaggableSegmentItem implements
}
}
+ @Override
public boolean equals(Object object) {
- if (!super.equals(object)) {
- return false;
- }
+ if ( ! super.equals(object)) return false;
+
IndexedItem other = (IndexedItem) object; // Ensured by superclass
- if (!this.index.equals(other.getIndexName())) {
- return false;
- }
+ if ( ! this.index.equals(other.getIndexName())) return false;
+
return true;
}
+ @Override
public int hashCode() {
return super.hashCode() + 31 * index.hashCode();
}
diff --git a/container-search/src/main/java/com/yahoo/prelude/query/PhraseItem.java b/container-search/src/main/java/com/yahoo/prelude/query/PhraseItem.java
index 4de0af1f408..2d648557d9c 100644
--- a/container-search/src/main/java/com/yahoo/prelude/query/PhraseItem.java
+++ b/container-search/src/main/java/com/yahoo/prelude/query/PhraseItem.java
@@ -76,15 +76,15 @@ public class PhraseItem extends CompositeIndexedItem {
public void addItem(Item item) {
if (item instanceof WordItem || item instanceof PhraseSegmentItem || item instanceof WordAlternativesItem) {
addIndexedItem((IndexedItem) item);
- } else if (item instanceof IntItem) {
+ }
+ else if (item instanceof IntItem) {
addIndexedItem(convertIntToWord(item));
- } else if (item instanceof PhraseItem) {
- PhraseItem phrase = (PhraseItem) item;
-
- for (Iterator<Item> i = phrase.getItemIterator(); i.hasNext();) {
+ }
+ else if (item instanceof PhraseItem || item instanceof AndSegmentItem) {
+ for (Iterator<Item> i = ((CompositeItem) item).getItemIterator(); i.hasNext();)
addIndexedItem((IndexedItem) i.next());
- }
- } else {
+ }
+ else {
throw new IllegalArgumentException("Can not add " + item + " to a phrase");
}
}
diff --git a/container-search/src/main/java/com/yahoo/prelude/query/PhraseSegmentItem.java b/container-search/src/main/java/com/yahoo/prelude/query/PhraseSegmentItem.java
index 53a57a968f5..542f1393852 100644
--- a/container-search/src/main/java/com/yahoo/prelude/query/PhraseSegmentItem.java
+++ b/container-search/src/main/java/com/yahoo/prelude/query/PhraseSegmentItem.java
@@ -52,7 +52,7 @@ public class PhraseSegmentItem extends IndexedSegmentItem {
}
public PhraseSegmentItem(String rawWord, String current, boolean isFromQuery,
- boolean stemmed, Substring substring) {
+ boolean stemmed, Substring substring) {
super(rawWord, current, isFromQuery, stemmed, substring);
}
@@ -146,16 +146,16 @@ public class PhraseSegmentItem extends IndexedSegmentItem {
}
- /**
- * Returns false, no parenthezes for phrases
- */
+ /** Returns false, no parenthezes for phrases */
protected boolean shouldParenthize() {
return false;
}
/** Segment phrase items uses a empty heading instead of "SPHRASE " */
+ @Override
protected void appendHeadingString(StringBuilder buffer) {}
+ @Override
protected void appendBodyString(StringBuilder buffer) {
appendIndexString(buffer);
appendContentsString(buffer);
@@ -175,14 +175,13 @@ public class PhraseSegmentItem extends IndexedSegmentItem {
}
// TODO: Must check all pertinent items
+ @Override
public boolean equals(Object object) {
- if (!super.equals(object)) {
- return false;
- }
- // PhraseSegmentItem other = (PhraseSegmentItem) object; // Ensured by superclass
+ if ( ! super.equals(object)) return false;
return true;
}
+ @Override
public String getIndexedString() {
StringBuilder buf = new StringBuilder();
diff --git a/container-search/src/main/java/com/yahoo/prelude/query/SegmentItem.java b/container-search/src/main/java/com/yahoo/prelude/query/SegmentItem.java
index 1227a7f80cf..1c3eb261f90 100644
--- a/container-search/src/main/java/com/yahoo/prelude/query/SegmentItem.java
+++ b/container-search/src/main/java/com/yahoo/prelude/query/SegmentItem.java
@@ -126,6 +126,7 @@ public abstract class SegmentItem extends CompositeItem implements BlockItem {
// TODO: Add a getItemIterator which is safe for immutability
/** Return a deep copy of this object */
+ @Override
public SegmentItem clone() {
SegmentItem copy;
synchronized(this) {
diff --git a/container-search/src/main/java/com/yahoo/prelude/query/SimpleIndexedItem.java b/container-search/src/main/java/com/yahoo/prelude/query/SimpleIndexedItem.java
index f230c76c954..a8dc9db1928 100644
--- a/container-search/src/main/java/com/yahoo/prelude/query/SimpleIndexedItem.java
+++ b/container-search/src/main/java/com/yahoo/prelude/query/SimpleIndexedItem.java
@@ -38,6 +38,7 @@ public abstract class SimpleIndexedItem extends SimpleTaggableItem implements In
}
/** Sets the name of the index to search */
+ @Override
public void setIndexName(String index) {
if (index == null) {
index = "";
diff --git a/container-search/src/main/java/com/yahoo/prelude/query/SimpleTaggableItem.java b/container-search/src/main/java/com/yahoo/prelude/query/SimpleTaggableItem.java
index 6cbd2286a8f..6f82f340f4b 100644
--- a/container-search/src/main/java/com/yahoo/prelude/query/SimpleTaggableItem.java
+++ b/container-search/src/main/java/com/yahoo/prelude/query/SimpleTaggableItem.java
@@ -25,7 +25,7 @@ public abstract class SimpleTaggableItem extends Item implements TaggableItem {
uniqueID = id;
}
- /** See {@link TaggableItem#setConnectivity} */
+ @Override
public void setConnectivity(Item item, double connectivity) {
setHasUniqueID(true);
item.setHasUniqueID(true);
diff --git a/container-search/src/main/java/com/yahoo/prelude/query/TaggableItem.java b/container-search/src/main/java/com/yahoo/prelude/query/TaggableItem.java
index 03e85fa3260..8e2a472fd7c 100644
--- a/container-search/src/main/java/com/yahoo/prelude/query/TaggableItem.java
+++ b/container-search/src/main/java/com/yahoo/prelude/query/TaggableItem.java
@@ -34,7 +34,6 @@ public interface TaggableItem {
Item getConnectedItem();
double getConnectivity();
-
/**
* Used for setting explicit term significance (in the tf/idf sense) to a single term or phrase,
* relative to the rest of the query.
diff --git a/container-search/src/main/java/com/yahoo/prelude/query/TermItem.java b/container-search/src/main/java/com/yahoo/prelude/query/TermItem.java
index 8fdf147ce0b..2c33e7a2630 100644
--- a/container-search/src/main/java/com/yahoo/prelude/query/TermItem.java
+++ b/container-search/src/main/java/com/yahoo/prelude/query/TermItem.java
@@ -44,7 +44,7 @@ public abstract class TermItem extends SimpleIndexedItem implements BlockItem {
this.origin = origin;
}
- final public int encode(ByteBuffer buffer) {
+ public final int encode(ByteBuffer buffer) {
encodeThis(buffer);
return 1;
}
diff --git a/container-search/src/main/java/com/yahoo/prelude/query/parser/AbstractParser.java b/container-search/src/main/java/com/yahoo/prelude/query/parser/AbstractParser.java
index 8297a566a72..cd8579be7f0 100644
--- a/container-search/src/main/java/com/yahoo/prelude/query/parser/AbstractParser.java
+++ b/container-search/src/main/java/com/yahoo/prelude/query/parser/AbstractParser.java
@@ -228,13 +228,13 @@ public abstract class AbstractParser implements CustomParser {
protected abstract Item parseItems();
/**
- * Assigns the default index to query terms having no default index The
- * parser _should_ have done this, for some reason it doesn't
+ * Assigns the default index to query terms having no default index. The
+ * parser _should_ have done this, for some reason it doesn't.
*
- * @param defaultIndex The default index to assign.
- * @param item The item to check.
+ * @param defaultIndex the default index to assign
+ * @param item the item to check
*/
- private static void assignDefaultIndex(final String defaultIndex, Item item) {
+ private static void assignDefaultIndex(String defaultIndex, Item item) {
if (defaultIndex == null || item == null) return;
if (item instanceof IndexedItem) {
@@ -253,9 +253,6 @@ public abstract class AbstractParser implements CustomParser {
/**
* Unicode normalizes some piece of natural language text. The chosen form
* is compatibility decomposition, canonical composition (NFKC).
- *
- * @param input The string to normalize.
- * @return The normalized string.
*/
protected String normalize(String input) {
if (input == null || input.length() == 0) return input;
@@ -272,8 +269,8 @@ public abstract class AbstractParser implements CustomParser {
/**
* Tokenizes the given string and initializes tokens with the found tokens.
*
- * @param query the string to tokenize.
- * @param defaultIndexName the name of the index to use as default.
+ * @param query the string to tokenize
+ * @param defaultIndexName the name of the index to use as default
* @param indexFacts resolved information about the index we are searching
* @param language the language set for this query, or null if none
*/
@@ -324,6 +321,13 @@ public abstract class AbstractParser implements CustomParser {
}
}
+ /**
+ * Segments a token
+ *
+ * @param indexName the index name which preceeded this token, or null if none
+ * @param token the token to segment
+ * @return the resulting item
+ */
// TODO: The segmenting stuff is a mess now, this will fix it:
// - Make Segmenter a class which is instantiated per parsing
// - Make the instance know the language, etc and do all dispatching internally
@@ -331,13 +335,13 @@ public abstract class AbstractParser implements CustomParser {
// TODO: Use segmenting for forced phrase searches?
//
// Language detection currently depends on tokenization (see generateLanguageDetectionTextFrom), but
- // - the API's was originally not constructed for that, so a careful nd somewhat unsatisfactory dance
- // most be carried out to make it work
+ // - the API's was originally not constructed for that, so a careful and somewhat unsatisfactory dance
+ // must be carried out to make it work
// - it should really depend on parsing
// This can be solved by making the segment method language independent by
// always producing a query item containing the token text and resolve it to a WordItem or
// SegmentItem after parsing and language detection.
- protected Item segment(Token token) {
+ protected Item segment(String indexName, Token token) {
String normalizedToken = normalize(token.toString());
if (token.isSpecial()) {
@@ -361,13 +365,23 @@ public abstract class AbstractParser implements CustomParser {
return new WordItem(segments.get(0), "", true, token.substring);
}
- CompositeItem composite = new PhraseSegmentItem(token.toString(), normalizedToken, true, false, token.substring);
+ CompositeItem composite;
+ if (indexFacts.getIndex(indexName).getPhraseSegmenting()) {
+ composite = new PhraseSegmentItem(token.toString(), normalizedToken, true, false, token.substring);
+ }
+ else {
+ composite = new AndSegmentItem(token.toString(), true, false);
+ }
int n = 0;
+ WordItem previous = null;
for (String segment : segments) {
WordItem w = new WordItem(segment, "", true, token.substring);
w.setFromSegmented(true);
w.setSegmentIndex(n++);
w.setStemmed(false);
+ if (previous != null)
+ previous.setConnectivity(w, 1.0);
+ previous = w;
composite.addItem(w);
}
composite.lock();
diff --git a/container-search/src/main/java/com/yahoo/prelude/query/parser/PhraseParser.java b/container-search/src/main/java/com/yahoo/prelude/query/parser/PhraseParser.java
index 2d7e55a10fc..6d4401aca04 100644
--- a/container-search/src/main/java/com/yahoo/prelude/query/parser/PhraseParser.java
+++ b/container-search/src/main/java/com/yahoo/prelude/query/parser/PhraseParser.java
@@ -38,7 +38,7 @@ public class PhraseParser extends AbstractParser {
}
// Note, this depends on segment never creating AndItems when quoted
// (the second argument) is true.
- Item newWord = segment(token);
+ Item newWord = segment(null, token);
if (firstWord == null) { // First pass
firstWord = newWord;
diff --git a/container-search/src/main/java/com/yahoo/prelude/query/parser/StructuredParser.java b/container-search/src/main/java/com/yahoo/prelude/query/parser/StructuredParser.java
index 8c77b3d2130..ee4c0d4d9f0 100644
--- a/container-search/src/main/java/com/yahoo/prelude/query/parser/StructuredParser.java
+++ b/container-search/src/main/java/com/yahoo/prelude/query/parser/StructuredParser.java
@@ -63,12 +63,12 @@ abstract class StructuredParser extends AbstractParser {
item = number();
if (item == null) {
- item = phrase();
+ item = phrase(indexName);
}
if (item == null && indexName != null) {
if (wordsAhead()) {
- item = phrase();
+ item = phrase(indexName);
}
}
@@ -407,12 +407,12 @@ abstract class StructuredParser extends AbstractParser {
}
/** Words for phrases also permits numerals as words */
- private Item phraseWord(boolean insidePhrase) {
+ private Item phraseWord(String indexName, boolean insidePhrase) {
int position = tokens.getPosition();
Item item = null;
try {
- item = word();
+ item = word(indexName);
if (item == null && tokens.currentIs(NUMBER)) {
Token t = tokens.next();
@@ -437,7 +437,7 @@ abstract class StructuredParser extends AbstractParser {
* a WordItem or PhraseSegmentItem if this is a CJK query,
* null if the current item is not a word
*/
- private Item word() {
+ private Item word(String indexName) {
int position = tokens.getPosition();
Item item = null;
@@ -452,7 +452,7 @@ abstract class StructuredParser extends AbstractParser {
if (submodes.url) {
item = new WordItem(word, true);
} else {
- item = segment(word);
+ item = segment(indexName, word);
}
if (submodes.url || submodes.site) {
@@ -499,15 +499,16 @@ abstract class StructuredParser extends AbstractParser {
* An phrase or word, either marked by quotes or by non-spaces between
* words or by a combination.
*
+ * @param indexName the index name which preceeded this phrase, or null if none
* @return a word if there's only one word, a phrase if there is
* several quoted or non-space-separated words, or null otherwise
*/
- private Item phrase() {
+ private Item phrase(String indexName) {
int position = tokens.getPosition();
Item item = null;
try {
- item = phraseBody();
+ item = phraseBody(indexName);
return item;
} finally {
if (item == null) {
@@ -516,8 +517,8 @@ abstract class StructuredParser extends AbstractParser {
}
}
- /** Returns a word, a phrase or another composite */
- private Item phraseBody() {
+ /** Returns a word, a phrase, or another composite */
+ private Item phraseBody(String indexName) {
boolean quoted = false;
PhraseItem phrase = null;
Item firstWord = null;
@@ -538,7 +539,7 @@ abstract class StructuredParser extends AbstractParser {
quoted = !quoted;
}
- Item word = phraseWord((firstWord != null) || (phrase != null));
+ Item word = phraseWord(indexName, (firstWord != null) || (phrase != null));
if (word == null) {
if (tokens.skipMultiple(QUOTE)) {
diff --git a/container-search/src/main/java/com/yahoo/search/query/parser/ParserEnvironment.java b/container-search/src/main/java/com/yahoo/search/query/parser/ParserEnvironment.java
index ca437fb9def..9e53f9d8ea9 100644
--- a/container-search/src/main/java/com/yahoo/search/query/parser/ParserEnvironment.java
+++ b/container-search/src/main/java/com/yahoo/search/query/parser/ParserEnvironment.java
@@ -50,19 +50,17 @@ public final class ParserEnvironment {
public static ParserEnvironment fromExecutionContext(Execution.Context context) {
ParserEnvironment env = new ParserEnvironment();
- if (context == null) {
- return env;
- }
- if (context.getIndexFacts() != null) {
+ if (context == null) return env;
+
+ if (context.getIndexFacts() != null)
env.setIndexFacts(context.getIndexFacts());
- }
- if (context.getLinguistics() != null) {
+
+ if (context.getLinguistics() != null)
env.setLinguistics(context.getLinguistics());
- }
- SpecialTokenRegistry registry = context.getTokenRegistry();
- if (registry != null) {
- env.setSpecialTokens(registry.getSpecialTokens("default"));
- }
+
+ if (context.getTokenRegistry() != null)
+ env.setSpecialTokens(context.getTokenRegistry().getSpecialTokens("default"));
+
return env;
}
diff --git a/container-search/src/main/java/com/yahoo/search/searchchain/Execution.java b/container-search/src/main/java/com/yahoo/search/searchchain/Execution.java
index e2c79fa5a7e..5cc34ff5b28 100644
--- a/container-search/src/main/java/com/yahoo/search/searchchain/Execution.java
+++ b/container-search/src/main/java/com/yahoo/search/searchchain/Execution.java
@@ -59,15 +59,10 @@ public class Execution extends com.yahoo.processing.execution.Execution {
*/
public static final class Context {
- /**
- * Whether the search should perform detailed diagnostics.
- */
+ /** Whether the search should perform detailed diagnostics. */
private boolean detailedDiagnostics = false;
- /**
- * Whether the container was considered to be in a breakdown state when
- * this query started.
- */
+ /** Whether the container was considered to be in a breakdown state when this query started. */
private boolean breakdown = false;
/**
@@ -79,19 +74,13 @@ public class Execution extends com.yahoo.processing.execution.Execution {
private IndexFacts indexFacts = null;
- /**
- * The current set of special tokens.
- */
+ /** The current set of special tokens */
private SpecialTokenRegistry tokenRegistry = null;
- /**
- * The current template registry.
- */
+ /** The current template registry */
private RendererRegistry rendererRegistry = null;
- /**
- * The current linguistics.
- */
+ /** The current linguistics */
private Linguistics linguistics = null;
/** Always set if this context belongs to an execution, never set if it does not. */
diff --git a/container-search/src/test/java/com/yahoo/prelude/query/parser/test/parseindexinfo.cfg b/container-search/src/test/java/com/yahoo/prelude/query/parser/test/parseindexinfo.cfg
index 0d264e04799..4a4a5c3a04d 100644
--- a/container-search/src/test/java/com/yahoo/prelude/query/parser/test/parseindexinfo.cfg
+++ b/container-search/src/test/java/com/yahoo/prelude/query/parser/test/parseindexinfo.cfg
@@ -1,6 +1,6 @@
indexinfo[3]
indexinfo[0].name one
-indexinfo[0].command[44]
+indexinfo[0].command[45]
indexinfo[0].command[0].indexname url.all
indexinfo[0].command[0].command fullurl
indexinfo[0].command[1].indexname host.all
@@ -89,6 +89,8 @@ indexinfo[0].command[42].indexname exactindex
indexinfo[0].command[42].command index
indexinfo[0].command[43].indexname exactindex
indexinfo[0].command[43].command exact
+indexinfo[0].command[44].indexname phraseSegment
+indexinfo[0].command[44].command "phrase-segmenting false"
indexinfo[1].name twoRanges
indexinfo[1].command[2]
@@ -109,3 +111,4 @@ indexinfo[2].command[3].indexname link
indexinfo[2].command[3].command index
indexinfo[2].command[4].indexname url
indexinfo[2].command[4].command index
+
diff --git a/container-search/src/test/java/com/yahoo/search/test/QueryTestCase.java b/container-search/src/test/java/com/yahoo/search/test/QueryTestCase.java
index 45f270b2e37..fd29f2c12fe 100644
--- a/container-search/src/test/java/com/yahoo/search/test/QueryTestCase.java
+++ b/container-search/src/test/java/com/yahoo/search/test/QueryTestCase.java
@@ -1,6 +1,7 @@
// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.search.test;
+import com.google.common.collect.ImmutableList;
import com.yahoo.component.chain.Chain;
import com.yahoo.language.Language;
import com.yahoo.language.Linguistics;
@@ -42,7 +43,9 @@ import org.junit.Test;
import java.io.UnsupportedEncodingException;
import java.net.URLEncoder;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collections;
+import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
@@ -877,6 +880,57 @@ public class QueryTestCase {
}
}
+ @Test
+ public void testImplicitPhraseIsDefault() {
+ Query query = new Query(httpEncode("?query=it's fine"));
+ assertEquals("AND 'it s' fine", query.getModel().getQueryTree().toString());
+ }
+
+ @Test
+ public void testImplicitPhrase() {
+ Query query = new Query(httpEncode("?query=myfield:it's myfield:fine"));
+
+ SearchDefinition test = new SearchDefinition("test");
+ Index myField = new Index("myfield");
+ myField.addCommand("phrase-segmenting true");
+ assertTrue(myField.getPhraseSegmenting());
+ test.addIndex(myField);
+ IndexModel indexModel = new IndexModel(test);
+ query.getModel().setExecution(new Execution(Execution.Context.createContextStub(new IndexFacts(indexModel))));
+
+ assertEquals("AND myfield:'it s' myfield:fine", query.getModel().getQueryTree().toString());
+ }
+
+ @Test
+ public void testImplicitAnd() {
+ Query query = new Query(httpEncode("?query=myfield:it's myfield:fine"));
+
+ SearchDefinition test = new SearchDefinition("test");
+ Index myField = new Index("myfield");
+ myField.addCommand("phrase-segmenting false");
+ assertFalse(myField.getPhraseSegmenting());
+ test.addIndex(myField);
+ IndexModel indexModel = new IndexModel(test);
+ query.getModel().setExecution(new Execution(Execution.Context.createContextStub(new IndexFacts(indexModel))));
+
+ assertEquals("AND (SAND myfield:it myfield:s) myfield:fine", query.getModel().getQueryTree().toString());
+ }
+
+ @Test
+ public void testImplicitAndInPhrase() {
+ Query query = new Query(httpEncode("?query=myfield:\"it's fine\""));
+
+ SearchDefinition test = new SearchDefinition("test");
+ Index myField = new Index("myfield");
+ myField.addCommand("phrase-segmenting false");
+ assertFalse(myField.getPhraseSegmenting());
+ test.addIndex(myField);
+ IndexModel indexModel = new IndexModel(test);
+ query.getModel().setExecution(new Execution(Execution.Context.createContextStub(new IndexFacts(indexModel))));
+
+ assertEquals("myfield:\"it s fine\"", query.getModel().getQueryTree().toString());
+ }
+
private void assertDetectionText(String expectedDetectionText, String queryString, String ... indexSpecs) {
Query q = new Query(httpEncode("/?query=" + queryString));
SearchDefinition sd = new SearchDefinition("testSearchDefinition");
diff --git a/dist/vespa.spec b/dist/vespa.spec
index 50216f6fdde..285c54287dc 100644
--- a/dist/vespa.spec
+++ b/dist/vespa.spec
@@ -37,6 +37,7 @@ BuildRequires: llvm5.0-devel
BuildRequires: vespa-boost-devel >= 1.59.0-6
BuildRequires: vespa-gtest >= 1.8.1-1
BuildRequires: vespa-protobuf-devel >= 3.7.0-4
+BuildRequires: vespa-openssl-devel >= 1.1.1c-1
%endif
%if 0%{?fedora}
BuildRequires: cmake >= 3.9.1
@@ -108,6 +109,7 @@ Requires: gdb
Requires: net-tools
%if 0%{?centos}
Requires: llvm5.0
+Requires: vespa-openssl >= 1.1.1c-1
Requires: vespa-protobuf >= 3.7.0-4
%define _vespa_llvm_version 5.0
%define _extra_link_directory /usr/lib64/llvm5.0/lib;%{_vespa_deps_prefix}/lib64
@@ -131,7 +133,6 @@ Requires: llvm-libs >= 8.0.0
%define _extra_include_directory %{_vespa_deps_prefix}/include
%endif
Requires: java-11-openjdk
-Requires: openssl
Requires(pre): shadow-utils
# Ugly workaround because vespamalloc/src/vespamalloc/malloc/mmap.cpp uses the private
diff --git a/searchcore/src/tests/proton/documentdb/document_subdbs/document_subdbs_test.cpp b/searchcore/src/tests/proton/documentdb/document_subdbs/document_subdbs_test.cpp
index 68c388111d3..76b6bf55534 100644
--- a/searchcore/src/tests/proton/documentdb/document_subdbs/document_subdbs_test.cpp
+++ b/searchcore/src/tests/proton/documentdb/document_subdbs/document_subdbs_test.cpp
@@ -235,7 +235,7 @@ MySearchableContext::MySearchableContext(IThreadingService &writeService,
IBucketDBHandlerInitializer & bucketDBHandlerInitializer)
: _fastUpdCtx(writeService, bucketDB, bucketDBHandlerInitializer),
_queryLimiter(), _clock(),
- _ctx(_fastUpdCtx._ctx, _queryLimiter, _clock, writeService.shared())
+ _ctx(_fastUpdCtx._ctx, _queryLimiter, _clock, dynamic_cast<vespalib::SyncableThreadExecutor &>(writeService.shared()))
{}
MySearchableContext::~MySearchableContext() = default;
diff --git a/searchcore/src/vespa/searchcore/proton/docsummary/summarymanagerinitializer.h b/searchcore/src/vespa/searchcore/proton/docsummary/summarymanagerinitializer.h
index e7cb37aea28..6c2d3e3d4cc 100644
--- a/searchcore/src/vespa/searchcore/proton/docsummary/summarymanagerinitializer.h
+++ b/searchcore/src/vespa/searchcore/proton/docsummary/summarymanagerinitializer.h
@@ -43,7 +43,7 @@ public:
search::transactionlog::SyncProxy &tlSyncer,
IBucketizerSP bucketizer,
std::shared_ptr<SummaryManager::SP> result);
- ~SummaryManagerInitializer();
+ ~SummaryManagerInitializer() override;
void run() override;
};
diff --git a/searchcore/src/vespa/searchcore/proton/index/index_manager_initializer.cpp b/searchcore/src/vespa/searchcore/proton/index/index_manager_initializer.cpp
index 9c680ec7768..e87f74e2258 100644
--- a/searchcore/src/vespa/searchcore/proton/index/index_manager_initializer.cpp
+++ b/searchcore/src/vespa/searchcore/proton/index/index_manager_initializer.cpp
@@ -15,7 +15,7 @@ IndexManagerInitializer(const vespalib::string &baseDir,
search::SerialNum serialNum,
searchcorespi::IIndexManager::Reconfigurer & reconfigurer,
searchcorespi::index::IThreadingService & threadingService,
- vespalib::ThreadExecutor & warmupExecutor,
+ vespalib::SyncableThreadExecutor & warmupExecutor,
const search::TuneFileIndexManager & tuneFileIndexManager,
const search::TuneFileAttributes &tuneFileAttributes,
const search::common::FileHeaderContext & fileHeaderContext,
diff --git a/searchcore/src/vespa/searchcore/proton/index/index_manager_initializer.h b/searchcore/src/vespa/searchcore/proton/index/index_manager_initializer.h
index 1a14c8d2ea3..8e446e222b3 100644
--- a/searchcore/src/vespa/searchcore/proton/index/index_manager_initializer.h
+++ b/searchcore/src/vespa/searchcore/proton/index/index_manager_initializer.h
@@ -20,7 +20,7 @@ class IndexManagerInitializer : public initializer::InitializerTask
search::SerialNum _serialNum;
searchcorespi::IIndexManager::Reconfigurer &_reconfigurer;
searchcorespi::index::IThreadingService &_threadingService;
- vespalib::ThreadExecutor &_warmupExecutor;
+ vespalib::SyncableThreadExecutor &_warmupExecutor;
const search::TuneFileIndexManager _tuneFileIndexManager;
const search::TuneFileAttributes _tuneFileAttributes;
const search::common::FileHeaderContext &_fileHeaderContext;
@@ -33,7 +33,7 @@ public:
search::SerialNum serialNum,
searchcorespi::IIndexManager::Reconfigurer & reconfigurer,
searchcorespi::index::IThreadingService & threadingService,
- vespalib::ThreadExecutor & warmupExecutor,
+ vespalib::SyncableThreadExecutor & warmupExecutor,
const search::TuneFileIndexManager & tuneFileIndexManager,
const search::TuneFileAttributes & tuneFileAttributes,
const search::common::FileHeaderContext & fileHeaderContext,
diff --git a/searchcore/src/vespa/searchcore/proton/index/indexmanager.cpp b/searchcore/src/vespa/searchcore/proton/index/indexmanager.cpp
index 8e838414015..6ffd3203019 100644
--- a/searchcore/src/vespa/searchcore/proton/index/indexmanager.cpp
+++ b/searchcore/src/vespa/searchcore/proton/index/indexmanager.cpp
@@ -78,7 +78,7 @@ IndexManager::IndexManager(const vespalib::string &baseDir,
SerialNum serialNum,
Reconfigurer &reconfigurer,
IThreadingService &threadingService,
- vespalib::ThreadExecutor & warmupExecutor,
+ vespalib::SyncableThreadExecutor & warmupExecutor,
const search::TuneFileIndexManager &tuneFileIndexManager,
const search::TuneFileAttributes &tuneFileAttributes,
const FileHeaderContext &fileHeaderContext) :
diff --git a/searchcore/src/vespa/searchcore/proton/index/indexmanager.h b/searchcore/src/vespa/searchcore/proton/index/indexmanager.h
index b14912239a3..7e305681f78 100644
--- a/searchcore/src/vespa/searchcore/proton/index/indexmanager.h
+++ b/searchcore/src/vespa/searchcore/proton/index/indexmanager.h
@@ -72,7 +72,7 @@ public:
SerialNum serialNum,
Reconfigurer &reconfigurer,
searchcorespi::index::IThreadingService &threadingService,
- vespalib::ThreadExecutor & warmupExecutor,
+ vespalib::SyncableThreadExecutor & warmupExecutor,
const search::TuneFileIndexManager &tuneFileIndexManager,
const search::TuneFileAttributes &tuneFileAttributes,
const search::common::FileHeaderContext &fileHeaderContext);
diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp b/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp
index 5aacfa2678c..45784fc2683 100644
--- a/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp
@@ -119,7 +119,7 @@ DocumentDB::DocumentDB(const vespalib::string &baseDir,
document::BucketSpace bucketSpace,
const ProtonConfig &protonCfg,
IDocumentDBOwner &owner,
- vespalib::ThreadExecutor &warmupExecutor,
+ vespalib::SyncableThreadExecutor &warmupExecutor,
vespalib::ThreadStackExecutorBase &sharedExecutor,
search::transactionlog::Writer &tlsDirectWriter,
MetricsWireService &metricsWireService,
diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdb.h b/searchcore/src/vespa/searchcore/proton/server/documentdb.h
index b4e58d6f178..d6448c0b515 100644
--- a/searchcore/src/vespa/searchcore/proton/server/documentdb.h
+++ b/searchcore/src/vespa/searchcore/proton/server/documentdb.h
@@ -247,7 +247,7 @@ public:
document::BucketSpace bucketSpace,
const ProtonConfig &protonCfg,
IDocumentDBOwner &owner,
- vespalib::ThreadExecutor &warmupExecutor,
+ vespalib::SyncableThreadExecutor &warmupExecutor,
vespalib::ThreadStackExecutorBase &sharedExecutor,
search::transactionlog::Writer &tlsDirectWriter,
MetricsWireService &metricsWireService,
diff --git a/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.cpp b/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.cpp
index 682719436f1..74010391fcb 100644
--- a/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.cpp
@@ -34,7 +34,7 @@ DocumentSubDBCollection::DocumentSubDBCollection(
const IGetSerialNum &getSerialNum,
const DocTypeName &docTypeName,
searchcorespi::index::IThreadingService &writeService,
- vespalib::ThreadExecutor &warmupExecutor,
+ vespalib::SyncableThreadExecutor &warmupExecutor,
const search::common::FileHeaderContext &fileHeaderContext,
MetricsWireService &metricsWireService,
DocumentDBTaggedMetrics &metrics,
diff --git a/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.h b/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.h
index 05427f5e545..d09afb92cc8 100644
--- a/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.h
+++ b/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.h
@@ -11,7 +11,7 @@
namespace vespalib {
class Clock;
- class ThreadExecutor;
+ class SyncableThreadExecutor;
class ThreadStackExecutorBase;
}
@@ -100,7 +100,7 @@ public:
const IGetSerialNum &getSerialNum,
const DocTypeName &docTypeName,
searchcorespi::index::IThreadingService &writeService,
- vespalib::ThreadExecutor &warmupExecutor,
+ vespalib::SyncableThreadExecutor &warmupExecutor,
const search::common::FileHeaderContext &fileHeaderContext,
MetricsWireService &metricsWireService,
DocumentDBTaggedMetrics &metrics,
diff --git a/searchcore/src/vespa/searchcore/proton/server/maintenancecontroller.cpp b/searchcore/src/vespa/searchcore/proton/server/maintenancecontroller.cpp
index e6afb8a19b2..a2672cc7972 100644
--- a/searchcore/src/vespa/searchcore/proton/server/maintenancecontroller.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/maintenancecontroller.cpp
@@ -32,7 +32,7 @@ public:
}
MaintenanceController::MaintenanceController(IThreadService &masterThread,
- vespalib::ThreadExecutor & defaultExecutor,
+ vespalib::SyncableThreadExecutor & defaultExecutor,
const DocTypeName &docTypeName)
: IBucketFreezeListener(),
_masterThread(masterThread),
diff --git a/searchcore/src/vespa/searchcore/proton/server/maintenancecontroller.h b/searchcore/src/vespa/searchcore/proton/server/maintenancecontroller.h
index 69416ff63d7..24c1c18959e 100644
--- a/searchcore/src/vespa/searchcore/proton/server/maintenancecontroller.h
+++ b/searchcore/src/vespa/searchcore/proton/server/maintenancecontroller.h
@@ -11,7 +11,7 @@
namespace vespalib {
class Timer;
- class ThreadExecutor;
+ class SyncableThreadExecutor;
class Executor;
}
namespace searchcorespi { namespace index { struct IThreadService; }}
@@ -34,7 +34,7 @@ public:
using JobList = std::vector<std::shared_ptr<MaintenanceJobRunner>>;
using UP = std::unique_ptr<MaintenanceController>;
- MaintenanceController(IThreadService &masterThread, vespalib::ThreadExecutor & defaultExecutor, const DocTypeName &docTypeName);
+ MaintenanceController(IThreadService &masterThread, vespalib::SyncableThreadExecutor & defaultExecutor, const DocTypeName &docTypeName);
virtual ~MaintenanceController();
void registerJobInMasterThread(IMaintenanceJob::UP job);
@@ -73,7 +73,7 @@ private:
using Guard = std::lock_guard<Mutex>;
IThreadService &_masterThread;
- vespalib::ThreadExecutor &_defaultExecutor;
+ vespalib::SyncableThreadExecutor &_defaultExecutor;
MaintenanceDocumentSubDB _readySubDB;
MaintenanceDocumentSubDB _remSubDB;
MaintenanceDocumentSubDB _notReadySubDB;
diff --git a/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.h b/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.h
index e39ada31673..f3cf95ebfa0 100644
--- a/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.h
+++ b/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.h
@@ -52,15 +52,15 @@ public:
};
struct Context {
- const FastAccessDocSubDB::Context _fastUpdCtx;
- matching::QueryLimiter &_queryLimiter;
- const vespalib::Clock &_clock;
- vespalib::ThreadExecutor &_warmupExecutor;
+ const FastAccessDocSubDB::Context _fastUpdCtx;
+ matching::QueryLimiter &_queryLimiter;
+ const vespalib::Clock &_clock;
+ vespalib::SyncableThreadExecutor &_warmupExecutor;
Context(const FastAccessDocSubDB::Context &fastUpdCtx,
matching::QueryLimiter &queryLimiter,
const vespalib::Clock &clock,
- vespalib::ThreadExecutor &warmupExecutor)
+ vespalib::SyncableThreadExecutor &warmupExecutor)
: _fastUpdCtx(fastUpdCtx),
_queryLimiter(queryLimiter),
_clock(clock),
@@ -80,7 +80,7 @@ private:
vespalib::eval::ConstantValueCache _constantValueCache;
matching::ConstantValueRepo _constantValueRepo;
SearchableDocSubDBConfigurer _configurer;
- vespalib::ThreadExecutor &_warmupExecutor;
+ vespalib::SyncableThreadExecutor &_warmupExecutor;
std::shared_ptr<GidToLidChangeHandler> _realGidToLidChangeHandler;
DocumentDBFlushConfig _flushConfig;
bool _nodeRetired;
diff --git a/searchcorespi/src/vespa/searchcorespi/index/i_thread_service.h b/searchcorespi/src/vespa/searchcorespi/index/i_thread_service.h
index 40d92010c9b..65f18ac2c79 100644
--- a/searchcorespi/src/vespa/searchcorespi/index/i_thread_service.h
+++ b/searchcorespi/src/vespa/searchcorespi/index/i_thread_service.h
@@ -9,7 +9,7 @@ namespace searchcorespi::index {
/**
* Interface for a single thread used for write tasks.
*/
-struct IThreadService : public vespalib::ThreadExecutor
+struct IThreadService : public vespalib::SyncableThreadExecutor
{
IThreadService(const IThreadService &) = delete;
IThreadService & operator = (const IThreadService &) = delete;
diff --git a/searchcorespi/src/vespa/searchcorespi/index/indexmaintainercontext.cpp b/searchcorespi/src/vespa/searchcorespi/index/indexmaintainercontext.cpp
index 213d2cb8705..78b9930c69a 100644
--- a/searchcorespi/src/vespa/searchcorespi/index/indexmaintainercontext.cpp
+++ b/searchcorespi/src/vespa/searchcorespi/index/indexmaintainercontext.cpp
@@ -11,7 +11,7 @@ namespace searchcorespi::index {
IndexMaintainerContext::IndexMaintainerContext(IThreadingService &threadingService,
IIndexManager::Reconfigurer &reconfigurer,
const FileHeaderContext &fileHeaderContext,
- vespalib::ThreadExecutor & warmupExecutor)
+ vespalib::SyncableThreadExecutor & warmupExecutor)
: _threadingService(threadingService),
_reconfigurer(reconfigurer),
_fileHeaderContext(fileHeaderContext),
diff --git a/searchcorespi/src/vespa/searchcorespi/index/indexmaintainercontext.h b/searchcorespi/src/vespa/searchcorespi/index/indexmaintainercontext.h
index 972a16effc3..ab4a447168e 100644
--- a/searchcorespi/src/vespa/searchcorespi/index/indexmaintainercontext.h
+++ b/searchcorespi/src/vespa/searchcorespi/index/indexmaintainercontext.h
@@ -15,15 +15,15 @@ namespace searchcorespi::index {
class IndexMaintainerContext {
private:
IThreadingService &_threadingService;
- searchcorespi::IIndexManager::Reconfigurer &_reconfigurer;
+ IIndexManager::Reconfigurer &_reconfigurer;
const search::common::FileHeaderContext &_fileHeaderContext;
- vespalib::ThreadExecutor & _warmupExecutor;
+ vespalib::SyncableThreadExecutor & _warmupExecutor;
public:
IndexMaintainerContext(IThreadingService &threadingService,
- searchcorespi::IIndexManager::Reconfigurer &reconfigurer,
+ IIndexManager::Reconfigurer &reconfigurer,
const search::common::FileHeaderContext &fileHeaderContext,
- vespalib::ThreadExecutor & warmupExecutor);
+ vespalib::SyncableThreadExecutor & warmupExecutor);
/**
* Returns the treading service that encapsulates the thread model used for writing.
@@ -35,7 +35,7 @@ public:
/**
* Returns the reconfigurer used to signal when the index maintainer has changed.
*/
- searchcorespi::IIndexManager::Reconfigurer &getReconfigurer() const {
+ IIndexManager::Reconfigurer &getReconfigurer() const {
return _reconfigurer;
}
@@ -49,7 +49,7 @@ public:
/**
* @return The executor that should be used for warmup.
*/
- vespalib::ThreadExecutor & getWarmupExecutor() const { return _warmupExecutor; }
+ vespalib::SyncableThreadExecutor & getWarmupExecutor() const { return _warmupExecutor; }
};
}
diff --git a/searchcorespi/src/vespa/searchcorespi/index/warmupindexcollection.cpp b/searchcorespi/src/vespa/searchcorespi/index/warmupindexcollection.cpp
index 911ff5f9eba..1786665c248 100644
--- a/searchcorespi/src/vespa/searchcorespi/index/warmupindexcollection.cpp
+++ b/searchcorespi/src/vespa/searchcorespi/index/warmupindexcollection.cpp
@@ -34,7 +34,7 @@ WarmupIndexCollection::WarmupIndexCollection(const WarmupConfig & warmupConfig,
ISearchableIndexCollection::SP prev,
ISearchableIndexCollection::SP next,
IndexSearchable & warmup,
- vespalib::ThreadExecutor & executor,
+ vespalib::SyncableThreadExecutor & executor,
IWarmupDone & warmupDone) :
_warmupConfig(warmupConfig),
_prev(prev),
diff --git a/searchcorespi/src/vespa/searchcorespi/index/warmupindexcollection.h b/searchcorespi/src/vespa/searchcorespi/index/warmupindexcollection.h
index 229d1a7c31d..992cedb1057 100644
--- a/searchcorespi/src/vespa/searchcorespi/index/warmupindexcollection.h
+++ b/searchcorespi/src/vespa/searchcorespi/index/warmupindexcollection.h
@@ -30,9 +30,9 @@ public:
ISearchableIndexCollection::SP prev,
ISearchableIndexCollection::SP next,
IndexSearchable & warmup,
- vespalib::ThreadExecutor & executor,
+ vespalib::SyncableThreadExecutor & executor,
IWarmupDone & warmupDone);
- ~WarmupIndexCollection();
+ ~WarmupIndexCollection() override;
// Implements IIndexCollection
const ISourceSelector &getSourceSelector() const override;
size_t getSourceCount() const override;
@@ -95,15 +95,15 @@ private:
void fireWarmup(Task::UP task);
bool handledBefore(uint32_t fieldId, const Node &term);
- const WarmupConfig _warmupConfig;
- ISearchableIndexCollection::SP _prev;
- ISearchableIndexCollection::SP _next;
- IndexSearchable & _warmup;
- vespalib::ThreadExecutor & _executor;
- IWarmupDone & _warmupDone;
- fastos::TimeStamp _warmupEndTime;
- std::mutex _lock;
- std::unique_ptr<FieldTermMap> _handledTerms;
+ const WarmupConfig _warmupConfig;
+ ISearchableIndexCollection::SP _prev;
+ ISearchableIndexCollection::SP _next;
+ IndexSearchable & _warmup;
+ vespalib::SyncableThreadExecutor & _executor;
+ IWarmupDone & _warmupDone;
+ fastos::TimeStamp _warmupEndTime;
+ std::mutex _lock;
+ std::unique_ptr<FieldTermMap> _handledTerms;
};
} // namespace searchcorespi
diff --git a/searchlib/src/vespa/searchlib/docstore/compacter.cpp b/searchlib/src/vespa/searchlib/docstore/compacter.cpp
index 4d154da9907..b53281b4dbb 100644
--- a/searchlib/src/vespa/searchlib/docstore/compacter.cpp
+++ b/searchlib/src/vespa/searchlib/docstore/compacter.cpp
@@ -7,8 +7,7 @@
#include <vespa/log/log.h>
LOG_SETUP(".searchlib.docstore.compacter");
-namespace search {
-namespace docstore {
+namespace search::docstore {
using vespalib::alloc::Alloc;
@@ -19,7 +18,7 @@ Compacter::write(LockGuard guard, uint32_t chunkId, uint32_t lid, const void *bu
_ds.write(std::move(guard), fileId, lid, buffer, sz);
}
-BucketCompacter::BucketCompacter(size_t maxSignificantBucketBits, const CompressionConfig & compression, LogDataStore & ds, ThreadExecutor & executor, const IBucketizer & bucketizer, FileId source, FileId destination) :
+BucketCompacter::BucketCompacter(size_t maxSignificantBucketBits, const CompressionConfig & compression, LogDataStore & ds, Executor & executor, const IBucketizer & bucketizer, FileId source, FileId destination) :
_unSignificantBucketBits((maxSignificantBucketBits > 8) ? (maxSignificantBucketBits - 8) : 0),
_sourceFileId(source),
_destinationFileId(destination),
@@ -97,4 +96,3 @@ BucketCompacter::write(BucketId bucketId, uint32_t chunkId, uint32_t lid, const
}
}
-}
diff --git a/searchlib/src/vespa/searchlib/docstore/compacter.h b/searchlib/src/vespa/searchlib/docstore/compacter.h
index 362896487aa..666943ed629 100644
--- a/searchlib/src/vespa/searchlib/docstore/compacter.h
+++ b/searchlib/src/vespa/searchlib/docstore/compacter.h
@@ -6,11 +6,9 @@
#include "storebybucket.h"
#include <vespa/vespalib/data/memorydatastore.h>
-namespace search {
+namespace search { class LogDataStore; }
-class LogDataStore;
-
-namespace docstore {
+namespace search::docstore {
/**
* A simple write through implementation of the IWriteData interface.
@@ -34,11 +32,11 @@ private:
class BucketCompacter : public IWriteData, public StoreByBucket::IWrite
{
using CompressionConfig = vespalib::compression::CompressionConfig;
- using ThreadExecutor = vespalib::ThreadExecutor;
+ using Executor = vespalib::Executor;
public:
using FileId = FileChunk::FileId;
BucketCompacter(size_t maxSignificantBucketBits, const CompressionConfig & compression, LogDataStore & ds,
- ThreadExecutor & exeutor, const IBucketizer & bucketizer, FileId source, FileId destination);
+ Executor & exeutor, const IBucketizer & bucketizer, FileId source, FileId destination);
void write(LockGuard guard, uint32_t chunkId, uint32_t lid, const void *buffer, size_t sz) override ;
void write(BucketId bucketId, uint32_t chunkId, uint32_t lid, const void *buffer, size_t sz) override;
void close() override;
@@ -60,4 +58,3 @@ private:
};
}
-}
diff --git a/searchlib/src/vespa/searchlib/docstore/logdatastore.cpp b/searchlib/src/vespa/searchlib/docstore/logdatastore.cpp
index 46fcdafc585..e96628bdf4f 100644
--- a/searchlib/src/vespa/searchlib/docstore/logdatastore.cpp
+++ b/searchlib/src/vespa/searchlib/docstore/logdatastore.cpp
@@ -103,7 +103,6 @@ LogDataStore::~LogDataStore()
{
// Must be called before ending threads as there are sanity checks.
_fileChunks.clear();
- _executor.sync();
_genHandler.updateFirstUsedGeneration();
_lidInfo.removeOldGenerations(_genHandler.getFirstUsedGeneration());
}
diff --git a/searchlib/src/vespa/searchlib/docstore/storebybucket.cpp b/searchlib/src/vespa/searchlib/docstore/storebybucket.cpp
index d0d2d1cdac2..13de5687ee0 100644
--- a/searchlib/src/vespa/searchlib/docstore/storebybucket.cpp
+++ b/searchlib/src/vespa/searchlib/docstore/storebybucket.cpp
@@ -12,20 +12,21 @@ using document::BucketId;
using vespalib::makeTask;
using vespalib::makeClosure;
-StoreByBucket::StoreByBucket(MemoryDataStore & backingMemory, ThreadExecutor & executor, const CompressionConfig & compression) :
- _chunkSerial(0),
- _current(),
- _where(),
- _backingMemory(backingMemory),
- _executor(executor),
- _lock(),
- _chunks(),
- _compression(compression)
+StoreByBucket::StoreByBucket(MemoryDataStore & backingMemory, Executor & executor, const CompressionConfig & compression)
+ : _chunkSerial(0),
+ _current(),
+ _where(),
+ _backingMemory(backingMemory),
+ _executor(executor),
+ _monitor(),
+ _numChunksPosted(0),
+ _chunks(),
+ _compression(compression)
{
createChunk().swap(_current);
}
-StoreByBucket::~StoreByBucket() { }
+StoreByBucket::~StoreByBucket() = default;
void
StoreByBucket::add(BucketId bucketId, uint32_t chunkId, uint32_t lid, const void *buffer, size_t sz)
@@ -33,6 +34,7 @@ StoreByBucket::add(BucketId bucketId, uint32_t chunkId, uint32_t lid, const void
if ( ! _current->hasRoom(sz)) {
Chunk::UP tmpChunk = createChunk();
_current.swap(tmpChunk);
+ incChunksPosted();
_executor.execute(makeTask(makeClosure(this, &StoreByBucket::closeChunk, std::move(tmpChunk))));
}
Index idx(bucketId, _current->getId(), chunkId, lid);
@@ -48,7 +50,7 @@ StoreByBucket::createChunk()
size_t
StoreByBucket::getChunkCount() const {
- vespalib::LockGuard guard(_lock);
+ vespalib::LockGuard guard(_monitor);
return _chunks.size();
}
@@ -59,15 +61,33 @@ StoreByBucket::closeChunk(Chunk::UP chunk)
chunk->pack(1, buffer, _compression);
buffer.shrink(buffer.getDataLen());
ConstBufferRef bufferRef(_backingMemory.push_back(buffer.getData(), buffer.getDataLen()).data(), buffer.getDataLen());
- vespalib::LockGuard guard(_lock);
+ vespalib::MonitorGuard guard(_monitor);
_chunks[chunk->getId()] = bufferRef;
+ if (_numChunksPosted == _chunks.size()) {
+ guard.signal();
+ }
+}
+
+void
+StoreByBucket::incChunksPosted() {
+ vespalib::MonitorGuard guard(_monitor);
+ _numChunksPosted++;
+}
+
+void
+StoreByBucket::waitAllProcessed() {
+ vespalib::MonitorGuard guard(_monitor);
+ while (_numChunksPosted != _chunks.size()) {
+ guard.wait();
+ }
}
void
StoreByBucket::drain(IWrite & drainer)
{
+ incChunksPosted();
_executor.execute(makeTask(makeClosure(this, &StoreByBucket::closeChunk, std::move(_current))));
- _executor.sync();
+ waitAllProcessed();
std::vector<Chunk::UP> chunks;
chunks.resize(_chunks.size());
for (const auto & it : _chunks) {
diff --git a/searchlib/src/vespa/searchlib/docstore/storebybucket.h b/searchlib/src/vespa/searchlib/docstore/storebybucket.h
index ac1f6fbe007..8be0610b588 100644
--- a/searchlib/src/vespa/searchlib/docstore/storebybucket.h
+++ b/searchlib/src/vespa/searchlib/docstore/storebybucket.h
@@ -5,13 +5,12 @@
#include "chunk.h"
#include <vespa/document/bucket/bucketid.h>
#include <vespa/vespalib/data/memorydatastore.h>
-#include <vespa/vespalib/util/threadexecutor.h>
+#include <vespa/vespalib/util/executor.h>
#include <vespa/vespalib/util/sync.h>
#include <vespa/vespalib/stllike/hash_map.h>
#include <map>
-namespace search {
-namespace docstore {
+namespace search::docstore {
/**
* StoreByBucket will organize the data you add to it by buckets.
@@ -21,12 +20,12 @@ namespace docstore {
class StoreByBucket
{
using MemoryDataStore = vespalib::MemoryDataStore;
- using ThreadExecutor = vespalib::ThreadExecutor;
+ using Executor = vespalib::Executor;
using ConstBufferRef = vespalib::ConstBufferRef;
using CompressionConfig = vespalib::compression::CompressionConfig;
public:
StoreByBucket(vespalib::MemoryDataStore & backingMemory, const CompressionConfig & compression);
- StoreByBucket(MemoryDataStore & backingMemory, ThreadExecutor & executor, const CompressionConfig & compression);
+ StoreByBucket(MemoryDataStore & backingMemory, Executor & executor, const CompressionConfig & compression);
StoreByBucket(StoreByBucket &&) = default;
~StoreByBucket();
class IWrite {
@@ -47,6 +46,8 @@ public:
return lidCount;
}
private:
+ void incChunksPosted();
+ void waitAllProcessed();
Chunk::UP createChunk();
void closeChunk(Chunk::UP chunk);
struct Index {
@@ -67,11 +68,11 @@ private:
Chunk::UP _current;
std::map<uint64_t, IndexVector> _where;
MemoryDataStore & _backingMemory;
- ThreadExecutor & _executor;
- vespalib::Lock _lock;
+ Executor & _executor;
+ vespalib::Monitor _monitor;
+ size_t _numChunksPosted;
vespalib::hash_map<uint64_t, ConstBufferRef> _chunks;
CompressionConfig _compression;
};
}
-}
diff --git a/searchlib/src/vespa/searchlib/transactionlog/domain.h b/searchlib/src/vespa/searchlib/transactionlog/domain.h
index c0ee484926c..d6f964d5140 100644
--- a/searchlib/src/vespa/searchlib/transactionlog/domain.h
+++ b/searchlib/src/vespa/searchlib/transactionlog/domain.h
@@ -39,7 +39,7 @@ class Domain
{
public:
using SP = std::shared_ptr<Domain>;
- using Executor = vespalib::ThreadExecutor;
+ using Executor = vespalib::SyncableThreadExecutor;
Domain(const vespalib::string &name, const vespalib::string &baseDir, Executor & commitExecutor,
Executor & sessionExecutor, uint64_t domainPartSize, DomainPart::Crc defaultCrcType,
const common::FileHeaderContext &fileHeaderContext);
diff --git a/security-utils/src/main/java/com/yahoo/security/KeyFormat.java b/security-utils/src/main/java/com/yahoo/security/KeyFormat.java
new file mode 100644
index 00000000000..a04e7951dfe
--- /dev/null
+++ b/security-utils/src/main/java/com/yahoo/security/KeyFormat.java
@@ -0,0 +1,11 @@
+// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.security;
+
+/**
+ * Key format
+ *
+ * @author bjorncs
+ */
+public enum KeyFormat {
+ PKCS1, PKCS8
+}
diff --git a/security-utils/src/main/java/com/yahoo/security/KeyUtils.java b/security-utils/src/main/java/com/yahoo/security/KeyUtils.java
index f847e78f3c5..ed3b41d6e2a 100644
--- a/security-utils/src/main/java/com/yahoo/security/KeyUtils.java
+++ b/security-utils/src/main/java/com/yahoo/security/KeyUtils.java
@@ -141,10 +141,36 @@ public class KeyUtils {
}
}
+ // Note: Encoding using PKCS#1 as default as this is to be read by tools only supporting PKCS#1
+ // Should ideally be PKCS#8
public static String toPem(PrivateKey privateKey) {
+ return toPem(privateKey, KeyFormat.PKCS1);
+ }
+
+ public static String toPem(PrivateKey privateKey, KeyFormat format) {
+ switch (format) {
+ case PKCS1:
+ return toPkcs1Pem(privateKey);
+ case PKCS8:
+ return toPkcs8Pem(privateKey);
+ default:
+ throw new IllegalArgumentException("Unknown format: " + format);
+ }
+ }
+
+ public static String toPem(PublicKey publicKey) {
+ try (StringWriter stringWriter = new StringWriter(); JcaPEMWriter pemWriter = new JcaPEMWriter(stringWriter)) {
+ pemWriter.writeObject(publicKey);
+ pemWriter.flush();
+ return stringWriter.toString();
+ } catch (IOException e) {
+ throw new UncheckedIOException(e);
+ }
+ }
+
+ private static String toPkcs1Pem(PrivateKey privateKey) {
try (StringWriter stringWriter = new StringWriter(); JcaPEMWriter pemWriter = new JcaPEMWriter(stringWriter)) {
String algorithm = privateKey.getAlgorithm();
- // Note: Encoding using PKCS#1 as this is to be read by tools only supporting PKCS#1
String type;
if (algorithm.equals(RSA.getAlgorithmName())) {
type = "RSA PRIVATE KEY";
@@ -161,9 +187,9 @@ public class KeyUtils {
}
}
- public static String toPem(PublicKey publicKey) {
+ private static String toPkcs8Pem(PrivateKey privateKey) {
try (StringWriter stringWriter = new StringWriter(); JcaPEMWriter pemWriter = new JcaPEMWriter(stringWriter)) {
- pemWriter.writeObject(publicKey);
+ pemWriter.writeObject(new PemObject("PRIVATE KEY", privateKey.getEncoded()));
pemWriter.flush();
return stringWriter.toString();
} catch (IOException e) {
diff --git a/security-utils/src/test/java/com/yahoo/security/KeyUtilsTest.java b/security-utils/src/test/java/com/yahoo/security/KeyUtilsTest.java
index dc0c0a126ea..8a1c72b1e59 100644
--- a/security-utils/src/test/java/com/yahoo/security/KeyUtilsTest.java
+++ b/security-utils/src/test/java/com/yahoo/security/KeyUtilsTest.java
@@ -32,25 +32,23 @@ public class KeyUtilsTest {
}
@Test
- public void can_serialize_and_deserialize_rsa_privatekey_using_pem_format() {
- KeyPair keyPair = KeyUtils.generateKeypair(KeyAlgorithm.RSA);
- String pem = KeyUtils.toPem(keyPair.getPrivate());
- assertThat(pem, containsString("BEGIN RSA PRIVATE KEY"));
- assertThat(pem, containsString("END RSA PRIVATE KEY"));
- PrivateKey deserializedKey = KeyUtils.fromPemEncodedPrivateKey(pem);
- assertEquals(keyPair.getPrivate(), deserializedKey);
- assertEquals(KeyAlgorithm.RSA.getAlgorithmName(), deserializedKey.getAlgorithm());
+ public void can_serialize_and_deserialize_rsa_privatekey_using_pkcs1_pem_format() {
+ testPrivateKeySerialization(KeyAlgorithm.RSA, KeyFormat.PKCS1, "RSA PRIVATE KEY");
}
@Test
- public void can_serialize_and_deserialize_ec_privatekey_using_pem_format() {
- KeyPair keyPair = KeyUtils.generateKeypair(KeyAlgorithm.EC);
- String pem = KeyUtils.toPem(keyPair.getPrivate());
- assertThat(pem, containsString("BEGIN EC PRIVATE KEY"));
- assertThat(pem, containsString("END EC PRIVATE KEY"));
- PrivateKey deserializedKey = KeyUtils.fromPemEncodedPrivateKey(pem);
- assertEquals(keyPair.getPrivate(), deserializedKey);
- assertEquals(KeyAlgorithm.EC.getAlgorithmName(), deserializedKey.getAlgorithm());
+ public void can_serialize_and_deserialize_rsa_privatekey_using_pkcs8_pem_format() {
+ testPrivateKeySerialization(KeyAlgorithm.RSA, KeyFormat.PKCS8, "PRIVATE KEY");
+ }
+
+ @Test
+ public void can_serialize_and_deserialize_ec_privatekey_using_pkcs1_pem_format() {
+ testPrivateKeySerialization(KeyAlgorithm.EC, KeyFormat.PKCS1, "EC PRIVATE KEY");
+ }
+
+ @Test
+ public void can_serialize_and_deserialize_ec_privatekey_using_pkcs8_pem_format() {
+ testPrivateKeySerialization(KeyAlgorithm.EC, KeyFormat.PKCS8, "PRIVATE KEY");
}
@Test
@@ -75,4 +73,14 @@ public class KeyUtilsTest {
assertEquals(KeyAlgorithm.EC.getAlgorithmName(), deserializedKey.getAlgorithm());
}
+ private static void testPrivateKeySerialization(KeyAlgorithm keyAlgorithm, KeyFormat keyFormat, String pemLabel) {
+ KeyPair keyPair = KeyUtils.generateKeypair(keyAlgorithm);
+ String pem = KeyUtils.toPem(keyPair.getPrivate(), keyFormat);
+ assertThat(pem, containsString("BEGIN " + pemLabel));
+ assertThat(pem, containsString("END " + pemLabel));
+ PrivateKey deserializedKey = KeyUtils.fromPemEncodedPrivateKey(pem);
+ assertEquals(keyPair.getPrivate(), deserializedKey);
+ assertEquals(keyAlgorithm.getAlgorithmName(), deserializedKey.getAlgorithm());
+ }
+
}
diff --git a/vespalib/src/vespa/vespalib/util/threadexecutor.h b/vespalib/src/vespa/vespalib/util/threadexecutor.h
index 158805288e9..2dcbb595bb3 100644
--- a/vespalib/src/vespa/vespalib/util/threadexecutor.h
+++ b/vespalib/src/vespa/vespalib/util/threadexecutor.h
@@ -2,16 +2,12 @@
#pragma once
-#include <vespa/vespalib/util/executor.h>
-#include <vespa/vespalib/util/syncable.h>
+#include "executor.h"
+#include "syncable.h"
namespace vespalib {
-/**
- * Can both execute and sync
- **/
-class ThreadExecutor : public Executor,
- public Syncable
+class ThreadExecutor : public Executor
{
public:
/**
@@ -21,5 +17,13 @@ public:
virtual size_t getNumThreads() const = 0;
};
+/**
+ * Can both execute and sync
+ **/
+class SyncableThreadExecutor : public ThreadExecutor, public Syncable
+{
+public:
+};
+
} // namespace vespalib
diff --git a/vespalib/src/vespa/vespalib/util/threadstackexecutorbase.cpp b/vespalib/src/vespa/vespalib/util/threadstackexecutorbase.cpp
index dfd835e0e8e..969b5e6f61e 100644
--- a/vespalib/src/vespa/vespalib/util/threadstackexecutorbase.cpp
+++ b/vespalib/src/vespa/vespalib/util/threadstackexecutorbase.cpp
@@ -135,7 +135,9 @@ ThreadStackExecutorBase::run()
ThreadStackExecutorBase::ThreadStackExecutorBase(uint32_t stackSize,
uint32_t taskLimit,
init_fun_t init_fun)
- : _pool(std::make_unique<FastOS_ThreadPool>(stackSize)),
+ : SyncableThreadExecutor(),
+ Runnable(),
+ _pool(std::make_unique<FastOS_ThreadPool>(stackSize)),
_monitor(),
_stats(),
_executorCompletion(),
diff --git a/vespalib/src/vespa/vespalib/util/threadstackexecutorbase.h b/vespalib/src/vespa/vespalib/util/threadstackexecutorbase.h
index 8ee08ed3929..21a6e9cabe0 100644
--- a/vespalib/src/vespa/vespalib/util/threadstackexecutorbase.h
+++ b/vespalib/src/vespa/vespalib/util/threadstackexecutorbase.h
@@ -32,7 +32,7 @@ namespace thread { struct ThreadInit; }
/**
* An executor service that executes tasks in multiple threads.
**/
-class ThreadStackExecutorBase : public ThreadExecutor,
+class ThreadStackExecutorBase : public SyncableThreadExecutor,
public Runnable
{
public:
@@ -238,7 +238,7 @@ public:
/**
* Will invoke shutdown then sync.
**/
- ~ThreadStackExecutorBase();
+ ~ThreadStackExecutorBase() override;
};
} // namespace vespalib