diff options
44 files changed, 538 insertions, 217 deletions
diff --git a/application/pom.xml b/application/pom.xml index c70efab05aa..7e1b54a7904 100644 --- a/application/pom.xml +++ b/application/pom.xml @@ -120,6 +120,16 @@ <groupId>org.apache.commons</groupId> <artifactId>commons-math3</artifactId> </dependency> + <dependency> + <groupId>org.apache.httpcomponents.client5</groupId> + <artifactId>httpclient5</artifactId> + <exclusions> + <exclusion> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-api</artifactId> + </exclusion> + </exclusions> + </dependency> <dependency> <groupId>com.yahoo.vespa</groupId> diff --git a/build_settings.cmake b/build_settings.cmake index d0cf6f38326..e36c56eb9e6 100644 --- a/build_settings.cmake +++ b/build_settings.cmake @@ -32,6 +32,13 @@ if (VESPA_USE_SANITIZER) else() set(C_WARN_OPTS "-Winline ${C_WARN_OPTS}") endif() +if (VESPA_USE_SANITIZER) + if (VESPA_USE_SANITIZER STREQUAL "thread" AND CMAKE_CXX_COMPILER_ID STREQUAL "GNU" AND CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL 12.0) + # Turn off warning about std::atomic_thread_fence not being supported by + # address sanitizer. + set(C_WARN_OPTS "${C_WARN_OPTS} -Wno-tsan") + endif() +endif() # Warnings that are specific to C++ compilation # Note: this is not a union of C_WARN_OPTS, since CMAKE_CXX_FLAGS already includes CMAKE_C_FLAGS, which in turn includes C_WARN_OPTS transitively diff --git a/config-model-api/abi-spec.json b/config-model-api/abi-spec.json index 13988d81e86..ea59c4409c3 100644 --- a/config-model-api/abi-spec.json +++ b/config-model-api/abi-spec.json @@ -297,7 +297,8 @@ "public" ], "methods": [ - "public void <init>(java.lang.String, java.util.List, java.lang.String)", + "public void <init>(int, java.lang.String, java.util.List, java.lang.String)", + "public int majorVersion()", "public java.lang.String humanReadableString()", "public java.lang.String toString()" ], diff --git a/config-model-api/src/main/java/com/yahoo/config/application/api/DeploymentSpec.java b/config-model-api/src/main/java/com/yahoo/config/application/api/DeploymentSpec.java index 22ffdeb7262..c4fec6e668e 100644 --- a/config-model-api/src/main/java/com/yahoo/config/application/api/DeploymentSpec.java +++ b/config-model-api/src/main/java/com/yahoo/config/application/api/DeploymentSpec.java @@ -637,21 +637,29 @@ public class DeploymentSpec { private final String tagName; private final List<String> attributes; private final String message; + private final int majorVersion; - public DeprecatedElement(String tagName, List<String> attributes, String message) { + public DeprecatedElement(int majorVersion, String tagName, List<String> attributes, String message) { this.tagName = Objects.requireNonNull(tagName); this.attributes = Objects.requireNonNull(attributes); this.message = Objects.requireNonNull(message); + this.majorVersion = majorVersion; if (message.isBlank()) throw new IllegalArgumentException("message must be non-empty"); } + /** Returns the major version that deprecated this element */ + public int majorVersion() { + return majorVersion; + } + public String humanReadableString() { + String deprecationDescription = "deprecated since major version " + majorVersion; if (attributes.isEmpty()) { - return "Element '" + tagName + "' is deprecated. " + message; + return "Element '" + tagName + "' is " + deprecationDescription + ". " + message; } - return "Element '" + tagName + "' contains deprecated attribute" + (attributes.size() > 1 ? "s" : "") + ": " + + return "Element '" + tagName + "' contains attribute" + (attributes.size() > 1 ? "s " : " ") + attributes.stream().map(attr -> "'" + attr + "'").collect(Collectors.joining(", ")) + - ". " + message; + " " + deprecationDescription + ". " + message; } @Override diff --git a/config-model-api/src/main/java/com/yahoo/config/application/api/xml/DeploymentSpecXmlReader.java b/config-model-api/src/main/java/com/yahoo/config/application/api/xml/DeploymentSpecXmlReader.java index 2f73bff83b8..09d61835ae3 100644 --- a/config-model-api/src/main/java/com/yahoo/config/application/api/xml/DeploymentSpecXmlReader.java +++ b/config-model-api/src/main/java/com/yahoo/config/application/api/xml/DeploymentSpecXmlReader.java @@ -439,7 +439,7 @@ public class DeploymentSpecXmlReader { private Optional<String> readGlobalServiceId(Element environmentTag) { String globalServiceId = environmentTag.getAttribute(globalServiceIdAttribute); if (globalServiceId.isEmpty()) return Optional.empty(); - deprecate(environmentTag, List.of(globalServiceIdAttribute), "See https://cloud.vespa.ai/en/reference/routing#deprecated-syntax"); + deprecate(environmentTag, List.of(globalServiceIdAttribute), 7, "See https://cloud.vespa.ai/en/reference/routing#deprecated-syntax"); return Optional.of(globalServiceId); } @@ -524,15 +524,15 @@ public class DeploymentSpecXmlReader { private boolean readActive(Element regionTag) { String activeValue = regionTag.getAttribute("active"); if ("".equals(activeValue)) return true; // Default to active - deprecate(regionTag, List.of("active"), "See https://cloud.vespa.ai/en/reference/routing#deprecated-syntax"); + deprecate(regionTag, List.of("active"), 7, "See https://cloud.vespa.ai/en/reference/routing#deprecated-syntax"); if ("true".equals(activeValue)) return true; if ("false".equals(activeValue)) return false; throw new IllegalArgumentException("Value of 'active' attribute in region tag must be 'true' or 'false' " + "to control whether this region should receive traffic from the global endpoint of this application"); } - private void deprecate(Element element, List<String> attributes, String message) { - deprecatedElements.add(new DeprecatedElement(element.getTagName(), attributes, message)); + private void deprecate(Element element, List<String> attributes, int majorVersion, String message) { + deprecatedElements.add(new DeprecatedElement(majorVersion, element.getTagName(), attributes, message)); } private static boolean isEmptySpec(Element root) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/BundleValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/BundleValidator.java index a1024425124..1e3b27bc2fa 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/BundleValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/BundleValidator.java @@ -84,7 +84,10 @@ public class BundleValidator extends AbstractBundleValidator { ORG_JSON("org.json:json", "This bundle is no longer provided on Vespa 8 - " + "see https://docs.vespa.ai/en/vespa8-release-notes.html#container-runtime.", - Set.of("org\\.json")); + Set.of("org\\.json")), + JETTY("jetty", "The Jetty bundles are no longer provided on Vespa 8 - " + + "see https://docs.vespa.ai/en/vespa8-release-notes.html#container-runtime.", + Set.of("org\\.eclipse\\.jetty.*")); final String name; final Collection<Predicate<String>> javaPackageMatchers; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/StreamingValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/StreamingValidator.java index 458f8dfbd68..773d696f3e8 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/StreamingValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/StreamingValidator.java @@ -11,6 +11,7 @@ import com.yahoo.schema.document.ImmutableSDField; import com.yahoo.schema.document.MatchType; import com.yahoo.vespa.model.VespaModel; import com.yahoo.vespa.model.search.SearchCluster; +import com.yahoo.vespa.model.search.StreamingSearchCluster; import java.util.List; import java.util.logging.Level; @@ -25,21 +26,21 @@ public class StreamingValidator extends Validator { List<SearchCluster> searchClusters = model.getSearchClusters(); for (SearchCluster cluster : searchClusters) { if ( ! cluster.isStreaming()) continue; - warnStreamingAttributes(cluster, deployState.getDeployLogger()); - warnStreamingGramMatching(cluster, deployState.getDeployLogger()); - failStreamingDocumentReferences(cluster); + + var streamingCluster = (StreamingSearchCluster)cluster; + warnStreamingAttributes(streamingCluster, deployState.getDeployLogger()); + warnStreamingGramMatching(streamingCluster, deployState.getDeployLogger()); + failStreamingDocumentReferences(streamingCluster); } } - private static void warnStreamingGramMatching(SearchCluster sc, DeployLogger logger) { - if (sc.getSchemaConfig() != null) { - for (ImmutableSDField sd : sc.getSchemaConfig().getSchema().allConcreteFields()) { - if (sd.getMatching().getType() == MatchType.GRAM) { - logger.logApplicationPackage(Level.WARNING, "For streaming search cluster '" + - sc.getClusterName() + - "', SD field '" + sd.getName() + - "': n-gram matching is not supported for streaming search."); - } + private static void warnStreamingGramMatching(StreamingSearchCluster sc, DeployLogger logger) { + for (ImmutableSDField sd : sc.derived().getSchema().allConcreteFields()) { + if (sd.getMatching().getType() == MatchType.GRAM) { + logger.logApplicationPackage(Level.WARNING, "For streaming search cluster '" + + sc.getClusterName() + + "', SD field '" + sd.getName() + + "': n-gram matching is not supported for streaming search."); } } } @@ -50,17 +51,15 @@ public class StreamingValidator extends Validator { * @param sc a search cluster to be checked for attributes in streaming search * @param logger a DeployLogger */ - private static void warnStreamingAttributes(SearchCluster sc, DeployLogger logger) { - if (sc.getSchemaConfig() != null) { - for (ImmutableSDField sd : sc.getSchemaConfig().getSchema().allConcreteFields()) { - if (sd.doesAttributing()) { - warnStreamingAttribute(sc, sd, logger); - } + private static void warnStreamingAttributes(StreamingSearchCluster sc, DeployLogger logger) { + for (ImmutableSDField sd : sc.derived().getSchema().allConcreteFields()) { + if (sd.doesAttributing()) { + warnStreamingAttribute(sc, sd, logger); } } } - private static void warnStreamingAttribute(SearchCluster sc, ImmutableSDField sd, DeployLogger logger) { + private static void warnStreamingAttribute(StreamingSearchCluster sc, ImmutableSDField sd, DeployLogger logger) { // If the field is numeric, we can't print this, because we may have converted the field to // attribute indexing ourselves (IntegerIndex2Attribute) if (sd.getDataType() instanceof NumericDataType) return; @@ -69,8 +68,8 @@ public class StreamingValidator extends Validator { "': 'attribute' has same match semantics as 'index'."); } - private static void failStreamingDocumentReferences(SearchCluster sc) { - for (Attribute attribute : sc.getSchemaConfig().getAttributeFields().attributes()) { + private static void failStreamingDocumentReferences(StreamingSearchCluster sc) { + for (Attribute attribute : sc.derived().getAttributeFields().attributes()) { DataType dataType = attribute.getDataType(); if (dataType instanceof NewDocumentReferenceDataType) { String errorMessage = String.format("For streaming search cluster '%s': Attribute '%s' has type '%s'. " + diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/StreamingSearchClusterChangeValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/StreamingSearchClusterChangeValidator.java index 0563aa02149..167aff1064b 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/StreamingSearchClusterChangeValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/StreamingSearchClusterChangeValidator.java @@ -64,11 +64,11 @@ public class StreamingSearchClusterChangeValidator implements ChangeValidator { getDocumentType(currentCluster, currentStreamingCluster), getDocumentType(nextCluster, nextStreamingCluster), overrides, now)); result.addAll(validateAttributeFastAccessAdded(currentCluster.id(), - currentStreamingCluster.getSchemaConfig().getAttributeFields(), - nextStreamingCluster.getSchemaConfig().getAttributeFields())); + currentStreamingCluster.derived().getAttributeFields(), + nextStreamingCluster.derived().getAttributeFields())); result.addAll(validateAttributeFastAccessRemoved(currentCluster.id(), - currentStreamingCluster.getSchemaConfig().getAttributeFields(), - nextStreamingCluster.getSchemaConfig().getAttributeFields())); + currentStreamingCluster.derived().getAttributeFields(), + nextStreamingCluster.derived().getAttributeFields())); return modifyActions(result, getSearchNodeServices(nextCluster), nextStreamingCluster.getDocTypeName()); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java index 4be5153293f..1ab5fd22648 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java @@ -329,7 +329,7 @@ public class ContentSearchCluster extends AbstractConfigProducer<SearchCluster> return getClusters().values().stream() .filter(StreamingSearchCluster.class::isInstance) .map(StreamingSearchCluster.class::cast) - .filter(ssc -> ssc.getSchemaConfig().getSchema().getName().equals(docType)) + .filter(ssc -> ssc.schemas().get(docType) != null) .findFirst(); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java index d63ae451f39..9efe61d134f 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java @@ -222,9 +222,6 @@ public class IndexedSearchCluster extends SearchCluster } @Override - public DerivedConfiguration getSchemaConfig() { return null; } - - @Override public void getConfig(IndexInfoConfig.Builder builder) { documentDbsConfigProducer.getConfig(builder); } @@ -239,12 +236,10 @@ public class IndexedSearchCluster extends SearchCluster documentDbsConfigProducer.getConfig(builder); } - @Override public void getConfig(AttributesConfig.Builder builder) { documentDbsConfigProducer.getConfig(builder); } - @Override public void getConfig(RankProfilesConfig.Builder builder) { documentDbsConfigProducer.getConfig(builder); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/SearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/search/SearchCluster.java index d727e6d6395..f033399c787 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/SearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/SearchCluster.java @@ -5,14 +5,13 @@ import com.yahoo.config.model.deploy.DeployState; import com.yahoo.search.config.SchemaInfoConfig; import com.yahoo.schema.derived.SchemaInfo; import com.yahoo.schema.derived.SummaryMap; +import com.yahoo.vespa.config.search.AttributesConfig; import com.yahoo.vespa.config.search.RankProfilesConfig; import com.yahoo.vespa.config.search.SummaryConfig; import com.yahoo.prelude.fastsearch.DocumentdbInfoConfig; import com.yahoo.vespa.config.search.SummarymapConfig; import com.yahoo.search.config.IndexInfoConfig; -import com.yahoo.vespa.config.search.AttributesConfig; import com.yahoo.vespa.configdefinition.IlscriptsConfig; -import com.yahoo.schema.derived.DerivedConfiguration; import com.yahoo.config.model.producer.AbstractConfigProducer; import java.util.Collections; @@ -147,35 +146,10 @@ public abstract class SearchCluster extends AbstractConfigProducer<SearchCluster public final int getClusterIndex() { return index; } public abstract void defaultDocumentsConfig(); - public abstract DerivedConfiguration getSchemaConfig(); - - // TODO: The get methods below should be moved to StreamingSearchCluster - - @Override - public void getConfig(IndexInfoConfig.Builder builder) { - if (getSchemaConfig() != null) getSchemaConfig().getIndexInfo().getConfig(builder); - } - - @Override - public void getConfig(SchemaInfoConfig.Builder builder) { - if (getSchemaConfig() != null) getSchemaConfig().getSchemaInfo().getConfig(builder); - } - - @Override - public void getConfig(IlscriptsConfig.Builder builder) { - if (getSchemaConfig() != null) getSchemaConfig().getIndexingScript().getConfig(builder); - } - public void getConfig(AttributesConfig.Builder builder) { - if (getSchemaConfig() != null) getSchemaConfig().getConfig(builder); - } - - public void getConfig(RankProfilesConfig.Builder builder) { - if (getSchemaConfig() != null) getSchemaConfig().getRankProfileList().getConfig(builder); - } + public abstract void getConfig(AttributesConfig.Builder builder); - @Override - public abstract void getConfig(DocumentdbInfoConfig.Builder builder); + public abstract void getConfig(RankProfilesConfig.Builder builder); @Override public String toString() { return "search-capable cluster '" + clusterName + "'"; } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/StreamingSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/search/StreamingSearchCluster.java index 724ba70510f..79adfb2ea13 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/StreamingSearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/StreamingSearchCluster.java @@ -7,12 +7,15 @@ import com.yahoo.prelude.fastsearch.DocumentdbInfoConfig; import com.yahoo.schema.Schema; import com.yahoo.schema.derived.AttributeFields; import com.yahoo.schema.derived.DerivedConfiguration; +import com.yahoo.search.config.IndexInfoConfig; +import com.yahoo.search.config.SchemaInfoConfig; import com.yahoo.vespa.config.search.AttributesConfig; import com.yahoo.vespa.config.search.RankProfilesConfig; import com.yahoo.vespa.config.search.SummaryConfig; import com.yahoo.vespa.config.search.SummarymapConfig; import com.yahoo.vespa.config.search.vsm.VsmfieldsConfig; import com.yahoo.vespa.config.search.vsm.VsmsummaryConfig; +import com.yahoo.vespa.configdefinition.IlscriptsConfig; /** * A search cluster of type streaming. @@ -20,8 +23,8 @@ import com.yahoo.vespa.config.search.vsm.VsmsummaryConfig; * @author baldersheim * @author vegardh */ -public class StreamingSearchCluster extends SearchCluster implements - DocumentdbInfoConfig.Producer, +public class StreamingSearchCluster extends SearchCluster implements + DocumentdbInfoConfig.Producer, RankProfilesConfig.Producer, VsmsummaryConfig.Producer, VsmfieldsConfig.Producer, @@ -31,7 +34,7 @@ public class StreamingSearchCluster extends SearchCluster implements private final String storageRouteSpec; private final AttributesProducer attributesConfig; private final String docTypeName; - private DerivedConfiguration schemaConfig = null; + private DerivedConfiguration derivedConfig = null; public StreamingSearchCluster(AbstractConfigProducer<SearchCluster> parent, String clusterName, @@ -51,9 +54,9 @@ public class StreamingSearchCluster extends SearchCluster implements protected IndexingMode getIndexingMode() { return IndexingMode.STREAMING; } public final String getStorageRouteSpec() { return storageRouteSpec; } - public String getDocTypeName() { - return docTypeName; - } + public String getDocTypeName() { return docTypeName; } + + public DerivedConfiguration derived() { return derivedConfig; } @Override public int getRowBits() { return 0; } @@ -61,10 +64,10 @@ public class StreamingSearchCluster extends SearchCluster implements @Override public void getConfig(DocumentdbInfoConfig.Builder builder) { DocumentdbInfoConfig.Documentdb.Builder docDb = new DocumentdbInfoConfig.Documentdb.Builder(); - docDb.name(schemaConfig.getSchema().getName()); - SummaryConfig.Producer prod = schemaConfig.getSummaries(); + docDb.name(derivedConfig.getSchema().getName()); + SummaryConfig.Producer prod = derivedConfig.getSummaries(); convertSummaryConfig(prod, null, docDb); - addRankProfilesConfig(schemaConfig.getSchema().getName(), docDb); + addRankProfilesConfig(derivedConfig.getSchema().getName(), docDb); builder.documentdb(docDb); } @@ -77,46 +80,58 @@ public class StreamingSearchCluster extends SearchCluster implements if ( ! schema.getName().equals(docTypeName)) throw new IllegalArgumentException("Document type name '" + docTypeName + "' must be the same as the schema name '" + schema.getName() + "'"); - this.schemaConfig = new DerivedConfiguration(schema, deployState); + this.derivedConfig = new DerivedConfiguration(schema, deployState); } @Override - public DerivedConfiguration getSchemaConfig() { return schemaConfig; } + public void defaultDocumentsConfig() { } @Override - public void defaultDocumentsConfig() { } + public void getConfig(IndexInfoConfig.Builder builder) { + derivedConfig.getIndexInfo().getConfig(builder); + } + + @Override + public void getConfig(SchemaInfoConfig.Builder builder) { + derivedConfig.getSchemaInfo().getConfig(builder); + } @Override + public void getConfig(IlscriptsConfig.Builder builder) { + derivedConfig.getIndexingScript().getConfig(builder); + } + public void getConfig(AttributesConfig.Builder builder) { - if (getSchemaConfig() != null) getSchemaConfig().getConfig(builder); + derivedConfig.getConfig(builder); } - + + @Override + public void getConfig(RankProfilesConfig.Builder builder) { + derivedConfig.getRankProfileList().getConfig(builder); + } + @Override public void getConfig(VsmsummaryConfig.Builder builder) { - if (getSchemaConfig() != null) - if (getSchemaConfig().getVsmSummary() != null) - getSchemaConfig().getVsmSummary().getConfig(builder); + if (derivedConfig.getVsmSummary() != null) + derivedConfig.getVsmSummary().getConfig(builder); } @Override public void getConfig(VsmfieldsConfig.Builder builder) { - if (getSchemaConfig() != null) - if (getSchemaConfig().getVsmFields() != null) - getSchemaConfig().getVsmFields().getConfig(builder); + if (derivedConfig.getVsmFields() != null) + derivedConfig.getVsmFields().getConfig(builder); } @Override public void getConfig(SummarymapConfig.Builder builder) { - if (getSchemaConfig() != null) - if (getSchemaConfig().getSummaryMap() != null) - getSchemaConfig().getSummaryMap().getConfig(builder); + if (derivedConfig.getSummaryMap() != null) + derivedConfig.getSummaryMap().getConfig(builder); } @Override public void getConfig(SummaryConfig.Builder builder) { - if (getSchemaConfig() != null) - if (getSchemaConfig().getSummaries() != null) - getSchemaConfig().getSummaries().getConfig(builder); + if (derivedConfig.getSummaries() != null) + derivedConfig.getSummaries().getConfig(builder); } private class AttributesProducer extends AbstractConfigProducer<AttributesProducer> implements AttributesConfig.Producer { @@ -127,9 +142,7 @@ public class StreamingSearchCluster extends SearchCluster implements @Override public void getConfig(AttributesConfig.Builder builder) { - if (getSchemaConfig() != null) { - getSchemaConfig().getConfig(builder, AttributeFields.FieldSet.FAST_ACCESS); - } + derivedConfig.getConfig(builder, AttributeFields.FieldSet.FAST_ACCESS); } } diff --git a/config-model/src/test/cfg/application/validation/testjars/import-warnings/META-INF/MANIFEST.MF b/config-model/src/test/cfg/application/validation/testjars/import-warnings/META-INF/MANIFEST.MF index 760a9ecf00f..3185bfb2879 100644 --- a/config-model/src/test/cfg/application/validation/testjars/import-warnings/META-INF/MANIFEST.MF +++ b/config-model/src/test/cfg/application/validation/testjars/import-warnings/META-INF/MANIFEST.MF @@ -6,5 +6,5 @@ Bundle-Version: 7.0.0 Created-By: vespa container maven plugin Bundle-Name: my-bundle Bundle-Vendor: Yahoo! -Import-Package: org.json;version="[0.0.0,1)" +Import-Package: org.json;version="[0.0.0,1)",org.eclipse.jetty.client.api;version="[9.4.46,10)" diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/BundleValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/BundleValidatorTest.java index b5b93be6cd7..facd3cfdff4 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/BundleValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/BundleValidatorTest.java @@ -63,9 +63,13 @@ public class BundleValidatorTest { BundleValidator validator = new BundleValidator(); JarFile jarFile = createTemporaryJarFile("import-warnings"); validator.validateJarFile(state, jarFile); - assertThat(buffer.toString()) + String output = buffer.toString(); + assertThat(output) .contains("JAR file 'import-warnings.jar' imports the packages [org.json] from 'org.json:json'. \n" + "This bundle is no longer provided on Vespa 8 - see https://docs.vespa.ai/en/vespa8-release-notes.html#container-runtime."); + assertThat(output) + .contains("JAR file 'import-warnings.jar' imports the packages [org.eclipse.jetty.client.api] from 'jetty'. \n" + + "The Jetty bundles are no longer provided on Vespa 8 - see https://docs.vespa.ai/en/vespa8-release-notes.html#container-runtime."); } private DeployState createDeployState(StringBuffer buffer) { diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/search/searchchain/MockSearchClusters.java b/config-model/src/test/java/com/yahoo/vespa/model/container/search/searchchain/MockSearchClusters.java index 0d51e4b6e74..81c7b778ed3 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/search/searchchain/MockSearchClusters.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/search/searchchain/MockSearchClusters.java @@ -2,7 +2,7 @@ package com.yahoo.vespa.model.container.search.searchchain; import com.yahoo.config.model.deploy.DeployState; -import com.yahoo.schema.derived.DerivedConfiguration; +import com.yahoo.search.config.SchemaInfoConfig; import com.yahoo.vespa.config.search.AttributesConfig; import com.yahoo.vespa.config.search.RankProfilesConfig; import com.yahoo.config.model.producer.AbstractConfigProducerRoot; @@ -33,17 +33,19 @@ public class MockSearchClusters { } @Override - protected SearchCluster.IndexingMode getIndexingMode() { return streaming ? SearchCluster.IndexingMode.STREAMING : SearchCluster.IndexingMode.REALTIME; } + public void defaultDocumentsConfig() {} @Override - public void getConfig(DocumentdbInfoConfig.Builder builder) { + public void getConfig(AttributesConfig.Builder builder) { } @Override - public void defaultDocumentsConfig() {} + public void getConfig(DocumentdbInfoConfig.Builder builder) { + } @Override - public DerivedConfiguration getSchemaConfig() { return null; } + public void getConfig(RankProfilesConfig.Builder builder) { + } @Override public void getConfig(IndexInfoConfig.Builder builder) { @@ -54,12 +56,11 @@ public class MockSearchClusters { } @Override - public void getConfig(AttributesConfig.Builder builder) { + public void getConfig(SchemaInfoConfig.Builder builder) { } @Override - public void getConfig(RankProfilesConfig.Builder builder) { - } + protected SearchCluster.IndexingMode getIndexingMode() { return streaming ? SearchCluster.IndexingMode.STREAMING : SearchCluster.IndexingMode.REALTIME; } } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java index 81649375c8a..701878f98ab 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java @@ -1089,9 +1089,9 @@ public class ContainerModelBuilderTest extends ContainerModelBuilderTestBase { assertFalse(logger.msgs.isEmpty()); assertEquals(Level.WARNING, logger.msgs.get(0).getFirst()); assertEquals(Level.WARNING, logger.msgs.get(1).getFirst()); - assertEquals("Element 'prod' contains deprecated attribute: 'global-service-id'. See https://cloud.vespa.ai/en/reference/routing#deprecated-syntax", + assertEquals("Element 'prod' contains attribute 'global-service-id' deprecated since major version 7. See https://cloud.vespa.ai/en/reference/routing#deprecated-syntax", logger.msgs.get(0).getSecond()); - assertEquals("Element 'region' contains deprecated attribute: 'active'. See https://cloud.vespa.ai/en/reference/routing#deprecated-syntax", + assertEquals("Element 'region' contains attribute 'active' deprecated since major version 7. See https://cloud.vespa.ai/en/reference/routing#deprecated-syntax", logger.msgs.get(1).getSecond()); } diff --git a/container-core/abi-spec.json b/container-core/abi-spec.json index 78a1c044d35..8ca6507f73e 100644 --- a/container-core/abi-spec.json +++ b/container-core/abi-spec.json @@ -2090,6 +2090,60 @@ "public static final java.lang.String[] CONFIG_DEF_SCHEMA" ] }, + "com.yahoo.jdisc.http.SslProvider$ConnectorSsl$ClientAuth": { + "superClass": "java.lang.Enum", + "interfaces": [], + "attributes": [ + "public", + "final", + "enum" + ], + "methods": [ + "public static com.yahoo.jdisc.http.SslProvider$ConnectorSsl$ClientAuth[] values()", + "public static com.yahoo.jdisc.http.SslProvider$ConnectorSsl$ClientAuth valueOf(java.lang.String)" + ], + "fields": [ + "public static final enum com.yahoo.jdisc.http.SslProvider$ConnectorSsl$ClientAuth DISABLED", + "public static final enum com.yahoo.jdisc.http.SslProvider$ConnectorSsl$ClientAuth WANT", + "public static final enum com.yahoo.jdisc.http.SslProvider$ConnectorSsl$ClientAuth NEED" + ] + }, + "com.yahoo.jdisc.http.SslProvider$ConnectorSsl": { + "superClass": "java.lang.Object", + "interfaces": [], + "attributes": [ + "public", + "interface", + "abstract" + ], + "methods": [ + "public abstract com.yahoo.jdisc.http.SslProvider$ConnectorSsl setSslContext(javax.net.ssl.SSLContext)", + "public abstract com.yahoo.jdisc.http.SslProvider$ConnectorSsl setClientAuth(com.yahoo.jdisc.http.SslProvider$ConnectorSsl$ClientAuth)", + "public abstract com.yahoo.jdisc.http.SslProvider$ConnectorSsl setEnabledCipherSuites(java.util.List)", + "public abstract com.yahoo.jdisc.http.SslProvider$ConnectorSsl setEnabledProtocolVersions(java.util.List)", + "public abstract com.yahoo.jdisc.http.SslProvider$ConnectorSsl setKeystore(java.security.KeyStore, char[])", + "public abstract com.yahoo.jdisc.http.SslProvider$ConnectorSsl setKeystore(java.security.KeyStore)", + "public abstract com.yahoo.jdisc.http.SslProvider$ConnectorSsl setTruststore(java.security.KeyStore, char[])", + "public abstract com.yahoo.jdisc.http.SslProvider$ConnectorSsl setTruststore(java.security.KeyStore)" + ], + "fields": [] + }, + "com.yahoo.jdisc.http.SslProvider": { + "superClass": "java.lang.Object", + "interfaces": [ + "java.lang.AutoCloseable" + ], + "attributes": [ + "public", + "interface", + "abstract" + ], + "methods": [ + "public abstract void configureSsl(com.yahoo.jdisc.http.SslProvider$ConnectorSsl, java.lang.String, int)", + "public void close()" + ], + "fields": [] + }, "com.yahoo.jdisc.http.filter.DiscFilterRequest$ThreadLocalSimpleDateFormat": { "superClass": "java.lang.ThreadLocal", "interfaces": [], @@ -2407,7 +2461,8 @@ "com.yahoo.jdisc.http.ssl.SslContextFactoryProvider": { "superClass": "java.lang.Object", "interfaces": [ - "java.lang.AutoCloseable" + "java.lang.AutoCloseable", + "com.yahoo.jdisc.http.SslProvider" ], "attributes": [ "public", @@ -2416,7 +2471,8 @@ ], "methods": [ "public abstract org.eclipse.jetty.util.ssl.SslContextFactory getInstance(java.lang.String, int)", - "public void close()" + "public void close()", + "public void configureSsl(com.yahoo.jdisc.http.SslProvider$ConnectorSsl, java.lang.String, int)" ], "fields": [] }, diff --git a/container-core/src/main/java/com/yahoo/container/handler/metrics/MetricsV2Handler.java b/container-core/src/main/java/com/yahoo/container/handler/metrics/MetricsV2Handler.java index e1a14ef541b..0a71489a605 100644 --- a/container-core/src/main/java/com/yahoo/container/handler/metrics/MetricsV2Handler.java +++ b/container-core/src/main/java/com/yahoo/container/handler/metrics/MetricsV2Handler.java @@ -1,25 +1,26 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.container.handler.metrics; -import ai.vespa.util.http.hc4.VespaHttpClientBuilder; +import ai.vespa.util.http.hc5.VespaHttpClientBuilder; import com.google.inject.Inject; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.restapi.Path; import com.yahoo.yolean.Exceptions; -import org.apache.http.client.HttpClient; -import org.apache.http.client.config.RequestConfig; -import org.apache.http.client.methods.HttpGet; -import org.apache.http.impl.client.BasicResponseHandler; -import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.hc.client5.http.classic.methods.HttpGet; +import org.apache.hc.client5.http.config.RequestConfig; +import org.apache.hc.client5.http.impl.classic.BasicHttpClientResponseHandler; +import org.apache.hc.client5.http.impl.classic.CloseableHttpClient; import java.io.IOException; import java.net.URI; import java.util.List; import java.util.Optional; import java.util.concurrent.Executor; +import java.util.logging.Level; import static com.yahoo.jdisc.Response.Status.INTERNAL_SERVER_ERROR; import static com.yahoo.jdisc.Response.Status.OK; +import static java.util.concurrent.TimeUnit.MILLISECONDS; /** * @author gjoranv @@ -33,7 +34,7 @@ public class MetricsV2Handler extends HttpHandlerBase { private static final int HTTP_SOCKET_TIMEOUT = 30000; private final String metricsProxyUri; - private final HttpClient httpClient = createHttpClient(); + private final CloseableHttpClient httpClient = createHttpClient(); @Inject public MetricsV2Handler(Executor executor, @@ -52,7 +53,7 @@ public class MetricsV2Handler extends HttpHandlerBase { private JsonResponse valuesResponse(String consumer) { try { String uri = metricsProxyUri + consumerQuery(consumer); - String metricsJson = httpClient.execute(new HttpGet(uri), new BasicResponseHandler()); + String metricsJson = httpClient.execute(new HttpGet(uri), new BasicHttpClientResponseHandler()); return new JsonResponse(OK, metricsJson); } catch (IOException e) { log.warning("Unable to retrieve metrics from " + metricsProxyUri + ": " + Exceptions.toMessageString(e)); @@ -64,9 +65,9 @@ public class MetricsV2Handler extends HttpHandlerBase { return VespaHttpClientBuilder.create() .setUserAgent("application-metrics-retriever") .setDefaultRequestConfig(RequestConfig.custom() - .setConnectTimeout(HTTP_CONNECT_TIMEOUT) - .setSocketTimeout(HTTP_SOCKET_TIMEOUT) - .build()) + .setConnectTimeout(HTTP_CONNECT_TIMEOUT, MILLISECONDS) + .setResponseTimeout(HTTP_SOCKET_TIMEOUT, MILLISECONDS) + .build()) .build(); } @@ -74,4 +75,15 @@ public class MetricsV2Handler extends HttpHandlerBase { return (consumer == null || consumer.isEmpty()) ? "" : "?consumer=" + consumer; } + @Override + public void destroy(){ + super.destroy(); + try { + httpClient.close(); + } + catch (IOException e) { + log.log(Level.WARNING, "Failed closing http client", e); + } + } + } diff --git a/container-core/src/main/java/com/yahoo/container/handler/metrics/PrometheusV1Handler.java b/container-core/src/main/java/com/yahoo/container/handler/metrics/PrometheusV1Handler.java index aa99beb3b24..6ea35138606 100644 --- a/container-core/src/main/java/com/yahoo/container/handler/metrics/PrometheusV1Handler.java +++ b/container-core/src/main/java/com/yahoo/container/handler/metrics/PrometheusV1Handler.java @@ -1,31 +1,32 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.container.handler.metrics; -import ai.vespa.util.http.hc4.VespaHttpClientBuilder; +import ai.vespa.util.http.hc5.VespaHttpClientBuilder; import com.google.inject.Inject; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.restapi.Path; import com.yahoo.restapi.StringResponse; import com.yahoo.yolean.Exceptions; -import org.apache.http.client.HttpClient; -import org.apache.http.client.config.RequestConfig; -import org.apache.http.client.methods.HttpGet; -import org.apache.http.impl.client.BasicResponseHandler; -import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.hc.client5.http.classic.methods.HttpGet; +import org.apache.hc.client5.http.config.RequestConfig; +import org.apache.hc.client5.http.impl.classic.BasicHttpClientResponseHandler; +import org.apache.hc.client5.http.impl.classic.CloseableHttpClient; import java.io.IOException; import java.net.URI; import java.util.List; import java.util.Optional; import java.util.concurrent.Executor; +import java.util.logging.Level; import static com.yahoo.container.handler.metrics.MetricsV2Handler.consumerQuery; import static com.yahoo.jdisc.Response.Status.INTERNAL_SERVER_ERROR; +import static java.util.concurrent.TimeUnit.MILLISECONDS; /** * @author Oracien */ -public class PrometheusV1Handler extends HttpHandlerBase{ +public class PrometheusV1Handler extends HttpHandlerBase { public static final String V1_PATH = "/prometheus/v1"; static final String VALUES_PATH = V1_PATH + "/values"; @@ -34,7 +35,7 @@ public class PrometheusV1Handler extends HttpHandlerBase{ private static final int HTTP_SOCKET_TIMEOUT = 30000; private final String metricsProxyUri; - private final HttpClient httpClient = createHttpClient(); + private final CloseableHttpClient httpClient = createHttpClient(); @Inject public PrometheusV1Handler(Executor executor, @@ -53,7 +54,7 @@ public class PrometheusV1Handler extends HttpHandlerBase{ private HttpResponse valuesResponse(String consumer) { try { String uri = metricsProxyUri + consumerQuery(consumer); - String prometheusText = httpClient.execute(new HttpGet(uri), new BasicResponseHandler()); + String prometheusText = httpClient.execute(new HttpGet(uri), new BasicHttpClientResponseHandler()); return new StringResponse(prometheusText); } catch (IOException e) { log.warning("Unable to retrieve metrics from " + metricsProxyUri + ": " + Exceptions.toMessageString(e)); @@ -65,9 +66,21 @@ public class PrometheusV1Handler extends HttpHandlerBase{ return VespaHttpClientBuilder.create() .setUserAgent("application-prometheus-receiver") .setDefaultRequestConfig(RequestConfig.custom() - .setConnectTimeout(HTTP_CONNECT_TIMEOUT) - .setSocketTimeout(HTTP_SOCKET_TIMEOUT) + .setConnectTimeout(HTTP_CONNECT_TIMEOUT, MILLISECONDS) + .setResponseTimeout(HTTP_SOCKET_TIMEOUT, MILLISECONDS) .build()) .build(); } + + @Override + public void destroy(){ + super.destroy(); + try { + httpClient.close(); + } + catch (IOException e) { + log.log(Level.WARNING, "Failed closing http client", e); + } + } + } diff --git a/container-core/src/main/java/com/yahoo/jdisc/http/SslProvider.java b/container-core/src/main/java/com/yahoo/jdisc/http/SslProvider.java new file mode 100644 index 00000000000..bbdba395910 --- /dev/null +++ b/container-core/src/main/java/com/yahoo/jdisc/http/SslProvider.java @@ -0,0 +1,36 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.jdisc.http; + +import javax.net.ssl.SSLContext; +import java.security.KeyStore; +import java.util.List; + +/** + * Provides SSL/TLS configuration for a server connector. + * + * @author bjorncs + */ +public interface SslProvider extends AutoCloseable { + + interface ConnectorSsl { + enum ClientAuth { DISABLED, WANT, NEED } + ConnectorSsl setSslContext(SSLContext ctx); + ConnectorSsl setClientAuth(ConnectorSsl.ClientAuth auth); + ConnectorSsl setEnabledCipherSuites(List<String> ciphers); + ConnectorSsl setEnabledProtocolVersions(List<String> versions); + ConnectorSsl setKeystore(KeyStore keystore, char[] password); + ConnectorSsl setKeystore(KeyStore keystore); + ConnectorSsl setTruststore(KeyStore truststore, char[] password); + ConnectorSsl setTruststore(KeyStore truststore); + } + + /** + * Invoked during configuration of server connector + * @param ssl provides methods to modify default SSL configuration + * @param name The connector name + * @param port The connector listen port + */ + void configureSsl(ConnectorSsl ssl, String name, int port); + + @Override default void close() {} +} diff --git a/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/ConnectorFactory.java b/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/ConnectorFactory.java index a7c5b83f6a6..b56743954f4 100644 --- a/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/ConnectorFactory.java +++ b/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/ConnectorFactory.java @@ -4,7 +4,9 @@ package com.yahoo.jdisc.http.server.jetty; import com.google.inject.Inject; import com.yahoo.jdisc.Metric; import com.yahoo.jdisc.http.ConnectorConfig; +import com.yahoo.jdisc.http.SslProvider; import com.yahoo.jdisc.http.ssl.SslContextFactoryProvider; +import com.yahoo.jdisc.http.ssl.impl.DefaultConnectorSsl; import com.yahoo.security.tls.MixedMode; import com.yahoo.security.tls.TransportSecurityUtils; import org.eclipse.jetty.alpn.server.ALPNServerConnectionFactory; @@ -41,14 +43,14 @@ public class ConnectorFactory { private static final Logger log = Logger.getLogger(ConnectorFactory.class.getName()); private final ConnectorConfig connectorConfig; - private final SslContextFactoryProvider sslContextFactoryProvider; + private final SslProvider sslProvider; @Inject public ConnectorFactory(ConnectorConfig connectorConfig, - SslContextFactoryProvider sslContextFactoryProvider) { + SslProvider sslProvider) { runtimeConnectorConfigValidation(connectorConfig); this.connectorConfig = connectorConfig; - this.sslContextFactoryProvider = sslContextFactoryProvider; + this.sslProvider = sslProvider; } // Perform extra connector config validation that can only be performed at runtime, @@ -180,12 +182,28 @@ public class ConnectorFactory { } private SslConnectionFactory newSslConnectionFactory(Metric metric, ConnectionFactory wrappedFactory) { - SslContextFactory ctxFactory = sslContextFactoryProvider.getInstance(connectorConfig.name(), connectorConfig.listenPort()); - SslConnectionFactory connectionFactory = new SslConnectionFactory(ctxFactory, wrappedFactory.getProtocol()); + SslConnectionFactory connectionFactory = new SslConnectionFactory(createSslContextFactory(), wrappedFactory.getProtocol()); connectionFactory.addBean(new SslHandshakeFailedListener(metric, connectorConfig.name(), connectorConfig.listenPort())); return connectionFactory; } + @SuppressWarnings("removal") + private SslContextFactory createSslContextFactory() { + try { + DefaultConnectorSsl ssl = new DefaultConnectorSsl(); + sslProvider.configureSsl(ssl, connectorConfig.name(), connectorConfig.listenPort()); + return ssl.createSslContextFactory(); + } catch (UnsupportedOperationException e) { + // TODO(bjorncs) Vespa 8 Remove this compatibility workaround + if (sslProvider instanceof SslContextFactoryProvider) { + return ((SslContextFactoryProvider) sslProvider) + .getInstance(connectorConfig.name(), connectorConfig.listenPort()); + } else { + throw e; + } + } + } + private ALPNServerConnectionFactory newAlpnConnectionFactory() { ALPNServerConnectionFactory factory = new ALPNServerConnectionFactory("h2", "http/1.1"); factory.setDefaultProtocol("http/1.1"); diff --git a/container-core/src/main/java/com/yahoo/jdisc/http/ssl/SslContextFactoryProvider.java b/container-core/src/main/java/com/yahoo/jdisc/http/ssl/SslContextFactoryProvider.java index 4383b511637..e786074e8d0 100644 --- a/container-core/src/main/java/com/yahoo/jdisc/http/ssl/SslContextFactoryProvider.java +++ b/container-core/src/main/java/com/yahoo/jdisc/http/ssl/SslContextFactoryProvider.java @@ -1,14 +1,17 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.jdisc.http.ssl; +import com.yahoo.jdisc.http.SslProvider; import org.eclipse.jetty.util.ssl.SslContextFactory; /** * A provider that is used to configure SSL connectors in JDisc * + * @deprecated Implement {@link SslProvider} instead * @author bjorncs */ -public interface SslContextFactoryProvider extends AutoCloseable { +@Deprecated(forRemoval = true, since = "7") +public interface SslContextFactoryProvider extends AutoCloseable, SslProvider { /** * This method is called once for each SSL connector. @@ -18,4 +21,10 @@ public interface SslContextFactoryProvider extends AutoCloseable { SslContextFactory getInstance(String containerId, int port); @Override default void close() {} + + @Override + default void configureSsl(ConnectorSsl ssl, String name, int port) { + // Signal that getInstance() should be invoked instead + throw new UnsupportedOperationException(); + } } diff --git a/container-core/src/main/java/com/yahoo/jdisc/http/ssl/impl/ConfiguredSslContextFactoryProvider.java b/container-core/src/main/java/com/yahoo/jdisc/http/ssl/impl/ConfiguredSslContextFactoryProvider.java index 8916fd7760d..05a013c036e 100644 --- a/container-core/src/main/java/com/yahoo/jdisc/http/ssl/impl/ConfiguredSslContextFactoryProvider.java +++ b/container-core/src/main/java/com/yahoo/jdisc/http/ssl/impl/ConfiguredSslContextFactoryProvider.java @@ -2,14 +2,12 @@ package com.yahoo.jdisc.http.ssl.impl; import com.yahoo.jdisc.http.ConnectorConfig; -import com.yahoo.jdisc.http.ConnectorConfig.Ssl.ClientAuth; -import com.yahoo.jdisc.http.ssl.SslContextFactoryProvider; +import com.yahoo.jdisc.http.SslProvider; import com.yahoo.security.KeyUtils; import com.yahoo.security.SslContextBuilder; import com.yahoo.security.X509CertificateUtils; import com.yahoo.security.tls.AutoReloadingX509KeyManager; import com.yahoo.security.tls.TlsContext; -import org.eclipse.jetty.util.ssl.SslContextFactory; import javax.net.ssl.SSLContext; import java.io.IOException; @@ -23,15 +21,12 @@ import java.util.ArrayList; import java.util.List; import java.util.Optional; -import static com.yahoo.jdisc.http.ssl.impl.SslContextFactoryUtils.setEnabledCipherSuites; -import static com.yahoo.jdisc.http.ssl.impl.SslContextFactoryUtils.setEnabledProtocols; - /** - * An implementation of {@link SslContextFactoryProvider} that uses the {@link ConnectorConfig} to construct a {@link SslContextFactory}. + * An implementation of {@link SslProvider} that uses the {@link ConnectorConfig} to configure SSL. * * @author bjorncs */ -public class ConfiguredSslContextFactoryProvider implements SslContextFactoryProvider { +public class ConfiguredSslContextFactoryProvider implements SslProvider { private volatile AutoReloadingX509KeyManager keyManager; private final ConnectorConfig connectorConfig; @@ -42,7 +37,7 @@ public class ConfiguredSslContextFactoryProvider implements SslContextFactoryPro } @Override - public SslContextFactory getInstance(String containerId, int port) { + public void configureSsl(ConnectorSsl ssl, String name, int port) { ConnectorConfig.Ssl sslConfig = connectorConfig.ssl(); if (!sslConfig.enabled()) throw new IllegalStateException(); @@ -62,23 +57,31 @@ public class ConfiguredSslContextFactoryProvider implements SslContextFactoryPro SSLContext sslContext = builder.build(); - SslContextFactory.Server factory = new SslContextFactory.Server(); - factory.setSslContext(sslContext); - - factory.setNeedClientAuth(sslConfig.clientAuth() == ClientAuth.Enum.NEED_AUTH); - factory.setWantClientAuth(sslConfig.clientAuth() == ClientAuth.Enum.WANT_AUTH); + ssl.setSslContext(sslContext); + + switch (sslConfig.clientAuth()) { + case NEED_AUTH: + ssl.setClientAuth(ConnectorSsl.ClientAuth.NEED); + break; + case WANT_AUTH: + ssl.setClientAuth(ConnectorSsl.ClientAuth.WANT); + break; + case DISABLED: + ssl.setClientAuth(ConnectorSsl.ClientAuth.DISABLED); + break; + default: + throw new IllegalArgumentException(sslConfig.clientAuth().toString()); + } List<String> protocols = !sslConfig.enabledProtocols().isEmpty() ? sslConfig.enabledProtocols() : new ArrayList<>(TlsContext.getAllowedProtocols(sslContext)); - setEnabledProtocols(factory, sslContext, protocols); + ssl.setEnabledProtocolVersions(protocols); List<String> ciphers = !sslConfig.enabledCipherSuites().isEmpty() ? sslConfig.enabledCipherSuites() : new ArrayList<>(TlsContext.getAllowedCipherSuites(sslContext)); - setEnabledCipherSuites(factory, sslContext, ciphers); - - return factory; + ssl.setEnabledCipherSuites(ciphers); } @Override diff --git a/container-core/src/main/java/com/yahoo/jdisc/http/ssl/impl/DefaultConnectorSsl.java b/container-core/src/main/java/com/yahoo/jdisc/http/ssl/impl/DefaultConnectorSsl.java new file mode 100644 index 00000000000..65f877a6029 --- /dev/null +++ b/container-core/src/main/java/com/yahoo/jdisc/http/ssl/impl/DefaultConnectorSsl.java @@ -0,0 +1,94 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.jdisc.http.ssl.impl; + +import com.yahoo.jdisc.http.SslProvider; +import org.eclipse.jetty.util.ssl.SslContextFactory; + +import javax.net.ssl.SSLContext; +import java.security.KeyStore; +import java.util.List; + +/** + * Default implementation of {@link SslProvider} backed by {@link SslContextFactory.Server} + * + * @author bjorncs + */ +public class DefaultConnectorSsl implements SslProvider.ConnectorSsl { + + private SSLContext sslContext; + private ClientAuth clientAuth; + private List<String> cipherSuites = List.of(); + private List<String> protocolVersions = List.of(); + private KeyStore keystore; + private char[] keystorePassword; + private KeyStore truststore; + private char[] truststorePassword; + + @Override + public SslProvider.ConnectorSsl setSslContext(SSLContext ctx) { + this.sslContext = ctx; return this; + } + + @Override + public SslProvider.ConnectorSsl setClientAuth(SslProvider.ConnectorSsl.ClientAuth auth) { + this.clientAuth = auth; return this; + } + + @Override + public SslProvider.ConnectorSsl setEnabledCipherSuites(List<String> ciphers) { + this.cipherSuites = ciphers; return this; + } + + @Override + public SslProvider.ConnectorSsl setEnabledProtocolVersions(List<String> versions) { + this.protocolVersions = versions; return this; + } + + @Override + public SslProvider.ConnectorSsl setKeystore(KeyStore keystore, char[] password) { + this.keystore = keystore; this.keystorePassword = password; return this; + } + + @Override + public SslProvider.ConnectorSsl setKeystore(KeyStore keystore) { + this.keystore = keystore; return this; + } + + @Override + public SslProvider.ConnectorSsl setTruststore(KeyStore truststore, char[] password) { + this.truststore = truststore; this.truststorePassword = password; return this; + } + + @Override + public SslProvider.ConnectorSsl setTruststore(KeyStore truststore) { + this.truststore = truststore; return this; + } + + public SslContextFactory.Server createSslContextFactory() { + SslContextFactory.Server ssl = new SslContextFactory.Server(); + if (sslContext != null) ssl.setSslContext(sslContext); + if (keystore != null) ssl.setKeyStore(keystore); + if (keystorePassword != null) ssl.setKeyStorePassword(new String(keystorePassword)); + if (truststore != null) ssl.setTrustStore(truststore); + if (truststorePassword != null) ssl.setTrustStorePassword(new String(truststorePassword)); + switch (clientAuth) { + case DISABLED: + ssl.setWantClientAuth(false); + ssl.setNeedClientAuth(false); + break; + case NEED: + ssl.setWantClientAuth(false); + ssl.setNeedClientAuth(true); + break; + case WANT: + ssl.setWantClientAuth(true); + ssl.setNeedClientAuth(false); + break; + default: + throw new IllegalArgumentException(clientAuth.name()); + } + if (!cipherSuites.isEmpty()) SslContextFactoryUtils.setEnabledCipherSuites(ssl, sslContext, cipherSuites); + if (!protocolVersions.isEmpty()) SslContextFactoryUtils.setEnabledProtocols(ssl, sslContext, protocolVersions); + return ssl; + } +} diff --git a/container-core/src/main/java/com/yahoo/jdisc/http/ssl/impl/DefaultSslContextFactoryProvider.java b/container-core/src/main/java/com/yahoo/jdisc/http/ssl/impl/DefaultSslContextFactoryProvider.java index c3c99b71c46..c8cf5195c4c 100644 --- a/container-core/src/main/java/com/yahoo/jdisc/http/ssl/impl/DefaultSslContextFactoryProvider.java +++ b/container-core/src/main/java/com/yahoo/jdisc/http/ssl/impl/DefaultSslContextFactoryProvider.java @@ -4,23 +4,22 @@ package com.yahoo.jdisc.http.ssl.impl; import com.google.inject.Inject; import com.yahoo.component.AbstractComponent; import com.yahoo.jdisc.http.ConnectorConfig; -import com.yahoo.jdisc.http.ssl.SslContextFactoryProvider; +import com.yahoo.jdisc.http.SslProvider; import com.yahoo.security.tls.ConfigFileBasedTlsContext; import com.yahoo.security.tls.PeerAuthentication; import com.yahoo.security.tls.TlsContext; import com.yahoo.security.tls.TransportSecurityUtils; -import org.eclipse.jetty.util.ssl.SslContextFactory; import java.nio.file.Path; /** - * The default implementation of {@link SslContextFactoryProvider} to be injected into connectors without explicit ssl configuration. + * The default implementation of {@link SslProvider} to be injected into connectors without explicit ssl configuration. * * @author bjorncs */ -public class DefaultSslContextFactoryProvider extends AbstractComponent implements SslContextFactoryProvider { +public class DefaultSslContextFactoryProvider extends AbstractComponent implements SslProvider { - private final SslContextFactoryProvider instance; + private final SslProvider instance; @Inject public DefaultSslContextFactoryProvider(ConnectorConfig connectorConfig) { @@ -29,7 +28,7 @@ public class DefaultSslContextFactoryProvider extends AbstractComponent implemen .orElseGet(ThrowingSslContextFactoryProvider::new); } - private static SslContextFactoryProvider createTlsContextBasedProvider(ConnectorConfig connectorConfig, Path configFile) { + private static SslProvider createTlsContextBasedProvider(ConnectorConfig connectorConfig, Path configFile) { return new StaticTlsContextBasedProvider( new ConfigFileBasedTlsContext( configFile, TransportSecurityUtils.getInsecureAuthorizationMode(), getPeerAuthenticationMode(connectorConfig))); @@ -46,8 +45,8 @@ public class DefaultSslContextFactoryProvider extends AbstractComponent implemen } @Override - public SslContextFactory getInstance(String containerId, int port) { - return instance.getInstance(containerId, port); + public void configureSsl(ConnectorSsl ssl, String name, int port) { + instance.configureSsl(ssl, name, port); } @Override @@ -55,9 +54,9 @@ public class DefaultSslContextFactoryProvider extends AbstractComponent implemen instance.close(); } - private static class ThrowingSslContextFactoryProvider implements SslContextFactoryProvider { + private static class ThrowingSslContextFactoryProvider implements SslProvider { @Override - public SslContextFactory getInstance(String containerId, int port) { + public void configureSsl(ConnectorSsl ssl, String name, int port) { throw new UnsupportedOperationException(); } } diff --git a/container-core/src/main/java/com/yahoo/jdisc/http/ssl/impl/SslContextFactoryUtils.java b/container-core/src/main/java/com/yahoo/jdisc/http/ssl/impl/SslContextFactoryUtils.java index 07c599aa229..e7c9e4f0bee 100644 --- a/container-core/src/main/java/com/yahoo/jdisc/http/ssl/impl/SslContextFactoryUtils.java +++ b/container-core/src/main/java/com/yahoo/jdisc/http/ssl/impl/SslContextFactoryUtils.java @@ -4,6 +4,8 @@ package com.yahoo.jdisc.http.ssl.impl; import org.eclipse.jetty.util.ssl.SslContextFactory; import javax.net.ssl.SSLContext; +import javax.net.ssl.SSLParameters; +import java.security.NoSuchAlgorithmException; import java.util.Arrays; import java.util.List; @@ -12,14 +14,14 @@ import java.util.List; */ class SslContextFactoryUtils { - static void setEnabledCipherSuites(SslContextFactory factory, SSLContext sslContext, List<String> enabledCiphers) { - String[] supportedCiphers = sslContext.getSupportedSSLParameters().getCipherSuites(); + static void setEnabledCipherSuites(SslContextFactory factory, SSLContext sslContextOrNull, List<String> enabledCiphers) { + String[] supportedCiphers = supportedSslParams(sslContextOrNull).getCipherSuites(); factory.setIncludeCipherSuites(enabledCiphers.toArray(String[]::new)); factory.setExcludeCipherSuites(createExclusionList(enabledCiphers, supportedCiphers)); } - static void setEnabledProtocols(SslContextFactory factory, SSLContext sslContext, List<String> enabledProtocols) { - String[] supportedProtocols = sslContext.getSupportedSSLParameters().getProtocols(); + static void setEnabledProtocols(SslContextFactory factory, SSLContext sslContextOrNull, List<String> enabledProtocols) { + String[] supportedProtocols = supportedSslParams(sslContextOrNull).getProtocols(); factory.setIncludeProtocols(enabledProtocols.toArray(String[]::new)); factory.setExcludeProtocols(createExclusionList(enabledProtocols, supportedProtocols)); } @@ -29,4 +31,14 @@ class SslContextFactoryUtils { .filter(supportedValue -> !enabledValues.contains(supportedValue)) .toArray(String[]::new); } + + private static SSLParameters supportedSslParams(SSLContext ctx) { + try { + return ctx != null + ? ctx.getSupportedSSLParameters() + : SSLContext.getDefault().getSupportedSSLParameters(); + } catch (NoSuchAlgorithmException e) { + throw new IllegalStateException(e); + } + } } diff --git a/container-core/src/main/java/com/yahoo/jdisc/http/ssl/impl/TlsContextBasedProvider.java b/container-core/src/main/java/com/yahoo/jdisc/http/ssl/impl/TlsContextBasedProvider.java index 3d9e0bf39d3..712388a305e 100644 --- a/container-core/src/main/java/com/yahoo/jdisc/http/ssl/impl/TlsContextBasedProvider.java +++ b/container-core/src/main/java/com/yahoo/jdisc/http/ssl/impl/TlsContextBasedProvider.java @@ -2,41 +2,34 @@ package com.yahoo.jdisc.http.ssl.impl; import com.yahoo.component.AbstractComponent; -import com.yahoo.jdisc.http.ssl.SslContextFactoryProvider; +import com.yahoo.jdisc.http.SslProvider; import com.yahoo.security.tls.TlsContext; -import org.eclipse.jetty.util.ssl.SslContextFactory; -import javax.net.ssl.SSLContext; import javax.net.ssl.SSLParameters; import java.util.List; -import static com.yahoo.jdisc.http.ssl.impl.SslContextFactoryUtils.setEnabledCipherSuites; -import static com.yahoo.jdisc.http.ssl.impl.SslContextFactoryUtils.setEnabledProtocols; - /** - * A {@link SslContextFactoryProvider} that creates {@link SslContextFactory} instances from {@link TlsContext} instances. + * A {@link SslProvider} that configures SSL from {@link TlsContext} instances. * * @author bjorncs */ -public abstract class TlsContextBasedProvider extends AbstractComponent implements SslContextFactoryProvider { +public abstract class TlsContextBasedProvider extends AbstractComponent implements SslProvider { protected abstract TlsContext getTlsContext(String containerId, int port); @Override - public final SslContextFactory getInstance(String containerId, int port) { - TlsContext tlsContext = getTlsContext(containerId, port); - SSLContext sslContext = tlsContext.context(); + public void configureSsl(ConnectorSsl ssl, String name, int port) { + TlsContext tlsContext = getTlsContext(name, port); SSLParameters parameters = tlsContext.parameters(); - - SslContextFactory.Server sslContextFactory = new SslContextFactory.Server(); - sslContextFactory.setSslContext(sslContext); - - sslContextFactory.setNeedClientAuth(parameters.getNeedClientAuth()); - sslContextFactory.setWantClientAuth(parameters.getWantClientAuth()); - - setEnabledProtocols(sslContextFactory, sslContext, List.of(parameters.getProtocols())); - setEnabledCipherSuites(sslContextFactory, sslContext, List.of(parameters.getCipherSuites())); - - return sslContextFactory; + ssl.setSslContext(tlsContext.context()); + ssl.setEnabledProtocolVersions(List.of(parameters.getProtocols())); + ssl.setEnabledCipherSuites(List.of(parameters.getCipherSuites())); + if (parameters.getNeedClientAuth()) { + ssl.setClientAuth(ConnectorSsl.ClientAuth.NEED); + } else if (parameters.getWantClientAuth()) { + ssl.setClientAuth(ConnectorSsl.ClientAuth.WANT); + } else { + ssl.setClientAuth(ConnectorSsl.ClientAuth.DISABLED); + } } } diff --git a/container-core/src/test/java/com/yahoo/jdisc/http/ssl/impl/TlsContextBasedProviderTest.java b/container-core/src/test/java/com/yahoo/jdisc/http/ssl/impl/TlsContextBasedProviderTest.java index 21597ceefcf..fce4d6ee74e 100644 --- a/container-core/src/test/java/com/yahoo/jdisc/http/ssl/impl/TlsContextBasedProviderTest.java +++ b/container-core/src/test/java/com/yahoo/jdisc/http/ssl/impl/TlsContextBasedProviderTest.java @@ -9,7 +9,6 @@ import com.yahoo.security.tls.HostnameVerification; import com.yahoo.security.tls.PeerAuthentication; import com.yahoo.security.tls.TlsContext; import com.yahoo.security.tls.policy.AuthorizedPeers; -import org.eclipse.jetty.util.ssl.SslContextFactory; import org.junit.Test; import javax.security.auth.x500.X500Principal; @@ -24,7 +23,6 @@ import java.util.Set; import static com.yahoo.security.KeyAlgorithm.EC; import static com.yahoo.security.SignatureAlgorithm.SHA256_WITH_ECDSA; import static org.junit.Assert.assertArrayEquals; -import static org.junit.Assert.assertNotNull; /** * @author bjorncs @@ -35,9 +33,9 @@ public class TlsContextBasedProviderTest { public void creates_sslcontextfactory_from_tlscontext() { TlsContext tlsContext = createTlsContext(); var provider = new SimpleTlsContextBasedProvider(tlsContext); - SslContextFactory sslContextFactory = provider.getInstance("dummyContainerId", 8080); - assertNotNull(sslContextFactory); - assertArrayEquals(tlsContext.parameters().getCipherSuites(), sslContextFactory.getIncludeCipherSuites()); + DefaultConnectorSsl ssl = new DefaultConnectorSsl(); + provider.configureSsl(ssl, "dummyContainerId", 8080); + assertArrayEquals(tlsContext.parameters().getCipherSuites(), ssl.createSslContextFactory().getIncludeCipherSuites()); } private static TlsContext createTlsContext() { diff --git a/container-test/pom.xml b/container-test/pom.xml index 9b5b7ba17a9..961d827a390 100644 --- a/container-test/pom.xml +++ b/container-test/pom.xml @@ -23,10 +23,6 @@ <version>${project.version}</version> <exclusions> <exclusion> - <groupId>org.apache.httpcomponents.client5</groupId> - <artifactId>httpclient5</artifactId> - </exclusion> - <exclusion> <groupId>biz.aQute.bnd</groupId> <artifactId>*</artifactId> </exclusion> diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackageValidator.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackageValidator.java index ccad4fe92ad..8aef09d1a40 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackageValidator.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackageValidator.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.application.pkg; +import com.yahoo.component.Version; import com.yahoo.config.application.api.DeploymentInstanceSpec; import com.yahoo.config.application.api.DeploymentSpec; import com.yahoo.config.application.api.Endpoint; @@ -20,6 +21,7 @@ import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.application.EndpointId; import com.yahoo.vespa.hosted.controller.deployment.DeploymentSteps; +import com.yahoo.vespa.hosted.controller.versions.VespaVersion; import java.time.Instant; import java.util.ArrayList; @@ -59,6 +61,22 @@ public class ApplicationPackageValidator { validateEndpointChange(application, applicationPackage, instant); validateCompactedEndpoint(applicationPackage); validateSecurityClientsPem(applicationPackage); + validateDeprecatedElements(applicationPackage); + } + + /** Verify that deployment spec does not use elements deprecated on a major version older than wanted major version */ + private void validateDeprecatedElements(ApplicationPackage applicationPackage) { + int wantedMajor = applicationPackage.compileVersion().map(Version::getMajor) + .or(() -> applicationPackage.deploymentSpec().majorVersion()) + .or(() -> controller.readVersionStatus().controllerVersion() + .map(VespaVersion::versionNumber) + .map(Version::getMajor)) + .orElseThrow(() -> new IllegalArgumentException("Could not determine wanted major version")); + for (var deprecatedElement : applicationPackage.deploymentSpec().deprecatedElements()) { + if (applicationPackage.compileVersion().isEmpty()) continue; + if (deprecatedElement.majorVersion() >= wantedMajor) continue; + throw new IllegalArgumentException(deprecatedElement.humanReadableString()); + } } /** Verify that we have the security/clients.pem file for public systems */ diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java index f4f50de59d7..75abf94bf0e 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java @@ -1195,4 +1195,20 @@ public class ControllerTest { assertEquals(cloudAccount, tester.controllerTester().configServer().cloudAccount(context.deploymentIdIn(zone)).get().value()); } + @Test + public void testSubmitWithElementDeprecatedOnPreviousMajor() { + DeploymentContext context = tester.newDeploymentContext(); + var applicationPackage = new ApplicationPackageBuilder() + .compileVersion(Version.fromString("8.1")) + .region("us-west-1") + .globalServiceId("qrs") + .build(); + try { + context.submit(applicationPackage).deploy(); + fail("Expected exception"); + } catch (IllegalArgumentException e) { + assertTrue(e.getMessage().contains("Element 'prod' contains attribute 'global-service-id' deprecated since major version 7")); + } + } + } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ApplicationPackageBuilder.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ApplicationPackageBuilder.java index 5d1a677bf51..490731ca06a 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ApplicationPackageBuilder.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ApplicationPackageBuilder.java @@ -147,7 +147,10 @@ public class ApplicationPackageBuilder { } public ApplicationPackageBuilder region(String regionName) { - return region(RegionName.from(regionName), true); + prodBody.append(" <region>") + .append(regionName) + .append("</region>\n"); + return this; } public ApplicationPackageBuilder region(RegionName regionName, boolean active) { diff --git a/searchlib/src/vespa/searchlib/attribute/multistringattribute.h b/searchlib/src/vespa/searchlib/attribute/multistringattribute.h index 4c3eeeed232..308ed6f82a3 100644 --- a/searchlib/src/vespa/searchlib/attribute/multistringattribute.h +++ b/searchlib/src/vespa/searchlib/attribute/multistringattribute.h @@ -44,9 +44,8 @@ protected: using generation_t = StringAttribute::generation_t; public: - MultiValueStringAttributeT(const vespalib::string & name, const AttributeVector::Config & c = - AttributeVector::Config(AttributeVector::BasicType::STRING, - attribute::CollectionType::ARRAY)); + MultiValueStringAttributeT(const vespalib::string & name, const AttributeVector::Config & c); + MultiValueStringAttributeT(const vespalib::string & name); ~MultiValueStringAttributeT(); void freezeEnumDictionary() override; diff --git a/searchlib/src/vespa/searchlib/attribute/multistringattribute.hpp b/searchlib/src/vespa/searchlib/attribute/multistringattribute.hpp index c3f8f99b4ab..ec67cafe90b 100644 --- a/searchlib/src/vespa/searchlib/attribute/multistringattribute.hpp +++ b/searchlib/src/vespa/searchlib/attribute/multistringattribute.hpp @@ -10,6 +10,7 @@ #include "multi_string_enum_hint_search_context.h" #include <vespa/vespalib/text/utf8.h> #include <vespa/vespalib/text/lowercase.h> +#include <vespa/searchcommon/attribute/config.h> #include <vespa/searchlib/util/bufferwriter.h> #include <vespa/vespalib/util/regexp.h> #include <vespa/vespalib/util/stash.h> @@ -28,6 +29,11 @@ MultiValueStringAttributeT(const vespalib::string &name, { } template <typename B, typename M> +MultiValueStringAttributeT<B, M>::MultiValueStringAttributeT(const vespalib::string &name) + : MultiValueStringAttributeT<B, M>(name, AttributeVector::Config(AttributeVector::BasicType::STRING, attribute::CollectionType::ARRAY)) +{ } + +template <typename B, typename M> MultiValueStringAttributeT<B, M>::~MultiValueStringAttributeT() = default; diff --git a/searchlib/src/vespa/searchlib/attribute/multistringpostattribute.h b/searchlib/src/vespa/searchlib/attribute/multistringpostattribute.h index b748dcf8f61..031d90a88ed 100644 --- a/searchlib/src/vespa/searchlib/attribute/multistringpostattribute.h +++ b/searchlib/src/vespa/searchlib/attribute/multistringpostattribute.h @@ -72,9 +72,8 @@ public: using Dictionary = EnumPostingTree; using PostingList = typename PostingParent::PostingList; - MultiValueStringPostingAttributeT(const vespalib::string & name, const AttributeVector::Config & c = - AttributeVector::Config(AttributeVector::BasicType::STRING, - attribute::CollectionType::ARRAY)); + MultiValueStringPostingAttributeT(const vespalib::string & name, const AttributeVector::Config & c); + MultiValueStringPostingAttributeT(const vespalib::string & name); ~MultiValueStringPostingAttributeT(); void removeOldGenerations(generation_t firstUsed) override; diff --git a/searchlib/src/vespa/searchlib/attribute/multistringpostattribute.hpp b/searchlib/src/vespa/searchlib/attribute/multistringpostattribute.hpp index d3cd338cacb..421e727761f 100644 --- a/searchlib/src/vespa/searchlib/attribute/multistringpostattribute.hpp +++ b/searchlib/src/vespa/searchlib/attribute/multistringpostattribute.hpp @@ -19,6 +19,12 @@ MultiValueStringPostingAttributeT<B, T>::MultiValueStringPostingAttributeT(const } template <typename B, typename T> +MultiValueStringPostingAttributeT<B, T>::MultiValueStringPostingAttributeT(const vespalib::string & name) + : MultiValueStringPostingAttributeT<B, T>(name, AttributeVector::Config(AttributeVector::BasicType::STRING, attribute::CollectionType::ARRAY)) +{ +} + +template <typename B, typename T> MultiValueStringPostingAttributeT<B, T>::~MultiValueStringPostingAttributeT() { this->disableFreeLists(); diff --git a/searchlib/src/vespa/searchlib/attribute/singlestringattribute.h b/searchlib/src/vespa/searchlib/attribute/singlestringattribute.h index e9bde76d4ec..ad522627a59 100644 --- a/searchlib/src/vespa/searchlib/attribute/singlestringattribute.h +++ b/searchlib/src/vespa/searchlib/attribute/singlestringattribute.h @@ -32,8 +32,8 @@ protected: using generation_t = StringAttribute::generation_t; public: - SingleValueStringAttributeT(const vespalib::string & name, const AttributeVector::Config & c = - AttributeVector::Config(AttributeVector::BasicType::STRING)); + SingleValueStringAttributeT(const vespalib::string & name, const AttributeVector::Config & c); + SingleValueStringAttributeT(const vespalib::string & name); ~SingleValueStringAttributeT(); void freezeEnumDictionary() override; diff --git a/searchlib/src/vespa/searchlib/attribute/singlestringattribute.hpp b/searchlib/src/vespa/searchlib/attribute/singlestringattribute.hpp index 50ea4ef42b6..64abbf86108 100644 --- a/searchlib/src/vespa/searchlib/attribute/singlestringattribute.hpp +++ b/searchlib/src/vespa/searchlib/attribute/singlestringattribute.hpp @@ -9,6 +9,7 @@ #include "single_string_enum_hint_search_context.h" #include <vespa/vespalib/text/utf8.h> #include <vespa/vespalib/text/lowercase.h> +#include <vespa/searchcommon/attribute/config.h> #include <vespa/searchlib/util/bufferwriter.h> #include <vespa/vespalib/util/regexp.h> #include <vespa/searchlib/query/query_term_ucs4.h> @@ -26,6 +27,11 @@ SingleValueStringAttributeT(const vespalib::string &name, { } template <typename B> +SingleValueStringAttributeT<B>::SingleValueStringAttributeT(const vespalib::string &name) + : SingleValueStringAttributeT<B>(name, AttributeVector::Config(AttributeVector::BasicType::STRING)) +{ } + +template <typename B> SingleValueStringAttributeT<B>::~SingleValueStringAttributeT() = default; template <typename B> diff --git a/searchlib/src/vespa/searchlib/attribute/singlestringpostattribute.h b/searchlib/src/vespa/searchlib/attribute/singlestringpostattribute.h index 2643f11eaf3..30549f3048a 100644 --- a/searchlib/src/vespa/searchlib/attribute/singlestringpostattribute.h +++ b/searchlib/src/vespa/searchlib/attribute/singlestringpostattribute.h @@ -67,8 +67,8 @@ private: void applyValueChanges(EnumStoreBatchUpdater& updater) override; public: - SingleValueStringPostingAttributeT(const vespalib::string & name, const AttributeVector::Config & c = - AttributeVector::Config(AttributeVector::BasicType::STRING)); + SingleValueStringPostingAttributeT(const vespalib::string & name, const AttributeVector::Config & c); + SingleValueStringPostingAttributeT(const vespalib::string & name); ~SingleValueStringPostingAttributeT(); void removeOldGenerations(generation_t firstUsed) override; diff --git a/searchlib/src/vespa/searchlib/attribute/singlestringpostattribute.hpp b/searchlib/src/vespa/searchlib/attribute/singlestringpostattribute.hpp index ddbc24d2b75..f340d9f70c3 100644 --- a/searchlib/src/vespa/searchlib/attribute/singlestringpostattribute.hpp +++ b/searchlib/src/vespa/searchlib/attribute/singlestringpostattribute.hpp @@ -4,6 +4,7 @@ #include "singlestringpostattribute.h" #include "single_string_enum_search_context.h" +#include <vespa/searchcommon/attribute/config.h> #include <vespa/searchlib/query/query_term_ucs4.h> namespace search { @@ -17,6 +18,12 @@ SingleValueStringPostingAttributeT<B>::SingleValueStringPostingAttributeT(const } template <typename B> +SingleValueStringPostingAttributeT<B>::SingleValueStringPostingAttributeT(const vespalib::string & name) + : SingleValueStringPostingAttributeT<B>(name, AttributeVector::Config(AttributeVector::BasicType::STRING)) +{ +} + +template <typename B> SingleValueStringPostingAttributeT<B>::~SingleValueStringPostingAttributeT() { this->disableFreeLists(); diff --git a/searchlib/src/vespa/searchlib/docstore/idocumentstore.h b/searchlib/src/vespa/searchlib/docstore/idocumentstore.h index 08983efb0da..f99302538e6 100644 --- a/searchlib/src/vespa/searchlib/docstore/idocumentstore.h +++ b/searchlib/src/vespa/searchlib/docstore/idocumentstore.h @@ -13,7 +13,7 @@ namespace document { } namespace vespalib { -class CacheStats; +struct CacheStats; class nbostream; } diff --git a/tsan-suppressions.txt b/tsan-suppressions.txt index 00cd7fe9a3a..5b364bcc497 100644 --- a/tsan-suppressions.txt +++ b/tsan-suppressions.txt @@ -34,6 +34,8 @@ deadlock:proton::ImportedAttributesContext::getAttributeStableEnum race:llvm::sys::MutexImpl race:llvm::sys::SmartMutex<false>::lock race:llvm::sys::SmartMutex<true>::lock +race:llvm::sys::DynamicLibrary::getPermanentLibrary +race:(anonymous namespace)::GDBJITRegistrationListener::notifyObjectLoaded # This is external code that we do not control. Complains that mutex # creation and locking is not strictly enough ordered. diff --git a/vespalib/src/vespa/vespalib/net/tls/crypto_codec.h b/vespalib/src/vespa/vespalib/net/tls/crypto_codec.h index 86ccaf3eb64..fc729d7dd6a 100644 --- a/vespalib/src/vespa/vespalib/net/tls/crypto_codec.h +++ b/vespalib/src/vespa/vespalib/net/tls/crypto_codec.h @@ -53,7 +53,7 @@ struct DecodeResult { }; struct TlsContext; -class PeerCredentials; +struct PeerCredentials; class AssumedRoles; // TODO move to different namespace, not dependent on TLS? diff --git a/vespamalloc/src/vespamalloc/malloc/load_as_huge.cpp b/vespamalloc/src/vespamalloc/malloc/load_as_huge.cpp index a5d595a8486..8494689ba85 100644 --- a/vespamalloc/src/vespamalloc/malloc/load_as_huge.cpp +++ b/vespamalloc/src/vespamalloc/malloc/load_as_huge.cpp @@ -29,8 +29,13 @@ mmap_huge(size_t sz) { size_t round_huge_down(size_t v) { return v & ~(HUGEPAGE_SIZE - 1); } size_t round_huge_up(size_t v) { return round_huge_down(v + (HUGEPAGE_SIZE - 1)); } +#ifdef __clang__ +void +non_optimized_non_inlined_memcpy(void *dest_in, const void *src_in, size_t n) __attribute__((noinline, optnone)) ; +#else void non_optimized_non_inlined_memcpy(void *dest_in, const void *src_in, size_t n) __attribute__((noinline, optimize(1))) ; +#endif // Simple memcpy replacement to avoid calling code in other dso. void |