diff options
40 files changed, 379 insertions, 91 deletions
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/SystemFlagsDeployResult.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/SystemFlagsDeployResult.java index 2994c4a83be..bc00090dc88 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/SystemFlagsDeployResult.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/SystemFlagsDeployResult.java @@ -215,7 +215,7 @@ class SystemFlagsDeployResult { } static OperationError deleteFailed(String message, FlagsTarget target, FlagId id) { - return new OperationError(message, Set.of(target), OperationType.UPDATE, id, null); + return new OperationError(message, Set.of(target), OperationType.DELETE, id, null); } String message() { return message; } diff --git a/http-utils/pom.xml b/http-utils/pom.xml index aea402aef87..0d340922042 100644 --- a/http-utils/pom.xml +++ b/http-utils/pom.xml @@ -11,6 +11,13 @@ <artifactId>http-utils</artifactId> <packaging>jar</packaging> <version>7-SNAPSHOT</version> + + <properties> + <!-- vespa-http-client targets jdk8 and uses this library --> + <!-- TODO remove once vespa-http-client no longer builds against jdk8 --> + <maven.compiler.release>8</maven.compiler.release> + </properties> + <dependencies> <!-- provided --> <dependency> @@ -49,4 +56,26 @@ <scope>test</scope> </dependency> </dependencies> + + <build> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-compiler-plugin</artifactId> + <configuration> + <jdkToolchain> + <version>${java.version}</version> + </jdkToolchain> + <source>${java.version}</source> + <target>${java.version}</target> + <showDeprecation>true</showDeprecation> + <compilerArgs> + <arg>-Xlint:all</arg> + <arg>-Xlint:-serial</arg> + <arg>-Werror</arg> + </compilerArgs> + </configuration> + </plugin> + </plugins> + </build> </project> diff --git a/http-utils/src/main/java/ai/vespa/util/http/VespaHttpClientBuilder.java b/http-utils/src/main/java/ai/vespa/util/http/VespaHttpClientBuilder.java index 529cfdc2aff..a408b0d79ae 100644 --- a/http-utils/src/main/java/ai/vespa/util/http/VespaHttpClientBuilder.java +++ b/http-utils/src/main/java/ai/vespa/util/http/VespaHttpClientBuilder.java @@ -71,7 +71,7 @@ public class VespaHttpClientBuilder { } private static HttpClientBuilder createBuilder(ConnectionManagerFactory connectionManagerFactory) { - var builder = HttpClientBuilder.create(); + HttpClientBuilder builder = HttpClientBuilder.create(); addSslSocketFactory(builder, connectionManagerFactory); addHttpsRewritingRoutePlanner(builder); return builder; diff --git a/http-utils/src/test/java/ai/vespa/util/http/VespaHttpClientBuilderTest.java b/http-utils/src/test/java/ai/vespa/util/http/VespaHttpClientBuilderTest.java index 85ee0913c58..b9a8fd748d6 100644 --- a/http-utils/src/test/java/ai/vespa/util/http/VespaHttpClientBuilderTest.java +++ b/http-utils/src/test/java/ai/vespa/util/http/VespaHttpClientBuilderTest.java @@ -6,6 +6,7 @@ import org.apache.http.HttpHost; import org.apache.http.HttpRequest; import org.apache.http.client.protocol.HttpClientContext; import org.apache.http.conn.routing.HttpRoute; +import org.apache.http.conn.routing.HttpRoutePlanner; import org.junit.Test; import static org.junit.Assert.assertEquals; @@ -32,7 +33,7 @@ public class VespaHttpClientBuilderTest { } private static void verifyProcessedUriMatchesExpectedOutput(String inputHostString, String expectedHostString) throws HttpException { - var routePlanner = new VespaHttpClientBuilder.HttpToHttpsRoutePlanner(); + HttpRoutePlanner routePlanner = new VespaHttpClientBuilder.HttpToHttpsRoutePlanner(); HttpRoute newRoute = routePlanner.determineRoute(HttpHost.create(inputHostString), mock(HttpRequest.class), new HttpClientContext()); HttpHost target = newRoute.getTargetHost(); assertEquals(expectedHostString, target.toURI()); diff --git a/searchcore/src/vespa/searchcore/proton/docsummary/documentstoreadapter.cpp b/searchcore/src/vespa/searchcore/proton/docsummary/documentstoreadapter.cpp index c65d18a590f..080aee88f3f 100644 --- a/searchcore/src/vespa/searchcore/proton/docsummary/documentstoreadapter.cpp +++ b/searchcore/src/vespa/searchcore/proton/docsummary/documentstoreadapter.cpp @@ -183,7 +183,7 @@ DocumentStoreAdapter::getMappedDocsum(uint32_t docId) LOG(warning, "Error while getting the docsum blob for docId %u. Returning empty docsum", docId); return DocsumStoreValue(); } - return DocsumStoreValue(buf, buflen); + return DocsumStoreValue(buf, buflen, std::move(document)); } } // namespace proton diff --git a/searchsummary/src/tests/docsumformat/docsum-pack.cpp b/searchsummary/src/tests/docsumformat/docsum-pack.cpp index 18b38db3fa1..c88513920ca 100644 --- a/searchsummary/src/tests/docsumformat/docsum-pack.cpp +++ b/searchsummary/src/tests/docsumformat/docsum-pack.cpp @@ -1,7 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/searchlib/util/rawbuf.h> -#include <vespa/searchsummary/docsummary/urlresult.h> +#include <vespa/searchsummary/docsummary/general_result.h> #include <vespa/searchsummary/docsummary/resultconfig.h> #include <vespa/searchsummary/docsummary/resultpacker.h> #include <vespa/fastos/app.h> diff --git a/searchsummary/src/tests/docsummary/matched_elements_filter/matched_elements_filter_test.cpp b/searchsummary/src/tests/docsummary/matched_elements_filter/matched_elements_filter_test.cpp index bbcab709e42..9019a212f3f 100644 --- a/searchsummary/src/tests/docsummary/matched_elements_filter/matched_elements_filter_test.cpp +++ b/searchsummary/src/tests/docsummary/matched_elements_filter/matched_elements_filter_test.cpp @@ -67,6 +67,7 @@ class DocsumStore { private: ResultConfig _config; ResultPacker _packer; + DocumentType _doc_type; StructDataType::UP _elem_type; ArrayDataType _array_type; MapDataType _map_type; @@ -91,10 +92,14 @@ public: DocsumStore() : _config(), _packer(&_config), + _doc_type("test"), _elem_type(make_struct_elem_type()), _array_type(*_elem_type), _map_type(*DataType::STRING, *_elem_type) { + _doc_type.addField(Field("array_in_doc", _array_type, true)); + _doc_type.addField(Field("map_in_doc", _map_type, true)); + auto* result_class = _config.AddResultClass("test", class_id); EXPECT_TRUE(result_class->AddConfigEntry("array", ResType::RES_JSONSTRING)); EXPECT_TRUE(result_class->AddConfigEntry("map", ResType::RES_JSONSTRING)); @@ -106,12 +111,14 @@ public: const ResultClass* get_class() const { return _config.LookupResultClass(class_id); } search::docsummary::DocsumStoreValue getMappedDocsum() { assert(_packer.Init(class_id)); + auto doc = std::make_unique<Document>(_doc_type, DocumentId("id:test:test::0")); { ArrayFieldValue array_value(_array_type); array_value.append(make_elem_value("a", 3)); array_value.append(make_elem_value("b", 5)); array_value.append(make_elem_value("c", 7)); write_field_value(array_value); + doc->setValue("array_in_doc", array_value); } { MapFieldValue map_value(_map_type); @@ -119,6 +126,7 @@ public: map_value.put(StringFieldValue("b"), *make_elem_value("b", 5)); map_value.put(StringFieldValue("c"), *make_elem_value("c", 7)); write_field_value(map_value); + doc->setValue("map_in_doc", map_value); } { MapFieldValue map2_value(_map_type); @@ -128,7 +136,7 @@ public: const char* buf; uint32_t buf_len; assert(_packer.GetDocsumBlob(&buf, &buf_len)); - return DocsumStoreValue(buf, buf_len); + return DocsumStoreValue(buf, buf_len, std::move(doc)); } }; @@ -186,7 +194,8 @@ private: Slime run_filter_field_writer(const std::string& input_field_name, const ElementVector& matching_elements) { auto writer = make_field_writer(input_field_name); GeneralResult result(_doc_store.get_class()); - result.inplaceUnpack(_doc_store.getMappedDocsum()); + auto docsum = _doc_store.getMappedDocsum(); + result.inplaceUnpack(docsum); StateCallback callback(input_field_name, matching_elements); GetDocsumsState state(callback); Slime slime; @@ -206,7 +215,6 @@ public: ~MatchedElementsFilterTest() {} std::unique_ptr<IDocsumFieldWriter> make_field_writer(const std::string& input_field_name) { int input_field_enum = _doc_store.get_config().GetFieldNameEnum().Lookup(input_field_name.c_str()); - EXPECT_GE(input_field_enum, 0); return MatchedElementsFilterDFW::create(input_field_name, input_field_enum, _attr_ctx, _mapper); } @@ -229,6 +237,17 @@ TEST_F(MatchedElementsFilterTest, filters_elements_in_array_field_value) "{'name':'c','weight':7}]"); } +TEST_F(MatchedElementsFilterTest, filters_elements_in_array_field_value_when_input_field_is_not_in_docsum_blob) +{ + expect_filtered("array_in_doc", {}, "[]"); + expect_filtered("array_in_doc", {0}, "[{'name':'a','weight':3}]"); + expect_filtered("array_in_doc", {1}, "[{'name':'b','weight':5}]"); + expect_filtered("array_in_doc", {2}, "[{'name':'c','weight':7}]"); + expect_filtered("array_in_doc", {0, 1, 2}, "[{'name':'a','weight':3}," + "{'name':'b','weight':5}," + "{'name':'c','weight':7}]"); +} + TEST_F(MatchedElementsFilterTest, struct_field_mapper_is_setup_for_array_field_value) { auto writer = make_field_writer("array"); @@ -248,6 +267,17 @@ TEST_F(MatchedElementsFilterTest, filters_elements_in_map_field_value) "{'key':'c','value':{'name':'c','weight':7}}]"); } +TEST_F(MatchedElementsFilterTest, filters_elements_in_map_field_value_when_input_field_is_not_in_docsum_blob) +{ + expect_filtered("map_in_doc", {}, "[]"); + expect_filtered("map_in_doc", {0}, "[{'key':'a','value':{'name':'a','weight':3}}]"); + expect_filtered("map_in_doc", {1}, "[{'key':'b','value':{'name':'b','weight':5}}]"); + expect_filtered("map_in_doc", {2}, "[{'key':'c','value':{'name':'c','weight':7}}]"); + expect_filtered("map_in_doc", {0, 1, 2}, "[{'key':'a','value':{'name':'a','weight':3}}," + "{'key':'b','value':{'name':'b','weight':5}}," + "{'key':'c','value':{'name':'c','weight':7}}]"); +} + TEST_F(MatchedElementsFilterTest, struct_field_mapper_is_setup_for_map_field_value) { { diff --git a/searchsummary/src/vespa/searchsummary/docsummary/CMakeLists.txt b/searchsummary/src/vespa/searchsummary/docsummary/CMakeLists.txt index 6d846a47d93..b5ce0a0619c 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/CMakeLists.txt +++ b/searchsummary/src/vespa/searchsummary/docsummary/CMakeLists.txt @@ -8,8 +8,10 @@ vespa_add_library(searchsummary_docsummary OBJECT docsumconfig.cpp docsumfieldwriter.cpp docsumstate.cpp + docsumstorevalue.cpp docsumwriter.cpp dynamicteaserdfw.cpp + general_result.cpp geoposdfw.cpp getdocsumargs.cpp juniperproperties.cpp @@ -28,7 +30,6 @@ vespa_add_library(searchsummary_docsummary OBJECT summaryfieldconverter.cpp textextractordfw.cpp tokenizer.cpp - urlresult.cpp AFTER searchsummary_config ) diff --git a/searchsummary/src/vespa/searchsummary/docsummary/docsumfieldwriter.h b/searchsummary/src/vespa/searchsummary/docsummary/docsumfieldwriter.h index 1a904d4f858..43375fb47f3 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/docsumfieldwriter.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/docsumfieldwriter.h @@ -2,7 +2,7 @@ #pragma once -#include "urlresult.h" +#include "general_result.h" #include "resultconfig.h" #include <vespa/searchlib/util/rawbuf.h> #include <vespa/vespalib/data/slime/inserter.h> diff --git a/searchsummary/src/vespa/searchsummary/docsummary/docsumstorevalue.cpp b/searchsummary/src/vespa/searchsummary/docsummary/docsumstorevalue.cpp new file mode 100644 index 00000000000..8ce986f6a16 --- /dev/null +++ b/searchsummary/src/vespa/searchsummary/docsummary/docsumstorevalue.cpp @@ -0,0 +1,28 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "docsumstorevalue.h" +#include <vespa/document/fieldvalue/document.h> + +namespace search::docsummary { + +DocsumStoreValue::DocsumStoreValue() + : _value(static_cast<const char*>(0), 0), + _document() +{ +} + +DocsumStoreValue::DocsumStoreValue(const char *pt_, uint32_t len_) + : _value(pt_, len_), + _document() +{ +} + +DocsumStoreValue::DocsumStoreValue(const char *pt_, uint32_t len_, std::unique_ptr<document::Document> document_) + : _value(pt_, len_), + _document(std::move(document_)) +{ +} + +DocsumStoreValue::~DocsumStoreValue() = default; + +} diff --git a/searchsummary/src/vespa/searchsummary/docsummary/docsumstorevalue.h b/searchsummary/src/vespa/searchsummary/docsummary/docsumstorevalue.h index a15fb6d8892..4aaf1bf5816 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/docsumstorevalue.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/docsumstorevalue.h @@ -2,8 +2,11 @@ #pragma once #include <cstdint> +#include <memory> #include <utility> +namespace document { class Document; } + namespace search::docsummary { /** @@ -16,12 +19,19 @@ class DocsumStoreValue { private: std::pair<const char *, uint32_t> _value; + // The document instance that was used to generate the docsum blob. + // Note: This is temporary until the docsummary framework is simplified, + // and the docsum blob concept is removed. + std::unique_ptr<document::Document> _document; public: + DocsumStoreValue(const DocsumStoreValue&) = delete; + DocsumStoreValue& operator=(const DocsumStoreValue&) = delete; + /** * Construct object representing an empty docsum blob. **/ - DocsumStoreValue() : _value(static_cast<const char*>(0), 0) {} + DocsumStoreValue(); /** * Construct object encapsulating the given location and size. @@ -29,7 +39,18 @@ public: * @param pt_ docsum location * @param len_ docsum size **/ - DocsumStoreValue(const char *pt_, uint32_t len_) : _value(pt_, len_) {} + DocsumStoreValue(const char *pt_, uint32_t len_); + + /** + * Construct object encapsulating the given location and size. + * + * @param pt_ docsum location + * @param len_ docsum size + * @param document_ document instance used to generate the docsum blob + **/ + DocsumStoreValue(const char *pt_, uint32_t len_, std::unique_ptr<document::Document> document_); + + ~DocsumStoreValue(); /** * @return docsum blob location @@ -55,6 +76,8 @@ public: * @return true if this has a valid blob **/ bool valid() const { return (_value.first != 0) && (_value.second >= sizeof(uint32_t)); } + + const document::Document* get_document() const { return _document.get(); } }; } diff --git a/searchsummary/src/vespa/searchsummary/docsummary/docsumwriter.h b/searchsummary/src/vespa/searchsummary/docsummary/docsumwriter.h index e5dd2793089..2b686afcb42 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/docsumwriter.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/docsumwriter.h @@ -3,7 +3,7 @@ #pragma once #include "juniperproperties.h" -#include "urlresult.h" +#include "general_result.h" #include "resultconfig.h" #include "docsumstore.h" #include "keywordextractor.h" diff --git a/searchsummary/src/vespa/searchsummary/docsummary/urlresult.cpp b/searchsummary/src/vespa/searchsummary/docsummary/general_result.cpp index 074cc1cadf1..a6a5263f5ac 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/urlresult.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/general_result.cpp @@ -1,7 +1,8 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include "urlresult.h" +#include "general_result.h" #include "resultconfig.h" +#include <vespa/document/fieldvalue/document.h> #include <zlib.h> #include <cassert> @@ -59,7 +60,8 @@ GeneralResult::GeneralResult(const ResultClass *resClass) _entrycnt(0), _entries(nullptr), _buf(nullptr), - _bufEnd(nullptr) + _bufEnd(nullptr), + _document() { } @@ -90,6 +92,15 @@ GeneralResult::GetEntryFromEnumValue(uint32_t value) return (idx >= 0 && (uint32_t)idx < _entrycnt) ? &_entries[idx] : nullptr; } +std::unique_ptr<document::FieldValue> +GeneralResult::get_field_value(const vespalib::string& field_name) const +{ + if (_document != nullptr) { + return _document->getValue(field_name); + } + return std::unique_ptr<document::FieldValue>(); +} + bool GeneralResult::unpack(const char *buf, const size_t buflen) { diff --git a/searchsummary/src/vespa/searchsummary/docsummary/urlresult.h b/searchsummary/src/vespa/searchsummary/docsummary/general_result.h index a4cdd1b7f69..4920985d9ae 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/urlresult.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/general_result.h @@ -5,6 +5,11 @@ #include "resultclass.h" #include "docsumstorevalue.h" +namespace document { +class Document; +class FieldValue; +} + namespace search::docsummary { class GeneralResult @@ -18,6 +23,7 @@ private: ResEntry *_entries; char *_buf; // allocated in same chunk as _entries char *_bufEnd; // first byte after _buf + const document::Document* _document; bool InBuf(const void *pt) const { return ((const char *)pt >= _buf && @@ -35,10 +41,12 @@ public: ResEntry *GetEntry(uint32_t idx); ResEntry *GetEntry(const char *name); ResEntry *GetEntryFromEnumValue(uint32_t val); + std::unique_ptr<document::FieldValue> get_field_value(const vespalib::string& field_name) const; bool unpack(const char *buf, const size_t buflen); bool inplaceUnpack(const DocsumStoreValue &value) { if (value.valid()) { + _document = value.get_document(); return unpack(value.fieldsPt(), value.fieldsSz()); } else { return false; diff --git a/searchsummary/src/vespa/searchsummary/docsummary/juniperdfw.h b/searchsummary/src/vespa/searchsummary/docsummary/juniperdfw.h index 17beddf1e60..1cdeeb2f679 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/juniperdfw.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/juniperdfw.h @@ -2,7 +2,7 @@ #pragma once -#include "urlresult.h" +#include "general_result.h" #include "resultconfig.h" #include "docsumfieldwriter.h" #include <vespa/searchlib/util/rawbuf.h> diff --git a/searchsummary/src/vespa/searchsummary/docsummary/matched_elements_filter_dfw.cpp b/searchsummary/src/vespa/searchsummary/docsummary/matched_elements_filter_dfw.cpp index 69085258a43..812c19bc3f2 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/matched_elements_filter_dfw.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/matched_elements_filter_dfw.cpp @@ -3,6 +3,9 @@ #include "docsumstate.h" #include "matched_elements_filter_dfw.h" #include "struct_fields_resolver.h" +#include "summaryfieldconverter.h" +#include <vespa/document/fieldvalue/document.h> +#include <vespa/document/fieldvalue/literalfieldvalue.h> #include <vespa/searchcommon/attribute/iattributecontext.h> #include <vespa/searchlib/common/matching_elements.h> #include <vespa/searchlib/common/struct_field_mapper.h> @@ -11,6 +14,8 @@ #include <vespa/vespalib/data/smart_buffer.h> #include <cassert> +using document::FieldValue; +using document::LiteralFieldValueB; using vespalib::Slime; using vespalib::slime::ArrayInserter; using vespalib::slime::BinaryFormat; @@ -54,12 +59,44 @@ MatchedElementsFilterDFW::~MatchedElementsFilterDFW() = default; namespace { void -decode_input_field(const ResEntry& entry, search::RawBuf& target_buf, Slime& input_field) +decode_input_field_to_slime(const ResEntry& entry, search::RawBuf& target_buf, Slime& input_field_as_slime) { const char* buf; uint32_t buf_len; entry._resolve_field(&buf, &buf_len, &target_buf); - BinaryFormat::decode(vespalib::Memory(buf, buf_len), input_field); + BinaryFormat::decode(vespalib::Memory(buf, buf_len), input_field_as_slime); +} + +void +convert_input_field_to_slime(const FieldValue& input_field_value, Slime& input_field_as_slime) +{ + // This is the same conversion that happens in proton::DocumentStoreAdapter. + auto converted = SummaryFieldConverter::convertSummaryField(false, input_field_value); + // This should hold as we also have asserted that (type == ResType::RES_JSONSTRING); + assert(converted->getClass().inherits(LiteralFieldValueB::classId)); + auto& literal = static_cast<const LiteralFieldValueB&>(*converted); + vespalib::stringref buf = literal.getValueRef(); + BinaryFormat::decode(vespalib::Memory(buf.data(), buf.size()), input_field_as_slime); +} + +bool +resolve_input_field_as_slime(GeneralResult& result, GetDocsumsState& state, + int entry_idx, const vespalib::string& input_field_name, + Slime& input_field_as_slime) +{ + ResEntry* entry = result.GetEntry(entry_idx); + if (entry != nullptr) { + decode_input_field_to_slime(*entry, state._docSumFieldSpace, input_field_as_slime); + return true; + } else { + // Use the document instance if the input field is not in the docsum blob. + auto field_value = result.get_field_value(input_field_name); + if (field_value) { + convert_input_field_to_slime(*field_value, input_field_as_slime); + return true; + } + } + return false; } void @@ -86,14 +123,12 @@ MatchedElementsFilterDFW::insertField(uint32_t docid, GeneralResult* result, Get { assert(type == ResType::RES_JSONSTRING); int entry_idx = result->GetClass()->GetIndexFromEnumValue(_input_field_enum); - ResEntry* entry = result->GetEntry(entry_idx); - if (entry != nullptr) { - Slime input_field; - decode_input_field(*entry, state->_docSumFieldSpace, input_field); - + Slime input_field; + if (resolve_input_field_as_slime(*result, *state, entry_idx, _input_field_name, input_field)) { Slime output_field; - filter_matching_elements_in_input_field(input_field, state->get_matching_elements(*_struct_field_mapper).get_matching_elements(docid, _input_field_name), output_field); - + filter_matching_elements_in_input_field(input_field, + state->get_matching_elements(*_struct_field_mapper) + .get_matching_elements(docid, _input_field_name), output_field); inject(output_field.get(), target); } } diff --git a/searchsummary/src/vespa/searchsummary/docsummary/resultconfig.h b/searchsummary/src/vespa/searchsummary/docsummary/resultconfig.h index 31218d94e93..3df96c3e838 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/resultconfig.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/resultconfig.h @@ -3,7 +3,7 @@ #pragma once #include "resultclass.h" -#include "urlresult.h" +#include "general_result.h" #include <vespa/config-summary.h> #include <vespa/searchlib/util/rawbuf.h> #include <vespa/searchlib/util/stringenum.h> diff --git a/security-utils/pom.xml b/security-utils/pom.xml index 10dec598915..4796a809293 100644 --- a/security-utils/pom.xml +++ b/security-utils/pom.xml @@ -11,6 +11,13 @@ <artifactId>security-utils</artifactId> <packaging>container-plugin</packaging> <version>7-SNAPSHOT</version> + + <properties> + <!-- vespa-http-client targets jdk8 and uses this library --> + <!-- TODO remove once vespa-http-client no longer builds against jdk8 --> + <maven.compiler.release>8</maven.compiler.release> + </properties> + <dependencies> <!-- provided --> <dependency> @@ -19,6 +26,13 @@ <version>${project.version}</version> <scope>provided</scope> </dependency> + <dependency> + <!-- required for bundle-plugin to generate import-package statements for Java's standard library --> + <groupId>com.yahoo.vespa</groupId> + <artifactId>jdisc_core</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> <!-- compile scope --> <dependency> @@ -67,6 +81,23 @@ <artifactId>bundle-plugin</artifactId> <extensions>true</extensions> </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-compiler-plugin</artifactId> + <configuration> + <jdkToolchain> + <version>${java.version}</version> + </jdkToolchain> + <source>${java.version}</source> + <target>${java.version}</target> + <showDeprecation>true</showDeprecation> + <compilerArgs> + <arg>-Xlint:all</arg> + <arg>-Xlint:-serial</arg> + <arg>-Werror</arg> + </compilerArgs> + </configuration> + </plugin> </plugins> </build> </project> diff --git a/security-utils/src/main/java/com/yahoo/security/SubjectAlternativeName.java b/security-utils/src/main/java/com/yahoo/security/SubjectAlternativeName.java index 81581c8146c..ab58c607891 100644 --- a/security-utils/src/main/java/com/yahoo/security/SubjectAlternativeName.java +++ b/security-utils/src/main/java/com/yahoo/security/SubjectAlternativeName.java @@ -64,7 +64,7 @@ public class SubjectAlternativeName { case GeneralName.directoryName: return X500Name.getInstance(name).toString(); case GeneralName.iPAddress: - var octets = DEROctetString.getInstance(name.toASN1Primitive()).getOctets(); + byte[] octets = DEROctetString.getInstance(name.toASN1Primitive()).getOctets(); try { return InetAddress.getByAddress(octets).getHostAddress(); } catch (UnknownHostException e) { diff --git a/security-utils/src/main/java/com/yahoo/security/tls/AutoReloadingX509KeyManager.java b/security-utils/src/main/java/com/yahoo/security/tls/AutoReloadingX509KeyManager.java index faf6ecb4348..18764f51dc5 100644 --- a/security-utils/src/main/java/com/yahoo/security/tls/AutoReloadingX509KeyManager.java +++ b/security-utils/src/main/java/com/yahoo/security/tls/AutoReloadingX509KeyManager.java @@ -64,8 +64,8 @@ public class AutoReloadingX509KeyManager extends X509ExtendedKeyManager implemen return KeyStoreBuilder.withType(KeyStoreType.PKCS12) .withKeyEntry( CERTIFICATE_ALIAS, - KeyUtils.fromPemEncodedPrivateKey(Files.readString(privateKey)), - X509CertificateUtils.certificateListFromPem(Files.readString(certificateChain))) + KeyUtils.fromPemEncodedPrivateKey(com.yahoo.vespa.jdk8compat.Files.readString(privateKey)), + X509CertificateUtils.certificateListFromPem(com.yahoo.vespa.jdk8compat.Files.readString(certificateChain))) .build(); } catch (IOException e) { throw new UncheckedIOException(e); diff --git a/security-utils/src/main/java/com/yahoo/security/tls/ConfigFileBasedTlsContext.java b/security-utils/src/main/java/com/yahoo/security/tls/ConfigFileBasedTlsContext.java index f5bd866eb27..f746480b126 100644 --- a/security-utils/src/main/java/com/yahoo/security/tls/ConfigFileBasedTlsContext.java +++ b/security-utils/src/main/java/com/yahoo/security/tls/ConfigFileBasedTlsContext.java @@ -85,7 +85,7 @@ public class ConfigFileBasedTlsContext implements TlsContext { private static KeyStore loadTruststore(Path caCertificateFile) { try { return KeyStoreBuilder.withType(KeyStoreType.PKCS12) - .withCertificateEntries("cert", X509CertificateUtils.certificateListFromPem(Files.readString(caCertificateFile))) + .withCertificateEntries("cert", X509CertificateUtils.certificateListFromPem(com.yahoo.vespa.jdk8compat.Files.readString(caCertificateFile))) .build(); } catch (IOException e) { throw new UncheckedIOException(e); @@ -97,8 +97,8 @@ public class ConfigFileBasedTlsContext implements TlsContext { return KeyStoreBuilder.withType(KeyStoreType.PKCS12) .withKeyEntry( "default", - KeyUtils.fromPemEncodedPrivateKey(Files.readString(privateKeyFile)), - X509CertificateUtils.certificateListFromPem(Files.readString(certificatesFile))) + KeyUtils.fromPemEncodedPrivateKey(com.yahoo.vespa.jdk8compat.Files.readString(privateKeyFile)), + X509CertificateUtils.certificateListFromPem(com.yahoo.vespa.jdk8compat.Files.readString(certificatesFile))) .build(); } catch (IOException e) { throw new UncheckedIOException(e); @@ -115,7 +115,7 @@ public class ConfigFileBasedTlsContext implements TlsContext { .withTrustManagerFactory( ignoredTruststore -> options.getAuthorizedPeers() .map(authorizedPeers -> (X509ExtendedTrustManager) new PeerAuthorizerTrustManager(authorizedPeers, mode, mutableTrustManager)) - .orElseGet(() -> new PeerAuthorizerTrustManager(new AuthorizedPeers(Set.of()), AuthorizationMode.DISABLE, mutableTrustManager))) + .orElseGet(() -> new PeerAuthorizerTrustManager(new AuthorizedPeers(com.yahoo.vespa.jdk8compat.Set.of()), AuthorizationMode.DISABLE, mutableTrustManager))) .build(); List<String> acceptedCiphers = options.getAcceptedCiphers(); Set<String> ciphers = acceptedCiphers.isEmpty() ? TlsContext.ALLOWED_CIPHER_SUITES : new HashSet<>(acceptedCiphers); diff --git a/security-utils/src/main/java/com/yahoo/security/tls/DefaultTlsContext.java b/security-utils/src/main/java/com/yahoo/security/tls/DefaultTlsContext.java index f4a89da988a..c3f10a464a5 100644 --- a/security-utils/src/main/java/com/yahoo/security/tls/DefaultTlsContext.java +++ b/security-utils/src/main/java/com/yahoo/security/tls/DefaultTlsContext.java @@ -63,14 +63,14 @@ public class DefaultTlsContext implements TlsContext { String.format("None of the accepted ciphers are supported (supported=%s, accepted=%s)", supportedCiphers, acceptedCiphers)); } - log.log(Level.FINE, () -> String.format("Allowed cipher suites that are supported: %s", List.of(allowedCiphers))); + log.log(Level.FINE, () -> String.format("Allowed cipher suites that are supported: %s", com.yahoo.vespa.jdk8compat.List.of(allowedCiphers))); return allowedCiphers; } private static String[] getAllowedProtocols(SSLContext sslContext) { Set<String> allowedProtocols = TlsContext.getAllowedProtocols(sslContext); - log.log(Level.FINE, () -> String.format("Allowed protocols that are supported: %s", List.of(allowedProtocols))); - return allowedProtocols.toArray(String[]::new); + log.log(Level.FINE, () -> String.format("Allowed protocols that are supported: %s", com.yahoo.vespa.jdk8compat.List.of(allowedProtocols))); + return com.yahoo.vespa.jdk8compat.Collection.toArray(allowedProtocols, String[]::new); } @Override @@ -131,7 +131,7 @@ public class DefaultTlsContext implements TlsContext { if (authorizedPeers != null) { builder.withTrustManagerFactory(truststore -> new PeerAuthorizerTrustManager(authorizedPeers, mode, truststore)); } else { - builder.withTrustManagerFactory(truststore -> new PeerAuthorizerTrustManager(new AuthorizedPeers(Set.of()), AuthorizationMode.DISABLE, truststore)); + builder.withTrustManagerFactory(truststore -> new PeerAuthorizerTrustManager(new AuthorizedPeers(com.yahoo.vespa.jdk8compat.Set.of()), AuthorizationMode.DISABLE, truststore)); } return builder.build(); } diff --git a/security-utils/src/main/java/com/yahoo/security/tls/KeyManagerUtils.java b/security-utils/src/main/java/com/yahoo/security/tls/KeyManagerUtils.java index 2e48de3c01f..c60f13f9729 100644 --- a/security-utils/src/main/java/com/yahoo/security/tls/KeyManagerUtils.java +++ b/security-utils/src/main/java/com/yahoo/security/tls/KeyManagerUtils.java @@ -30,7 +30,7 @@ public class KeyManagerUtils { .filter(manager -> manager instanceof X509ExtendedKeyManager) .map(X509ExtendedKeyManager.class::cast) .findFirst() - .orElseThrow(() -> new RuntimeException("No X509ExtendedKeyManager in " + List.of(keyManagers))); + .orElseThrow(() -> new RuntimeException("No X509ExtendedKeyManager in " + com.yahoo.vespa.jdk8compat.List.of(keyManagers))); } catch (GeneralSecurityException e) { throw new RuntimeException(e); } diff --git a/security-utils/src/main/java/com/yahoo/security/tls/TlsContext.java b/security-utils/src/main/java/com/yahoo/security/tls/TlsContext.java index 4eea1eb3d72..886cf3e886b 100644 --- a/security-utils/src/main/java/com/yahoo/security/tls/TlsContext.java +++ b/security-utils/src/main/java/com/yahoo/security/tls/TlsContext.java @@ -23,7 +23,7 @@ public interface TlsContext extends AutoCloseable { * For TLSv1.3 we allow the DEFAULT group ciphers. * Note that we _only_ allow AEAD ciphers for either TLS version. */ - Set<String> ALLOWED_CIPHER_SUITES = Set.of( + Set<String> ALLOWED_CIPHER_SUITES = com.yahoo.vespa.jdk8compat.Set.of( "TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384", "TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384", "TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256", // Java 12 @@ -34,8 +34,8 @@ public interface TlsContext extends AutoCloseable { "TLS_AES_256_GCM_SHA384", // TLSv1.3 "TLS_CHACHA20_POLY1305_SHA256"); // TLSv1.3, Java 12 - Set<String> ALLOWED_PROTOCOLS = Set.of("TLSv1.2"); // TODO Enable TLSv1.3 - String SSL_CONTEXT_VERSION = "TLSv1.2"; // TODO Enable TLSv1.3 + Set<String> ALLOWED_PROTOCOLS = com.yahoo.vespa.jdk8compat.Set.of("TLSv1.2"); // TODO Enable TLSv1.3 + String SSL_CONTEXT_VERSION = "TLS"; // Use SSLContext implementations that supports all TLS versions /** * @return the allowed cipher suites supported by the provided context instance diff --git a/security-utils/src/main/java/com/yahoo/security/tls/TrustManagerUtils.java b/security-utils/src/main/java/com/yahoo/security/tls/TrustManagerUtils.java index 7c1d7070617..17f56011261 100644 --- a/security-utils/src/main/java/com/yahoo/security/tls/TrustManagerUtils.java +++ b/security-utils/src/main/java/com/yahoo/security/tls/TrustManagerUtils.java @@ -29,7 +29,7 @@ public class TrustManagerUtils { .filter(manager -> manager instanceof X509ExtendedTrustManager) .map(X509ExtendedTrustManager.class::cast) .findFirst() - .orElseThrow(() -> new RuntimeException("No X509ExtendedTrustManager in " + List.of(trustManagers))); + .orElseThrow(() -> new RuntimeException("No X509ExtendedTrustManager in " + com.yahoo.vespa.jdk8compat.List.of(trustManagers))); } catch (GeneralSecurityException e) { throw new RuntimeException(e); } diff --git a/security-utils/src/main/java/com/yahoo/vespa/jdk8compat/Collection.java b/security-utils/src/main/java/com/yahoo/vespa/jdk8compat/Collection.java new file mode 100644 index 00000000000..fbfea01b2c7 --- /dev/null +++ b/security-utils/src/main/java/com/yahoo/vespa/jdk8compat/Collection.java @@ -0,0 +1,16 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.jdk8compat; + +import java.util.function.IntFunction; + +/** + * Backport of new {@link java.util.Collection} methods added after JDK8 + * + * @author bjorncs + */ +public interface Collection { + static <T> T[] toArray(java.util.Collection<T> collection, IntFunction<T[]> generator) { + return collection.toArray(generator.apply(collection.size())); + } + +} diff --git a/security-utils/src/main/java/com/yahoo/vespa/jdk8compat/Files.java b/security-utils/src/main/java/com/yahoo/vespa/jdk8compat/Files.java new file mode 100644 index 00000000000..cc3bd698cd5 --- /dev/null +++ b/security-utils/src/main/java/com/yahoo/vespa/jdk8compat/Files.java @@ -0,0 +1,24 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.jdk8compat; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.OpenOption; +import java.nio.file.Path; + +/** + * Backport of new {@link java.nio.file.Files} methods added after JDK8 + * + * @author bjorncs + */ +public interface Files { + + static String readString(Path path) throws IOException { + byte[] bytes = java.nio.file.Files.readAllBytes(path); + return new String(bytes, StandardCharsets.UTF_8); + } + + static Path writeString(Path path, CharSequence string, OpenOption... options) throws IOException { + return java.nio.file.Files.write(path, string.toString().getBytes(), options); + } +} diff --git a/security-utils/src/main/java/com/yahoo/vespa/jdk8compat/List.java b/security-utils/src/main/java/com/yahoo/vespa/jdk8compat/List.java new file mode 100644 index 00000000000..f57834e93cb --- /dev/null +++ b/security-utils/src/main/java/com/yahoo/vespa/jdk8compat/List.java @@ -0,0 +1,17 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.jdk8compat; + +import java.util.Arrays; + +/** + * Backport of new {@link java.util.List} methods added after JDK8 + * + * @author bjorncs + */ +public interface List { + @SafeVarargs + @SuppressWarnings("varargs") + static <E> java.util.List<E> of(E... elements) { + return Arrays.asList(elements); + } +} diff --git a/security-utils/src/main/java/com/yahoo/vespa/jdk8compat/Set.java b/security-utils/src/main/java/com/yahoo/vespa/jdk8compat/Set.java new file mode 100644 index 00000000000..b2c998bb716 --- /dev/null +++ b/security-utils/src/main/java/com/yahoo/vespa/jdk8compat/Set.java @@ -0,0 +1,18 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.jdk8compat; + +import java.util.Arrays; +import java.util.HashSet; + +/** + * Backport of new {@link java.util.Set} methods added after JDK8 + * + * @author bjorncs + */ +public interface Set { + @SafeVarargs + @SuppressWarnings("varargs") + static <E> java.util.Set<E> of(E... elements) { + return new HashSet<>(Arrays.asList(elements)); + } +} diff --git a/security-utils/src/main/java/com/yahoo/vespa/jdk8compat/package-info.java b/security-utils/src/main/java/com/yahoo/vespa/jdk8compat/package-info.java new file mode 100644 index 00000000000..40d74321438 --- /dev/null +++ b/security-utils/src/main/java/com/yahoo/vespa/jdk8compat/package-info.java @@ -0,0 +1,8 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * JDK8 port of types and methods added in later JDK versions. + * TODO Remove this package once vespa-http-client/security-utils no longer targets JDK8 + * + * @author bjorncs + */ +package com.yahoo.vespa.jdk8compat;
\ No newline at end of file diff --git a/security-utils/src/test/java/com/yahoo/security/tls/AutoReloadingX509KeyManagerTest.java b/security-utils/src/test/java/com/yahoo/security/tls/AutoReloadingX509KeyManagerTest.java index 823989ac4ca..22710e7f393 100644 --- a/security-utils/src/test/java/com/yahoo/security/tls/AutoReloadingX509KeyManagerTest.java +++ b/security-utils/src/test/java/com/yahoo/security/tls/AutoReloadingX509KeyManagerTest.java @@ -15,7 +15,6 @@ import org.mockito.Mockito; import javax.security.auth.x500.X500Principal; import java.io.IOException; import java.math.BigInteger; -import java.nio.file.Files; import java.nio.file.Path; import java.security.KeyPair; import java.security.Principal; @@ -42,12 +41,12 @@ public class AutoReloadingX509KeyManagerTest { public void crypto_material_is_reloaded_when_scheduler_task_is_executed() throws IOException { KeyPair keyPair = KeyUtils.generateKeypair(KeyAlgorithm.EC); Path privateKeyFile = tempDirectory.newFile().toPath(); - Files.writeString(privateKeyFile, KeyUtils.toPem(keyPair.getPrivate())); + com.yahoo.vespa.jdk8compat.Files.writeString(privateKeyFile, KeyUtils.toPem(keyPair.getPrivate())); Path certificateFile = tempDirectory.newFile().toPath(); BigInteger serialNumberInitialCertificate = BigInteger.ONE; X509Certificate initialCertificate = generateCertificate(keyPair, serialNumberInitialCertificate); - Files.writeString(certificateFile, X509CertificateUtils.toPem(initialCertificate)); + com.yahoo.vespa.jdk8compat.Files.writeString(certificateFile, X509CertificateUtils.toPem(initialCertificate)); ScheduledExecutorService scheduler = Mockito.mock(ScheduledExecutorService.class); ArgumentCaptor<Runnable> updaterTaskCaptor = ArgumentCaptor.forClass(Runnable.class); @@ -60,9 +59,9 @@ public class AutoReloadingX509KeyManagerTest { assertThat(certChain).hasSize(1); assertThat(certChain[0].getSerialNumber()).isEqualTo(serialNumberInitialCertificate); - BigInteger serialNumberUpdatedCertificate = BigInteger.TWO; + BigInteger serialNumberUpdatedCertificate = BigInteger.TEN; X509Certificate updatedCertificate = generateCertificate(keyPair, serialNumberUpdatedCertificate); - Files.writeString(certificateFile, X509CertificateUtils.toPem(updatedCertificate)); + com.yahoo.vespa.jdk8compat.Files.writeString(certificateFile, X509CertificateUtils.toPem(updatedCertificate)); updaterTaskCaptor.getValue().run(); // run update task in ReloadingX509KeyManager diff --git a/security-utils/src/test/java/com/yahoo/security/tls/ConfigFileBasedTlsContextTest.java b/security-utils/src/test/java/com/yahoo/security/tls/ConfigFileBasedTlsContextTest.java index 4e6f0a141b0..54a1e3847f9 100644 --- a/security-utils/src/test/java/com/yahoo/security/tls/ConfigFileBasedTlsContextTest.java +++ b/security-utils/src/test/java/com/yahoo/security/tls/ConfigFileBasedTlsContextTest.java @@ -35,17 +35,17 @@ public class ConfigFileBasedTlsContextTest { public void can_create_sslcontext_from_credentials() throws IOException, InterruptedException { KeyPair keyPair = KeyUtils.generateKeypair(EC); Path privateKeyFile = tempDirectory.newFile().toPath(); - Files.writeString(privateKeyFile, KeyUtils.toPem(keyPair.getPrivate())); + com.yahoo.vespa.jdk8compat.Files.writeString(privateKeyFile, KeyUtils.toPem(keyPair.getPrivate())); X509Certificate certificate = X509CertificateBuilder .fromKeypair(keyPair, new X500Principal("CN=dummy"), EPOCH, EPOCH.plus(1, DAYS), SHA256_WITH_ECDSA, BigInteger.ONE) .build(); Path certificateChainFile = tempDirectory.newFile().toPath(); String certificatePem = X509CertificateUtils.toPem(certificate); - Files.writeString(certificateChainFile, certificatePem); + com.yahoo.vespa.jdk8compat.Files.writeString(certificateChainFile, certificatePem); Path caCertificatesFile = tempDirectory.newFile().toPath(); - Files.writeString(caCertificatesFile, certificatePem); + com.yahoo.vespa.jdk8compat.Files.writeString(caCertificatesFile, certificatePem); TransportSecurityOptions options = new TransportSecurityOptions.Builder() .withCertificates(certificateChainFile, privateKeyFile) diff --git a/security-utils/src/test/java/com/yahoo/security/tls/MutableX509KeyManagerTest.java b/security-utils/src/test/java/com/yahoo/security/tls/MutableX509KeyManagerTest.java index 30e54d3c09d..2a6ee35ec18 100644 --- a/security-utils/src/test/java/com/yahoo/security/tls/MutableX509KeyManagerTest.java +++ b/security-utils/src/test/java/com/yahoo/security/tls/MutableX509KeyManagerTest.java @@ -42,7 +42,7 @@ public class MutableX509KeyManagerTest { assertThat(certChain).hasSize(1); assertThat(certChain[0].getSerialNumber()).isEqualTo(serialNumberInitialCertificate); - BigInteger serialNumberUpdatedCertificate = BigInteger.TWO; + BigInteger serialNumberUpdatedCertificate = BigInteger.TEN; KeyStore updatedKeystore = generateKeystore(keyPair, serialNumberUpdatedCertificate); keyManager.updateKeystore(updatedKeystore, new char[0]); diff --git a/security-utils/src/test/java/com/yahoo/security/tls/TransportSecurityOptionsTest.java b/security-utils/src/test/java/com/yahoo/security/tls/TransportSecurityOptionsTest.java index 9d8f26cdd2c..28dc10d31d5 100644 --- a/security-utils/src/test/java/com/yahoo/security/tls/TransportSecurityOptionsTest.java +++ b/security-utils/src/test/java/com/yahoo/security/tls/TransportSecurityOptionsTest.java @@ -8,9 +8,8 @@ import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; -import java.util.List; -import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; /** * @author bjorncs @@ -21,7 +20,7 @@ public class TransportSecurityOptionsTest { private static final TransportSecurityOptions OPTIONS = new TransportSecurityOptions.Builder() .withCertificates(Paths.get("certs.pem"), Paths.get("myhost.key")) .withCaCertificates(Paths.get("my_cas.pem")) - .withAcceptedCiphers(List.of("TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384" , "TLS_AES_256_GCM_SHA384")) + .withAcceptedCiphers(com.yahoo.vespa.jdk8compat.List.of("TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384" , "TLS_AES_256_GCM_SHA384")) .build(); @Test diff --git a/security-utils/src/test/java/com/yahoo/security/tls/json/TransportSecurityOptionsJsonSerializerTest.java b/security-utils/src/test/java/com/yahoo/security/tls/json/TransportSecurityOptionsJsonSerializerTest.java index 03489a60784..078aa58c948 100644 --- a/security-utils/src/test/java/com/yahoo/security/tls/json/TransportSecurityOptionsJsonSerializerTest.java +++ b/security-utils/src/test/java/com/yahoo/security/tls/json/TransportSecurityOptionsJsonSerializerTest.java @@ -65,7 +65,7 @@ public class TransportSecurityOptionsJsonSerializerTest { TransportSecurityOptions options = new TransportSecurityOptions.Builder() .withCertificates(Paths.get("certs.pem"), Paths.get("myhost.key")) .withCaCertificates(Paths.get("my_cas.pem")) - .withAcceptedCiphers(List.of("TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384" , "TLS_AES_256_GCM_SHA384")) + .withAcceptedCiphers(com.yahoo.vespa.jdk8compat.List.of("TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384" , "TLS_AES_256_GCM_SHA384")) .build(); File outputFile = tempDirectory.newFile(); try (OutputStream out = Files.newOutputStream(outputFile.toPath())) { diff --git a/vespa-documentgen-plugin/src/main/java/com/yahoo/vespa/DocumentGenMojo.java b/vespa-documentgen-plugin/src/main/java/com/yahoo/vespa/DocumentGenMojo.java index eb883662095..d95eddac57f 100644 --- a/vespa-documentgen-plugin/src/main/java/com/yahoo/vespa/DocumentGenMojo.java +++ b/vespa-documentgen-plugin/src/main/java/com/yahoo/vespa/DocumentGenMojo.java @@ -483,23 +483,25 @@ public class DocumentGenMojo extends AbstractMojo { out.write("}\n"); } - private static boolean hasAnyPostionDataType(DataType dt) { - if (dt instanceof CollectionDataType) { - return hasAnyPostionDataType(((CollectionDataType)dt).getNestedType()); + private static boolean hasAnyPositionDataType(DataType dt) { + if (PositionDataType.INSTANCE.equals(dt)) { + return true; + } else if (dt instanceof CollectionDataType) { + return hasAnyPositionDataType(((CollectionDataType)dt).getNestedType()); } else if (dt instanceof StructuredDataType) { return hasAnyPositionField(((StructuredDataType)dt).getFields()); } else { - return PositionDataType.INSTANCE.equals(dt); + return false; } } private static boolean hasAnyPositionField(Collection<Field> fields) { for (Field f : fields) { - if (hasAnyPostionDataType(f.getDataType())) { + if (hasAnyPositionDataType(f.getDataType())) { return true; } } - return true; + return false; } private Collection<Field> getAllUniqueFields(Boolean multipleInheritance, Collection<Field> allFields) { diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/ApacheGatewayConnection.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/ApacheGatewayConnection.java index 00f52d6337f..e7a1e6615f4 100644 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/ApacheGatewayConnection.java +++ b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/ApacheGatewayConnection.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.http.client.core.communication; +import ai.vespa.util.http.VespaHttpClientBuilder; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.yahoo.vespa.http.client.config.ConnectionParams; @@ -396,7 +397,7 @@ class ApacheGatewayConnection implements GatewayConnection { public HttpClient createClient() { HttpClientBuilder clientBuilder; if (connectionParams.useTlsConfigFromEnvironment()) { - clientBuilder = VespaTlsAwareClientBuilder.createHttpClientBuilder(); + clientBuilder = VespaHttpClientBuilder.create(); } else { clientBuilder = HttpClientBuilder.create(); if (useSsl && connectionParams.getSslContext() != null) { diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/VespaTlsAwareClientBuilder.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/VespaTlsAwareClientBuilder.java deleted file mode 100644 index be67e11963e..00000000000 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/VespaTlsAwareClientBuilder.java +++ /dev/null @@ -1,26 +0,0 @@ -// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.client.core.communication; - -import org.apache.http.impl.client.HttpClientBuilder; - -/** - * A static factory for VespaHttpClientBuilder. - * The main purpose of this class is to avoid references to classes not compiled with JDK8. - * - * @author bjorncs - */ -// TODO Remove use of reflection once vespa-http-client only targets JDK11 -// The VespaTlsAwareClientBuilder class refers to classes in security-utils / http-utils that targets JDK11+. -class VespaTlsAwareClientBuilder { - - private VespaTlsAwareClientBuilder() {} - - static HttpClientBuilder createHttpClientBuilder() { - try { - Class<?> builderClass = Class.forName("ai.vespa.util.http.VespaHttpClientBuilder"); - return (HttpClientBuilder) builderClass.getMethod("create").invoke(null); - } catch (ReflectiveOperationException e) { - throw new RuntimeException(e); - } - } -} diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/runner/CommandLineArguments.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/runner/CommandLineArguments.java index 8a2a1652b4a..06affe9fe1e 100644 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/runner/CommandLineArguments.java +++ b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/runner/CommandLineArguments.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.http.client.runner; import com.google.common.base.Splitter; +import com.yahoo.security.SslContextBuilder; import com.yahoo.vespa.http.client.config.Cluster; import com.yahoo.vespa.http.client.config.ConnectionParams; import com.yahoo.vespa.http.client.config.Endpoint; @@ -18,8 +19,10 @@ import org.apache.http.conn.ssl.SSLConnectionSocketFactory; import org.apache.http.message.BasicLineParser; import javax.inject.Inject; +import javax.net.ssl.SSLContext; import java.net.MalformedURLException; import java.net.URL; +import java.nio.file.Paths; import java.time.Duration; import java.util.ArrayList; import java.util.List; @@ -82,6 +85,12 @@ public class CommandLineArguments { } } + if (cmdArgs.privateKeyPath == null && cmdArgs.certificatePath != null || + cmdArgs.privateKeyPath != null && cmdArgs.certificatePath == null) { + System.err.println("Both '--privateKey' and '--certificate' must be set"); + return null; + } + return cmdArgs; } @@ -204,7 +213,7 @@ public class CommandLineArguments { description = "Use TLS when connecting to endpoint") private boolean useTls = false; - @Option(name = {"--insecure"}, + @Option(name = {"--insecure", "--disable-hostname-verification"}, description = "Skip hostname verification when using TLS") private boolean insecure = false; @@ -220,6 +229,18 @@ public class CommandLineArguments { description = "Maximum time to live for persistent connections. Specified as integer, in seconds.") private long connectionTimeToLive = 15; + @Option(name = {"--certificate"}, + description = "Path to a file containing a PEM encoded x509 certificate") + private String certificatePath; + + @Option(name = {"--privateKey"}, + description = "Path to a file containing a PEM encoded private key") + private String privateKeyPath; + + @Option(name = "--caCertificates", + description = "Path to a file containing a PEM encoded CA certificates") + private String caCertificatesPath; + private final List<Header> parsedHeaders = new ArrayList<>(); int getWhenVerboseEnabledPrintMessageForEveryXDocuments() { @@ -232,6 +253,17 @@ public class CommandLineArguments { public boolean getAddRootElementToXml() { return addRootElementToXml; } + private SSLContext createSslContext() { + SslContextBuilder builder = new SslContextBuilder(); + if (privateKeyPath != null && certificatePath != null) { + builder.withKeyStore(Paths.get(privateKeyPath), Paths.get(certificatePath)); + } + if (caCertificatesPath != null) { + builder.withTrustStore(Paths.get(caCertificatesPath)); + } + return builder.build(); + } + SessionParams createSessionParams(boolean useJson) { final int minThrottleValue = useDynamicThrottlingArg ? 10 : 0; ConnectionParams.Builder connectionParamsBuilder = new ConnectionParams.Builder(); @@ -263,6 +295,7 @@ public class CommandLineArguments { .setTraceEveryXOperation(traceEveryXOperation) .setPrintTraceToStdErr(traceArg > 0) .setNumPersistentConnectionsPerEndpoint(numPersistentConnectionsPerEndpoint) + .setSslContext(createSslContext()) .setUseTlsConfigFromEnvironment(useTlsConfigFromEnvironment) .setConnectionTimeToLive(Duration.ofSeconds(connectionTimeToLive)) .build() diff --git a/zookeeper-server/zookeeper-server-3.5/src/test/java/com/yahoo/vespa/zookeeper/VespaZooKeeperServerImplTest.java b/zookeeper-server/zookeeper-server-3.5/src/test/java/com/yahoo/vespa/zookeeper/VespaZooKeeperServerImplTest.java index 19166047fb1..386c0dff85a 100644 --- a/zookeeper-server/zookeeper-server-3.5/src/test/java/com/yahoo/vespa/zookeeper/VespaZooKeeperServerImplTest.java +++ b/zookeeper-server/zookeeper-server-3.5/src/test/java/com/yahoo/vespa/zookeeper/VespaZooKeeperServerImplTest.java @@ -220,7 +220,7 @@ public class VespaZooKeeperServerImplTest { "ssl.quorum.clientAuth=NEED\n" + "ssl.quorum.ciphersuites=TLS_AES_128_GCM_SHA256,TLS_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384\n" + "ssl.quorum.enabledProtocols=TLSv1.2\n" + - "ssl.quorum.protocol=TLSv1.2\n"; + "ssl.quorum.protocol=TLS\n"; } private String commonTlsClientServerConfig() { @@ -228,7 +228,7 @@ public class VespaZooKeeperServerImplTest { "ssl.clientAuth=NEED\n" + "ssl.ciphersuites=TLS_AES_128_GCM_SHA256,TLS_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384\n" + "ssl.enabledProtocols=TLSv1.2\n" + - "ssl.protocol=TLSv1.2\n"; + "ssl.protocol=TLS\n"; } private void validateConfigFileMultipleHosts(File cfgFile) throws IOException { |