diff options
| author | Henning Baldersheim <balder@yahoo-inc.com> | 2018-06-12 11:48:38 +0200 |
|---|---|---|
| committer | GitHub <noreply@github.com> | 2018-06-12 11:48:38 +0200 |
| commit | 23d9213d4f58410cd78410dc10950b3bab86c563 (patch) | |
| tree | 68f4e14449fe5581fde1a950380286c3d973e3f5 | |
| parent | 49ea989662f402f6c96737636b497724a4bb457f (diff) | |
| parent | 06d3ba8c9e0edea16dafce70c13280330c4ab59f (diff) | |
Merge branch 'master' into balder/reduce-code-visibility-rebased
78 files changed, 289 insertions, 621 deletions
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/IdentityProvider.java b/config-model/src/main/java/com/yahoo/vespa/model/container/IdentityProvider.java index fbfff408cb7..874a7933fbe 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/IdentityProvider.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/IdentityProvider.java @@ -4,6 +4,8 @@ package com.yahoo.vespa.model.container; import com.yahoo.config.provision.AthenzDomain; import com.yahoo.config.provision.AthenzService; import com.yahoo.config.provision.HostName; +import com.yahoo.config.provision.SystemName; +import com.yahoo.config.provision.Zone; import com.yahoo.container.bundle.BundleInstantiationSpecification; import com.yahoo.container.core.identity.IdentityConfig; import com.yahoo.osgi.provider.model.ComponentModel; @@ -23,14 +25,21 @@ public class IdentityProvider extends SimpleComponent implements IdentityConfig. private final HostName loadBalancerName; private final URI ztsUrl; private final String athenzDnsSuffix; + private final Zone zone; - public IdentityProvider(AthenzDomain domain, AthenzService service, HostName loadBalancerName, URI ztsUrl, String athenzDnsSuffix) { + public IdentityProvider(AthenzDomain domain, + AthenzService service, + HostName loadBalancerName, + URI ztsUrl, + String athenzDnsSuffix, + Zone zone) { super(new ComponentModel(BundleInstantiationSpecification.getFromStrings(CLASS, CLASS, BUNDLE))); this.domain = domain; this.service = service; this.loadBalancerName = loadBalancerName; this.ztsUrl = ztsUrl; this.athenzDnsSuffix = athenzDnsSuffix; + this.zone = zone; } @Override @@ -42,5 +51,15 @@ public class IdentityProvider extends SimpleComponent implements IdentityConfig. builder.loadBalancerAddress(loadBalancerName.value()); builder.ztsUrl(ztsUrl != null ? ztsUrl.toString() : ""); builder.athenzDnsSuffix(athenzDnsSuffix != null ? athenzDnsSuffix : ""); + builder.nodeIdentityName("vespa.vespa.tenant"); // TODO Move to Oath configmodel amender + builder.configserverIdentityName(getConfigserverIdentityName()); + } + + // TODO Move to Oath configmodel amender + private String getConfigserverIdentityName() { + return String.format("%s.provider_%s_%s", + zone.system() == SystemName.main ? "vespa.vespa" : "vespa.vespa.cd", + zone.environment().value(), + zone.region().value()); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java index 46d968554d8..2572b0d772b 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java @@ -759,7 +759,8 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { spec.athenzDomain().ifPresent(domain -> { AthenzService service = spec.athenzService(zone.environment(), zone.region()) .orElseThrow(() -> new RuntimeException("Missing Athenz service configuration")); - IdentityProvider identityProvider = new IdentityProvider(domain, service, getLoadBalancerName(loadBalancerName, configServerSpecs), ztsUrl, athenzDnsSuffix); + String zoneDnsSuffix = zone.environment().value() + "-" + zone.region().value() + "." + athenzDnsSuffix; + IdentityProvider identityProvider = new IdentityProvider(domain, service, getLoadBalancerName(loadBalancerName, configServerSpecs), ztsUrl, zoneDnsSuffix, zone); cluster.addComponent(identityProvider); cluster.getContainers().forEach(container -> { diff --git a/config/src/.gitignore b/config/src/.gitignore index 528c8f7183d..2e8e6fd906a 100644 --- a/config/src/.gitignore +++ b/config/src/.gitignore @@ -1,4 +1,3 @@ /Makefile.ini -/config.mak /config_command.sh /project.dsw diff --git a/configd/src/.gitignore b/configd/src/.gitignore index 2129b57c8a4..a39df0815b3 100644 --- a/configd/src/.gitignore +++ b/configd/src/.gitignore @@ -1,4 +1,3 @@ Makefile.ini config_command.sh -configd.mak project.dsw diff --git a/configdefinitions/src/.gitignore b/configdefinitions/src/.gitignore index 02d2647f806..a39df0815b3 100644 --- a/configdefinitions/src/.gitignore +++ b/configdefinitions/src/.gitignore @@ -1,4 +1,3 @@ Makefile.ini config_command.sh -configdefinitions.mak project.dsw diff --git a/configutil/src/.gitignore b/configutil/src/.gitignore index f7e40c87b26..2e8e6fd906a 100644 --- a/configutil/src/.gitignore +++ b/configutil/src/.gitignore @@ -1,4 +1,3 @@ /Makefile.ini /config_command.sh -/configutil.mak /project.dsw diff --git a/container-core/src/main/resources/configdefinitions/identity.def b/container-core/src/main/resources/configdefinitions/identity.def index 6c722868e61..539bf07fa7d 100644 --- a/container-core/src/main/resources/configdefinitions/identity.def +++ b/container-core/src/main/resources/configdefinitions/identity.def @@ -1,8 +1,12 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. namespace=container.core.identity +# Tenant identity domain name domain string +# Tenant identity service name service string +nodeIdentityName string +configserverIdentityName string loadBalancerAddress string athenzDnsSuffix string ztsUrl string diff --git a/defaults/src/.gitignore b/defaults/src/.gitignore index 3e338e617f9..d4590f3a58c 100644 --- a/defaults/src/.gitignore +++ b/defaults/src/.gitignore @@ -1,3 +1,2 @@ Makefile.ini config_command.sh -defaults.mak diff --git a/document/src/.gitignore b/document/src/.gitignore index 12cd838c514..a39df0815b3 100644 --- a/document/src/.gitignore +++ b/document/src/.gitignore @@ -1,4 +1,3 @@ Makefile.ini config_command.sh -document.mak project.dsw diff --git a/documentapi/src/.gitignore b/documentapi/src/.gitignore index afa6b2d43c5..a39df0815b3 100644 --- a/documentapi/src/.gitignore +++ b/documentapi/src/.gitignore @@ -1,4 +1,3 @@ Makefile.ini config_command.sh -documentapi.mak project.dsw diff --git a/fastlib/src/.gitignore b/fastlib/src/.gitignore index b5b26fa8473..9d5b23b7747 100644 --- a/fastlib/src/.gitignore +++ b/fastlib/src/.gitignore @@ -6,5 +6,4 @@ Makefile.ini config_command.bat config_command.sh -fastlib.mak output diff --git a/fastos/src/.gitignore b/fastos/src/.gitignore index 87807fb815c..2e8e6fd906a 100644 --- a/fastos/src/.gitignore +++ b/fastos/src/.gitignore @@ -1,4 +1,3 @@ /Makefile.ini /config_command.sh -/fastos.mak /project.dsw diff --git a/fnet/src/.gitignore b/fnet/src/.gitignore index 516a0a876bc..7bf7b0038f0 100644 --- a/fnet/src/.gitignore +++ b/fnet/src/.gitignore @@ -1,7 +1,6 @@ *.dsp Makefile.ini config_command.sh -fnet.mak project.dsw xsync.log /gen diff --git a/fsa/src/.gitignore b/fsa/src/.gitignore index 65ad4d24f75..b4cda6d3af4 100644 --- a/fsa/src/.gitignore +++ b/fsa/src/.gitignore @@ -3,4 +3,3 @@ test.out /Makefile.ini /config_command.sh -/fsa.mak diff --git a/juniper/.gitignore b/juniper/.gitignore index 0c9cc229d0e..a106509440d 100644 --- a/juniper/.gitignore +++ b/juniper/.gitignore @@ -2,7 +2,6 @@ Makefile.ini config_command.sh include -juniper.mak lib project.dsw Makefile diff --git a/messagebus/src/.gitignore b/messagebus/src/.gitignore index 3b9f1ee8e62..7752884b41f 100644 --- a/messagebus/src/.gitignore +++ b/messagebus/src/.gitignore @@ -1,5 +1,4 @@ Makefile.ini config_command.sh doxygen -messagebus.mak project.dsw diff --git a/messagebus_test/src/.gitignore b/messagebus_test/src/.gitignore index 8689bfd3624..40fd3439da1 100644 --- a/messagebus_test/src/.gitignore +++ b/messagebus_test/src/.gitignore @@ -4,5 +4,3 @@ config.cfg config_command.sh configure project.dsw -versiontag.mak -/messagebus_test.mak diff --git a/metrics/src/.gitignore b/metrics/src/.gitignore index 698739d7bbc..a39df0815b3 100644 --- a/metrics/src/.gitignore +++ b/metrics/src/.gitignore @@ -1,4 +1,3 @@ Makefile.ini config_command.sh project.dsw -/metrics.mak diff --git a/persistence/src/.gitignore b/persistence/src/.gitignore index ea5bfd4b499..2e8e6fd906a 100644 --- a/persistence/src/.gitignore +++ b/persistence/src/.gitignore @@ -1,4 +1,3 @@ /Makefile.ini /config_command.sh /project.dsw -/persistence.mak diff --git a/persistencetypes/src/.gitignore b/persistencetypes/src/.gitignore index b17e583dfac..2e8e6fd906a 100644 --- a/persistencetypes/src/.gitignore +++ b/persistencetypes/src/.gitignore @@ -1,4 +1,3 @@ /Makefile.ini /config_command.sh -/persistencetypes.mak /project.dsw diff --git a/searchcommon/src/.gitignore b/searchcommon/src/.gitignore index 8b68901f2ce..2e8e6fd906a 100644 --- a/searchcommon/src/.gitignore +++ b/searchcommon/src/.gitignore @@ -1,4 +1,3 @@ /Makefile.ini /config_command.sh /project.dsw -/searchcommon.mak diff --git a/searchcore/src/tests/proton/documentdb/feedhandler/feedhandler_test.cpp b/searchcore/src/tests/proton/documentdb/feedhandler/feedhandler_test.cpp index 0a73e0b2b6e..e774728b41e 100644 --- a/searchcore/src/tests/proton/documentdb/feedhandler/feedhandler_test.cpp +++ b/searchcore/src/tests/proton/documentdb/feedhandler/feedhandler_test.cpp @@ -28,6 +28,7 @@ #include <vespa/searchlib/transactionlog/translogserver.h> #include <vespa/vespalib/testkit/testapp.h> #include <vespa/vespalib/util/closuretask.h> +#include <vespa/vespalib/util/lambdatask.h> #include <vespa/vespalib/util/exceptions.h> #include <vespa/vespalib/io/fileutil.h> diff --git a/searchcore/src/tests/proton/index/fusionrunner_test.cpp b/searchcore/src/tests/proton/index/fusionrunner_test.cpp index e6efa246484..429452df2ec 100644 --- a/searchcore/src/tests/proton/index/fusionrunner_test.cpp +++ b/searchcore/src/tests/proton/index/fusionrunner_test.cpp @@ -11,6 +11,7 @@ #include <vespa/searchlib/index/docbuilder.h> #include <vespa/searchlib/index/dummyfileheadercontext.h> #include <vespa/searchlib/query/tree/simplequery.h> +#include <vespa/searchlib/common/isequencedtaskexecutor.h> #include <vespa/vespalib/testkit/testapp.h> #include <vespa/fastos/file.h> #include <set> diff --git a/searchcore/src/versiontag.mak b/searchcore/src/versiontag.mak deleted file mode 100644 index 3f8d09a48e0..00000000000 --- a/searchcore/src/versiontag.mak +++ /dev/null @@ -1,34 +0,0 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -VTAG_DATE=$(shell date +%Y.%m.%d-%H.%M.%S) -VTAG_SYSTEM=$(shell uname -s) -VTAG_SYSTEM_REV=$(shell uname -r) -VTAG_BUILDER=$(shell (whoami) 2>/dev/null||logname)@$(shell uname -n) -ifneq (X$(SPECIFIED_VTAG),XDISABLE) - ifeq (X$(UNAME), XWin32) - VTAG=-DV_TAG='\"$(SPECIFIED_VTAG)\"' - else - VTAG=-DV_TAG='"$(SPECIFIED_VTAG)"' - endif -else - ifeq (X$(UNAME), XWin32) - VTAG= - else - VTAG_TAG=$(shell cat $(TOP)/CVS/Tag 2>/dev/null | sed "s/^.//" 2>/dev/null) - ifeq (X$(VTAG_TAG),X) - VTAG_TAG=CURRENT - endif - ifeq ($(findstring _RELEASE, $(VTAG_TAG)),_RELEASE) - VTAG_SYSTEM=$(shell uname -s) - VTAG=-DV_TAG='"$(VTAG_TAG)-$(VTAG_SYSTEM)"' - else - VTAG_DATE=$(shell date +%Y.%m.%d-%H:%M:%S) - VTAG_SYSTEM=$(shell (whoami) 2>/dev/null||logname)@$(shell uname -n)-$(shell uname -s)-$(shell uname -r) - VTAG=-DV_TAG='"$(VTAG_TAG)-$(VTAG_SYSTEM)-$(VTAG_DATE)"' - endif - endif -endif -VTAG+= -DV_TAG_DATE='"$(VTAG_DATE)"' -VTAG+= -DV_TAG_SYSTEM='"$(VTAG_SYSTEM)"' -VTAG+= -DV_TAG_SYSTEM_REV='"$(VTAG_SYSTEM_REV)"' -VTAG+= -DV_TAG_BUILDER='"$(VTAG_BUILDER)"' - diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.cpp index 2d78f837ecf..239f91b449f 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.cpp +++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.cpp @@ -21,6 +21,7 @@ LOG_SETUP(".proton.server.attributeadapter"); using namespace document; using namespace search; using search::attribute::ImportedAttributeVector; +using ExecutorId = search::ISequencedTaskExecutor::ExecutorId; namespace proton { @@ -50,7 +51,7 @@ AttributeWriter::WriteField::buildFieldPath(const DocumentType &docType) _fieldPath = std::move(fp); } -AttributeWriter::WriteContext::WriteContext(uint32_t executorId) +AttributeWriter::WriteContext::WriteContext(ExecutorId executorId) : _executorId(executorId), _fields(), _hasStructFieldAttribute(false) @@ -214,15 +215,15 @@ struct BatchUpdateTask : public vespalib::Executor::Task { class FieldContext { - vespalib::string _name; - uint32_t _executorId; - AttributeVector *_attr; + vespalib::string _name; + ExecutorId _executorId; + AttributeVector *_attr; public: FieldContext(ISequencedTaskExecutor &writer, AttributeVector *attr); ~FieldContext(); bool operator<(const FieldContext &rhs) const; - uint32_t getExecutorId() const { return _executorId; } + ExecutorId getExecutorId() const { return _executorId; } AttributeVector *getAttribute() const { return _attr; } }; @@ -554,7 +555,7 @@ AttributeWriter::update(SerialNum serialNum, const DocumentUpdate &upd, Document // document and attribute. if (attrp->getStatus().getLastSyncToken() >= serialNum) continue; - args[_attributeFieldWriter.getExecutorId(attrp->getName())]->_updates.emplace_back(attrp, &fupd); + args[_attributeFieldWriter.getExecutorId(attrp->getName()).getId()]->_updates.emplace_back(attrp, &fupd); LOG(debug, "About to apply update for docId %u in attribute vector '%s'.", lid, attrp->getName().c_str()); } // NOTE: The lifetime of the field update will be ensured by keeping the document update alive @@ -562,7 +563,7 @@ AttributeWriter::update(SerialNum serialNum, const DocumentUpdate &upd, Document for (uint32_t id(0); id < args.size(); id++) { if ( ! args[id]->_updates.empty()) { args[id]->_onWriteDone = onWriteDone; - _attributeFieldWriter.executeTask(id, std::move(args[id])); + _attributeFieldWriter.executeTask(ExecutorId(id), std::move(args[id])); } } diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.h b/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.h index bed2a7cb23d..f89089ed335 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.h +++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.h @@ -5,6 +5,7 @@ #include "i_attribute_writer.h" #include <vespa/searchcore/proton/common/commit_time_tracker.h> #include <vespa/document/base/fieldpath.h> +#include <vespa/searchlib/common/isequencedtaskexecutor.h> namespace document { class DocumentType; } @@ -25,6 +26,7 @@ private: const IAttributeManager::SP _mgr; search::ISequencedTaskExecutor &_attributeFieldWriter; const std::vector<search::AttributeVector *> &_writableAttributes; + using ExecutorId = search::ISequencedTaskExecutor::ExecutorId; public: class WriteField { @@ -41,17 +43,17 @@ public: }; class WriteContext { - uint32_t _executorId; + ExecutorId _executorId; std::vector<WriteField> _fields; bool _hasStructFieldAttribute; public: - WriteContext(uint32_t executorId); + WriteContext(ExecutorId executorId); WriteContext(WriteContext &&rhs); ~WriteContext(); WriteContext &operator=(WriteContext &&rhs); void buildFieldPaths(const DocumentType &docType); void add(AttributeVector &attr); - uint32_t getExecutorId() const { return _executorId; } + ExecutorId getExecutorId() const { return _executorId; } const std::vector<WriteField> &getFields() const { return _fields; } bool hasStructFieldAttribute() const { return _hasStructFieldAttribute; } }; diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.cpp index b4f087c65c0..ef818f7b407 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.cpp +++ b/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.cpp @@ -87,8 +87,7 @@ std::shared_ptr<ShrinkLidSpaceFlushTarget> allocShrinker(const AttributeVector:: } -AttributeManager::AttributeWrap::AttributeWrap(const AttributeVectorSP & a, - bool isExtra_) +AttributeManager::AttributeWrap::AttributeWrap(const AttributeVectorSP & a, bool isExtra_) : _attr(a), _isExtra(isExtra_) { @@ -100,9 +99,7 @@ AttributeManager::AttributeWrap::AttributeWrap() { } -AttributeManager::AttributeWrap::~AttributeWrap() -{ -} +AttributeManager::AttributeWrap::~AttributeWrap() = default; AttributeManager::AttributeWrap AttributeManager::AttributeWrap::extraAttribute(const AttributeVectorSP &a) @@ -128,9 +125,7 @@ AttributeManager::FlushableWrap::FlushableWrap(FlushableAttributeSP flusher, Shr { } -AttributeManager::FlushableWrap::~FlushableWrap() -{ -} +AttributeManager::FlushableWrap::~FlushableWrap() = default; AttributeVector::SP AttributeManager::internalAddAttribute(const AttributeSpec &spec, @@ -168,7 +163,7 @@ AttributeManager::findAttribute(const vespalib::string &name) const { AttributeMap::const_iterator itr = _attributes.find(name); return (itr != _attributes.end()) - ? static_cast<const AttributeVector::SP &>(itr->second.getAttribute()) + ? itr->second.getAttribute() : AttributeVector::SP(); } diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.h b/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.h index b8f245d9c68..f904c1b4b53 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.h +++ b/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.h @@ -63,7 +63,7 @@ private: static AttributeWrap extraAttribute(const AttributeVectorSP &a); static AttributeWrap normalAttribute(const AttributeVectorSP &a); bool isExtra() const { return _isExtra; } - const AttributeVectorSP getAttribute() const { return _attr; } + const AttributeVectorSP & getAttribute() const { return _attr; } }; class FlushableWrap diff --git a/searchcore/src/vespa/searchcore/proton/reference/gid_to_lid_change_listener.h b/searchcore/src/vespa/searchcore/proton/reference/gid_to_lid_change_listener.h index 0e9298e3e51..d4c349bd1d7 100644 --- a/searchcore/src/vespa/searchcore/proton/reference/gid_to_lid_change_listener.h +++ b/searchcore/src/vespa/searchcore/proton/reference/gid_to_lid_change_listener.h @@ -7,7 +7,6 @@ #include <vespa/searchlib/common/sequencedtaskexecutor.h> #include <vespa/searchcore/proton/common/monitored_refcount.h> - namespace proton { /* @@ -16,12 +15,12 @@ namespace proton { */ class GidToLidChangeListener : public IGidToLidChangeListener { - search::ISequencedTaskExecutor &_attributeFieldWriter; - uint32_t _executorId; + search::ISequencedTaskExecutor &_attributeFieldWriter; + search::ISequencedTaskExecutor::ExecutorId _executorId; std::shared_ptr<search::attribute::ReferenceAttribute> _attr; - MonitoredRefCount &_refCount; - vespalib::string _name; - vespalib::string _docTypeName; + MonitoredRefCount &_refCount; + vespalib::string _name; + vespalib::string _docTypeName; public: GidToLidChangeListener(search::ISequencedTaskExecutor &attributeFieldWriter, diff --git a/searchcore/src/vespa/searchcore/proton/server/executor_thread_service.h b/searchcore/src/vespa/searchcore/proton/server/executor_thread_service.h index c938288c714..4b1e8408c8e 100644 --- a/searchcore/src/vespa/searchcore/proton/server/executor_thread_service.h +++ b/searchcore/src/vespa/searchcore/proton/server/executor_thread_service.h @@ -24,15 +24,15 @@ public: /** * Implements IThreadService */ - virtual vespalib::Executor::Task::UP execute(vespalib::Executor::Task::UP task) override { + vespalib::Executor::Task::UP execute(vespalib::Executor::Task::UP task) override { return _executor.execute(std::move(task)); } - virtual void run(vespalib::Runnable &runnable) override; - virtual vespalib::Syncable &sync() override { + void run(vespalib::Runnable &runnable) override; + vespalib::Syncable &sync() override { _executor.sync(); return *this; } - virtual bool isCurrentThread() const override; + bool isCurrentThread() const override; size_t getNumThreads() const override { return _executor.getNumThreads(); } }; diff --git a/searchcore/src/vespa/searchcore/proton/server/executorthreadingservice.cpp b/searchcore/src/vespa/searchcore/proton/server/executorthreadingservice.cpp index 1c4825753a8..3d9525f0471 100644 --- a/searchcore/src/vespa/searchcore/proton/server/executorthreadingservice.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/executorthreadingservice.cpp @@ -1,16 +1,15 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "executorthreadingservice.h" -#include <vespa/vespalib/util/executor.h> #include <vespa/searchcore/proton/metrics/executor_threading_service_stats.h> +#include <vespa/searchlib/common/sequencedtaskexecutor.h> using vespalib::ThreadStackExecutorBase; +using search::SequencedTaskExecutor; namespace proton { -ExecutorThreadingService::ExecutorThreadingService(uint32_t threads, - uint32_t stackSize, - uint32_t taskLimit) +ExecutorThreadingService::ExecutorThreadingService(uint32_t threads, uint32_t stackSize, uint32_t taskLimit) : _masterExecutor(1, stackSize), _indexExecutor(1, stackSize, taskLimit), @@ -18,14 +17,13 @@ ExecutorThreadingService::ExecutorThreadingService(uint32_t threads, _masterService(_masterExecutor), _indexService(_indexExecutor), _summaryService(_summaryExecutor), - _indexFieldInverter(threads, taskLimit), - _indexFieldWriter(threads, taskLimit), - _attributeFieldWriter(threads, taskLimit) + _indexFieldInverter(std::make_unique<SequencedTaskExecutor>(threads, taskLimit)), + _indexFieldWriter(std::make_unique<SequencedTaskExecutor>(threads, taskLimit)), + _attributeFieldWriter(std::make_unique<SequencedTaskExecutor>(threads, taskLimit)) { } -ExecutorThreadingService::~ExecutorThreadingService() { -} +ExecutorThreadingService::~ExecutorThreadingService() = default; vespalib::Syncable & ExecutorThreadingService::sync() @@ -34,11 +32,11 @@ ExecutorThreadingService::sync() if (!isMasterThread) { _masterExecutor.sync(); } - _attributeFieldWriter.sync(); + _attributeFieldWriter->sync(); _indexExecutor.sync(); _summaryExecutor.sync(); - _indexFieldInverter.sync(); - _indexFieldWriter.sync(); + _indexFieldInverter->sync(); + _indexFieldWriter->sync(); if (!isMasterThread) { _masterExecutor.sync(); } @@ -50,13 +48,13 @@ ExecutorThreadingService::shutdown() { _masterExecutor.shutdown(); _masterExecutor.sync(); - _attributeFieldWriter.sync(); + _attributeFieldWriter->sync(); _summaryExecutor.shutdown(); _summaryExecutor.sync(); _indexExecutor.shutdown(); _indexExecutor.sync(); - _indexFieldInverter.sync(); - _indexFieldWriter.sync(); + _indexFieldInverter->sync(); + _indexFieldWriter->sync(); } void @@ -64,9 +62,9 @@ ExecutorThreadingService::setTaskLimit(uint32_t taskLimit, uint32_t summaryTaskL { _indexExecutor.setTaskLimit(taskLimit); _summaryExecutor.setTaskLimit(summaryTaskLimit); - _indexFieldInverter.setTaskLimit(taskLimit); - _indexFieldWriter.setTaskLimit(taskLimit); - _attributeFieldWriter.setTaskLimit(taskLimit); + _indexFieldInverter->setTaskLimit(taskLimit); + _indexFieldWriter->setTaskLimit(taskLimit); + _attributeFieldWriter->setTaskLimit(taskLimit); } ExecutorThreadingServiceStats @@ -75,9 +73,24 @@ ExecutorThreadingService::getStats() return ExecutorThreadingServiceStats(_masterExecutor.getStats(), _indexExecutor.getStats(), _summaryExecutor.getStats(), - _indexFieldInverter.getStats(), - _indexFieldWriter.getStats(), - _attributeFieldWriter.getStats()); + _indexFieldInverter->getStats(), + _indexFieldWriter->getStats(), + _attributeFieldWriter->getStats()); +} + +search::ISequencedTaskExecutor & +ExecutorThreadingService::indexFieldInverter() { + return *_indexFieldInverter; +} + +search::ISequencedTaskExecutor & +ExecutorThreadingService::indexFieldWriter() { + return *_indexFieldWriter; +} + +search::ISequencedTaskExecutor & +ExecutorThreadingService::attributeFieldWriter() { + return *_attributeFieldWriter; } } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/server/executorthreadingservice.h b/searchcore/src/vespa/searchcore/proton/server/executorthreadingservice.h index 25aa65d43a7..240eefab2a7 100644 --- a/searchcore/src/vespa/searchcore/proton/server/executorthreadingservice.h +++ b/searchcore/src/vespa/searchcore/proton/server/executorthreadingservice.h @@ -5,8 +5,8 @@ #include <vespa/searchcorespi/index/ithreadingservice.h> #include <vespa/vespalib/util/blockingthreadstackexecutor.h> #include <vespa/vespalib/util/threadstackexecutor.h> -#include <vespa/searchlib/common/sequencedtaskexecutor.h> +namespace search { class SequencedTaskExecutor; } namespace proton { class ExecutorThreadingServiceStats; @@ -24,9 +24,9 @@ private: ExecutorThreadService _masterService; ExecutorThreadService _indexService; ExecutorThreadService _summaryService; - search::SequencedTaskExecutor _indexFieldInverter; - search::SequencedTaskExecutor _indexFieldWriter; - search::SequencedTaskExecutor _attributeFieldWriter; + std::unique_ptr<search::SequencedTaskExecutor> _indexFieldInverter; + std::unique_ptr<search::SequencedTaskExecutor> _indexFieldWriter; + std::unique_ptr<search::SequencedTaskExecutor> _attributeFieldWriter; public: /** @@ -38,12 +38,12 @@ public: ExecutorThreadingService(uint32_t threads = 1, uint32_t stackSize = 128 * 1024, uint32_t taskLimit = 1000); - ~ExecutorThreadingService(); + ~ExecutorThreadingService() override; /** * Implements vespalib::Syncable */ - virtual vespalib::Syncable &sync() override; + vespalib::Syncable &sync() override; void shutdown(); @@ -63,29 +63,20 @@ public: /** * Implements IThreadingService */ - virtual searchcorespi::index::IThreadService &master() override { + searchcorespi::index::IThreadService &master() override { return _masterService; } - virtual searchcorespi::index::IThreadService &index() override { + searchcorespi::index::IThreadService &index() override { return _indexService; } - virtual searchcorespi::index::IThreadService &summary() override { + searchcorespi::index::IThreadService &summary() override { return _summaryService; } - virtual search::ISequencedTaskExecutor &indexFieldInverter() override { - return _indexFieldInverter; - } - - virtual search::ISequencedTaskExecutor &indexFieldWriter() override { - return _indexFieldWriter; - } - - virtual search::ISequencedTaskExecutor &attributeFieldWriter() override { - return _attributeFieldWriter; - } - + search::ISequencedTaskExecutor &indexFieldInverter() override; + search::ISequencedTaskExecutor &indexFieldWriter() override; + search::ISequencedTaskExecutor &attributeFieldWriter() override; ExecutorThreadingServiceStats getStats(); }; diff --git a/searchcore/src/vespa/searchcore/proton/server/fast_access_feed_view.cpp b/searchcore/src/vespa/searchcore/proton/server/fast_access_feed_view.cpp index 858e1d51d3f..6364e772f94 100644 --- a/searchcore/src/vespa/searchcore/proton/server/fast_access_feed_view.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/fast_access_feed_view.cpp @@ -5,6 +5,7 @@ #include "operationdonecontext.h" #include "removedonecontext.h" #include "putdonecontext.h" +#include <vespa/searchlib/common/isequencedtaskexecutor.h> using document::Document; using document::DocumentUpdate; diff --git a/searchcore/src/vespa/searchcore/proton/server/feedhandler.cpp b/searchcore/src/vespa/searchcore/proton/server/feedhandler.cpp index e8dc4eecbf3..66e721f9222 100644 --- a/searchcore/src/vespa/searchcore/proton/server/feedhandler.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/feedhandler.cpp @@ -18,6 +18,7 @@ #include <vespa/searchcorespi/index/ithreadingservice.h> #include <vespa/searchlib/common/gatecallback.h> #include <vespa/vespalib/util/exceptions.h> +#include <vespa/vespalib/util/lambdatask.h> #include <unistd.h> #include <vespa/log/log.h> diff --git a/searchcore/src/vespa/searchcore/proton/server/searchable_feed_view.cpp b/searchcore/src/vespa/searchcore/proton/server/searchable_feed_view.cpp index 4cda07eee8b..28b1c407429 100644 --- a/searchcore/src/vespa/searchcore/proton/server/searchable_feed_view.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/searchable_feed_view.cpp @@ -6,6 +6,7 @@ #include "removedonecontext.h" #include <vespa/searchcore/proton/common/feedtoken.h> #include <vespa/searchcore/proton/documentmetastore/ilidreusedelayer.h> +#include <vespa/searchlib/common/isequencedtaskexecutor.h> #include <vespa/vespalib/text/stringtokenizer.h> #include <vespa/vespalib/util/closuretask.h> #include <vespa/vespalib/util/exceptions.h> diff --git a/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.cpp b/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.cpp index ab47a72e899..29615b0daf9 100644 --- a/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.cpp @@ -12,14 +12,15 @@ #include <vespa/searchcore/proton/common/feedtoken.h> #include <vespa/searchcore/proton/documentmetastore/ilidreusedelayer.h> #include <vespa/searchcore/proton/reference/i_gid_to_lid_change_handler.h> +#include <vespa/searchcore/proton/attribute/ifieldupdatecallback.h> + +#include <vespa/searchlib/common/isequencedtaskexecutor.h> #include <vespa/document/datatype/documenttype.h> #include <vespa/document/repo/documenttyperepo.h> #include <vespa/document/fieldvalue/document.h> #include <vespa/vespalib/util/exceptions.h> #include <vespa/log/log.h> -#include <vespa/searchcore/proton/attribute/ifieldupdatecallback.h> - LOG_SETUP(".proton.server.storeonlyfeedview"); using document::BucketId; diff --git a/searchcore/src/vespa/searchcore/proton/server/visibilityhandler.cpp b/searchcore/src/vespa/searchcore/proton/server/visibilityhandler.cpp index dd12a5cfd3d..71197a5c530 100644 --- a/searchcore/src/vespa/searchcore/proton/server/visibilityhandler.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/visibilityhandler.cpp @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "visibilityhandler.h" +#include <vespa/searchlib/common/isequencedtaskexecutor.h> #include <vespa/vespalib/util/closuretask.h> using vespalib::makeTask; diff --git a/searchcorespi/src/.gitignore b/searchcorespi/src/.gitignore index 49bfd9b85d3..2e8e6fd906a 100644 --- a/searchcorespi/src/.gitignore +++ b/searchcorespi/src/.gitignore @@ -1,4 +1,3 @@ /Makefile.ini /config_command.sh /project.dsw -/searchcorespi.mak diff --git a/searchcorespi/src/vespa/searchcorespi/index/ithreadingservice.h b/searchcorespi/src/vespa/searchcorespi/index/ithreadingservice.h index bded09143ab..a4b5277b616 100644 --- a/searchcorespi/src/vespa/searchcorespi/index/ithreadingservice.h +++ b/searchcorespi/src/vespa/searchcorespi/index/ithreadingservice.h @@ -2,11 +2,9 @@ #pragma once #include "i_thread_service.h" -#include <vespa/vespalib/util/runnable.h> -#include <vespa/vespalib/util/executor.h> #include <vespa/vespalib/util/syncable.h> -#include <vespa/searchlib/common/isequencedtaskexecutor.h> +namespace search { class ISequencedTaskExecutor; } namespace searchcorespi::index { /** diff --git a/searchlib/src/.gitignore b/searchlib/src/.gitignore index 3e2fb17989e..b7456dab392 100644 --- a/searchlib/src/.gitignore +++ b/searchlib/src/.gitignore @@ -1,5 +1,4 @@ *.dsp -*.mak Makefile.ini config_command.sh html diff --git a/searchlib/src/tests/common/sequencedtaskexecutor/sequencedtaskexecutor_test.cpp b/searchlib/src/tests/common/sequencedtaskexecutor/sequencedtaskexecutor_test.cpp index 7fd3f33f20a..805a6d3b962 100644 --- a/searchlib/src/tests/common/sequencedtaskexecutor/sequencedtaskexecutor_test.cpp +++ b/searchlib/src/tests/common/sequencedtaskexecutor/sequencedtaskexecutor_test.cpp @@ -161,7 +161,7 @@ vespalib::string makeAltComponentId(Fixture &f) { int tryCnt = 0; char altComponentId[20]; - uint32_t executorId0 = f._threads.getExecutorId("0"); + ISequencedTaskExecutor::ExecutorId executorId0 = f._threads.getExecutorId("0"); for (tryCnt = 1; tryCnt < 100; ++tryCnt) { sprintf(altComponentId, "%d", tryCnt); if (f._threads.getExecutorId(altComponentId) == executorId0) { @@ -227,7 +227,7 @@ TEST_F("require that executeLambda works", Fixture) std::vector<int> res; const auto lambda = [i, &res]() mutable { res.push_back(i--); res.push_back(i--); }; - f._threads.executeLambda(0, lambda); + f._threads.executeLambda(ISequencedTaskExecutor::ExecutorId(0), lambda); f._threads.sync(); std::vector<int> exp({5, 4}); EXPECT_EQUAL(exp, res); diff --git a/searchlib/src/vespa/searchlib/common/foregroundtaskexecutor.cpp b/searchlib/src/vespa/searchlib/common/foregroundtaskexecutor.cpp index 1ab3c6b8b51..91ca91be4cd 100644 --- a/searchlib/src/vespa/searchlib/common/foregroundtaskexecutor.cpp +++ b/searchlib/src/vespa/searchlib/common/foregroundtaskexecutor.cpp @@ -23,12 +23,12 @@ ForegroundTaskExecutor::~ForegroundTaskExecutor() { } -uint32_t +ISequencedTaskExecutor::ExecutorId ForegroundTaskExecutor::getExecutorId(uint64_t componentId) { auto itr = _ids.find(componentId); if (itr == _ids.end()) { - auto insarg = std::make_pair(componentId, _ids.size() % _threads); + auto insarg = std::make_pair(componentId, ExecutorId(_ids.size() % _threads)); auto insres = _ids.insert(insarg); assert(insres.second); itr = insres.first; @@ -37,13 +37,12 @@ ForegroundTaskExecutor::getExecutorId(uint64_t componentId) } void -ForegroundTaskExecutor::executeTask(uint32_t executorId, vespalib::Executor::Task::UP task) +ForegroundTaskExecutor::executeTask(ExecutorId id, vespalib::Executor::Task::UP task) { - assert(executorId < _threads); + assert(id.getId() < _threads); task->run(); } - void ForegroundTaskExecutor::sync() { diff --git a/searchlib/src/vespa/searchlib/common/foregroundtaskexecutor.h b/searchlib/src/vespa/searchlib/common/foregroundtaskexecutor.h index 10743baf216..cfd135d3fa0 100644 --- a/searchlib/src/vespa/searchlib/common/foregroundtaskexecutor.h +++ b/searchlib/src/vespa/searchlib/common/foregroundtaskexecutor.h @@ -17,7 +17,7 @@ namespace search { class ForegroundTaskExecutor : public ISequencedTaskExecutor { const uint32_t _threads; - vespalib::hash_map<size_t, uint32_t> _ids; + vespalib::hash_map<size_t, ExecutorId> _ids; public: using ISequencedTaskExecutor::getExecutorId; @@ -26,8 +26,8 @@ public: ~ForegroundTaskExecutor() override; uint32_t getNumExecutors() const override { return _threads; } - uint32_t getExecutorId(uint64_t componentId) override; - void executeTask(uint32_t executorId, vespalib::Executor::Task::UP task) override; + ExecutorId getExecutorId(uint64_t componentId) override; + void executeTask(ExecutorId id, vespalib::Executor::Task::UP task) override; void sync() override; }; diff --git a/searchlib/src/vespa/searchlib/common/isequencedtaskexecutor.h b/searchlib/src/vespa/searchlib/common/isequencedtaskexecutor.h index 8488d986bbe..05347e790fb 100644 --- a/searchlib/src/vespa/searchlib/common/isequencedtaskexecutor.h +++ b/searchlib/src/vespa/searchlib/common/isequencedtaskexecutor.h @@ -14,6 +14,17 @@ namespace search { class ISequencedTaskExecutor { public: + class ExecutorId { + public: + ExecutorId() : ExecutorId(0) { } + explicit ExecutorId(uint32_t id) : _id(id) { } + uint32_t getId() const { return _id; } + bool operator != (ExecutorId rhs) const { return _id != rhs._id; } + bool operator == (ExecutorId rhs) const { return _id == rhs._id; } + bool operator < (ExecutorId rhs) const { return _id < rhs._id; } + private: + uint32_t _id; + }; virtual ~ISequencedTaskExecutor() { } /** @@ -23,10 +34,10 @@ public: * @param componentId component id * @return executor id */ - virtual uint32_t getExecutorId(uint64_t componentId) = 0; + virtual ExecutorId getExecutorId(uint64_t componentId) = 0; virtual uint32_t getNumExecutors() const = 0; - uint32_t getExecutorId(vespalib::stringref componentId) { + ExecutorId getExecutorId(vespalib::stringref componentId) { vespalib::hash<vespalib::stringref> hashfun; return getExecutorId(hashfun(componentId)); } @@ -35,22 +46,22 @@ public: * Schedule a task to run after all previously scheduled tasks with * same id. * - * @param executorId which internal executor to use - * @param task unique pointer to the task to be executed + * @param id which internal executor to use + * @param task unique pointer to the task to be executed */ - virtual void executeTask(uint32_t exeucutorId, vespalib::Executor::Task::UP task) = 0; + virtual void executeTask(ExecutorId id, vespalib::Executor::Task::UP task) = 0; /** * Wrap lambda function into a task and schedule it to be run. * Caller must ensure that pointers and references are valid and * call sync before tearing down pointed to/referenced data. * - * @param executorId which internal executor to use - * @param function function to be wrapped in a task and later executed + * @param id which internal executor to use + * @param function function to be wrapped in a task and later executed */ template <class FunctionType> - void executeLambda(uint32_t executorId, FunctionType &&function) { - executeTask(executorId, vespalib::makeLambdaTask(std::forward<FunctionType>(function))); + void executeLambda(ExecutorId id, FunctionType &&function) { + executeTask(id, vespalib::makeLambdaTask(std::forward<FunctionType>(function))); } /** * Wait for all scheduled tasks to complete. @@ -68,8 +79,8 @@ public: */ template <class FunctionType> void execute(uint64_t componentId, FunctionType &&function) { - uint32_t executorId = getExecutorId(componentId); - executeTask(executorId, vespalib::makeLambdaTask(std::forward<FunctionType>(function))); + ExecutorId id = getExecutorId(componentId); + executeTask(id, vespalib::makeLambdaTask(std::forward<FunctionType>(function))); } /** @@ -83,8 +94,8 @@ public: */ template <class FunctionType> void execute(vespalib::stringref componentId, FunctionType &&function) { - uint32_t executorId = getExecutorId(componentId); - executeTask(executorId, vespalib::makeLambdaTask(std::forward<FunctionType>(function))); + ExecutorId id = getExecutorId(componentId); + executeTask(id, vespalib::makeLambdaTask(std::forward<FunctionType>(function))); } }; diff --git a/searchlib/src/vespa/searchlib/common/sequencedtaskexecutor.cpp b/searchlib/src/vespa/searchlib/common/sequencedtaskexecutor.cpp index 953449ee496..5306cabba8c 100644 --- a/searchlib/src/vespa/searchlib/common/sequencedtaskexecutor.cpp +++ b/searchlib/src/vespa/searchlib/common/sequencedtaskexecutor.cpp @@ -37,12 +37,12 @@ SequencedTaskExecutor::setTaskLimit(uint32_t taskLimit) } } -uint32_t +ISequencedTaskExecutor::ExecutorId SequencedTaskExecutor::getExecutorId(uint64_t componentId) { auto itr = _ids.find(componentId); if (itr == _ids.end()) { - auto insarg = std::make_pair(componentId, _ids.size() % _executors.size()); + auto insarg = std::make_pair(componentId, ExecutorId(_ids.size() % _executors.size())); auto insres = _ids.insert(insarg); assert(insres.second); itr = insres.first; @@ -51,10 +51,10 @@ SequencedTaskExecutor::getExecutorId(uint64_t componentId) } void -SequencedTaskExecutor::executeTask(uint32_t executorId, vespalib::Executor::Task::UP task) +SequencedTaskExecutor::executeTask(ExecutorId id, vespalib::Executor::Task::UP task) { - assert(executorId < _executors.size()); - vespalib::ThreadStackExecutorBase &executor(*_executors[executorId]); + assert(id.getId() < _executors.size()); + vespalib::ThreadStackExecutorBase &executor(*_executors[id.getId()]); auto rejectedTask = executor.execute(std::move(task)); assert(!rejectedTask); } diff --git a/searchlib/src/vespa/searchlib/common/sequencedtaskexecutor.h b/searchlib/src/vespa/searchlib/common/sequencedtaskexecutor.h index 0c455225c89..41209da09ef 100644 --- a/searchlib/src/vespa/searchlib/common/sequencedtaskexecutor.h +++ b/searchlib/src/vespa/searchlib/common/sequencedtaskexecutor.h @@ -20,7 +20,7 @@ class SequencedTaskExecutor : public ISequencedTaskExecutor { using Stats = vespalib::ExecutorStats; std::vector<std::shared_ptr<vespalib::BlockingThreadStackExecutor>> _executors; - vespalib::hash_map<size_t, size_t> _ids; + vespalib::hash_map<size_t, ExecutorId> _ids; public: using ISequencedTaskExecutor::getExecutorId; @@ -29,8 +29,8 @@ public: void setTaskLimit(uint32_t taskLimit); uint32_t getNumExecutors() const override { return _executors.size(); } - uint32_t getExecutorId(uint64_t componentId) override; - void executeTask(uint32_t executorId, vespalib::Executor::Task::UP task) override; + ExecutorId getExecutorId(uint64_t componentId) override; + void executeTask(ExecutorId id, vespalib::Executor::Task::UP task) override; void sync() override; Stats getStats(); }; diff --git a/searchlib/src/vespa/searchlib/common/sequencedtaskexecutorobserver.cpp b/searchlib/src/vespa/searchlib/common/sequencedtaskexecutorobserver.cpp index e11b39de5ed..b693c976ebe 100644 --- a/searchlib/src/vespa/searchlib/common/sequencedtaskexecutorobserver.cpp +++ b/searchlib/src/vespa/searchlib/common/sequencedtaskexecutorobserver.cpp @@ -2,8 +2,7 @@ #include "sequencedtaskexecutorobserver.h" -namespace search -{ +namespace search { SequencedTaskExecutorObserver::SequencedTaskExecutorObserver(ISequencedTaskExecutor &executor) : _executor(executor), @@ -14,26 +13,23 @@ SequencedTaskExecutorObserver::SequencedTaskExecutorObserver(ISequencedTaskExecu { } -SequencedTaskExecutorObserver::~SequencedTaskExecutorObserver() -{ -} +SequencedTaskExecutorObserver::~SequencedTaskExecutorObserver() = default; -uint32_t +ISequencedTaskExecutor::ExecutorId SequencedTaskExecutorObserver::getExecutorId(uint64_t componentId) { return _executor.getExecutorId(componentId); } void -SequencedTaskExecutorObserver::executeTask(uint32_t executorId, - vespalib::Executor::Task::UP task) +SequencedTaskExecutorObserver::executeTask(ExecutorId id, vespalib::Executor::Task::UP task) { ++_executeCnt; { std::lock_guard<std::mutex> guard(_mutex); - _executeHistory.emplace_back(executorId); + _executeHistory.emplace_back(id.getId()); } - _executor.executeTask(executorId, std::move(task)); + _executor.executeTask(id, std::move(task)); } void diff --git a/searchlib/src/vespa/searchlib/common/sequencedtaskexecutorobserver.h b/searchlib/src/vespa/searchlib/common/sequencedtaskexecutorobserver.h index e7cbe7f54bc..b4561148bca 100644 --- a/searchlib/src/vespa/searchlib/common/sequencedtaskexecutorobserver.h +++ b/searchlib/src/vespa/searchlib/common/sequencedtaskexecutorobserver.h @@ -26,8 +26,8 @@ public: virtual ~SequencedTaskExecutorObserver() override; uint32_t getNumExecutors() const override { return _executor.getNumExecutors(); } - uint32_t getExecutorId(uint64_t componentId) override; - void executeTask(uint32_t executorId, vespalib::Executor::Task::UP task) override; + ExecutorId getExecutorId(uint64_t componentId) override; + void executeTask(ExecutorId id, vespalib::Executor::Task::UP task) override; void sync() override; uint32_t getExecuteCnt() const { return _executeCnt; } diff --git a/searchlib/src/vespa/searchlib/common/threaded_compactable_lid_space.cpp b/searchlib/src/vespa/searchlib/common/threaded_compactable_lid_space.cpp index 079a1f493de..defb537be0e 100644 --- a/searchlib/src/vespa/searchlib/common/threaded_compactable_lid_space.cpp +++ b/searchlib/src/vespa/searchlib/common/threaded_compactable_lid_space.cpp @@ -6,19 +6,18 @@ #include "isequencedtaskexecutor.h" #include <future> -namespace search { -namespace common { +namespace search::common { -ThreadedCompactableLidSpace::ThreadedCompactableLidSpace(std::shared_ptr<ICompactableLidSpace> target, ISequencedTaskExecutor &executor, uint32_t executorId) +ThreadedCompactableLidSpace::ThreadedCompactableLidSpace(std::shared_ptr<ICompactableLidSpace> target, + ISequencedTaskExecutor &executor, + ISequencedTaskExecutor::ExecutorId id) : _target(target), _executor(executor), - _executorId(executorId) + _executorId(id) { } -ThreadedCompactableLidSpace::~ThreadedCompactableLidSpace() -{ -} +ThreadedCompactableLidSpace::~ThreadedCompactableLidSpace() = default; void ThreadedCompactableLidSpace::compactLidSpace(uint32_t wantedDocLidLimit) @@ -51,4 +50,3 @@ ThreadedCompactableLidSpace::shrinkLidSpace() } } -} diff --git a/searchlib/src/vespa/searchlib/common/threaded_compactable_lid_space.h b/searchlib/src/vespa/searchlib/common/threaded_compactable_lid_space.h index 84c9c0d6495..02d54acf666 100644 --- a/searchlib/src/vespa/searchlib/common/threaded_compactable_lid_space.h +++ b/searchlib/src/vespa/searchlib/common/threaded_compactable_lid_space.h @@ -3,13 +3,10 @@ #pragma once #include "i_compactable_lid_space.h" +#include "isequencedtaskexecutor.h" #include <memory> -namespace search { - -class ISequencedTaskExecutor; - -namespace common { +namespace search::common { /** * Adapter class for a component that has a lid space that can be @@ -19,16 +16,16 @@ namespace common { class ThreadedCompactableLidSpace : public ICompactableLidSpace { std::shared_ptr<ICompactableLidSpace> _target; - ISequencedTaskExecutor &_executor; - uint32_t _executorId; + ISequencedTaskExecutor &_executor; + ISequencedTaskExecutor::ExecutorId _executorId; public: - ThreadedCompactableLidSpace(std::shared_ptr<ICompactableLidSpace> target, ISequencedTaskExecutor &executor, uint32_t executorId); - virtual ~ThreadedCompactableLidSpace() override; - virtual void compactLidSpace(uint32_t wantedDocLidLimit) override; - virtual bool canShrinkLidSpace() const override; - virtual size_t getEstimatedShrinkLidSpaceGain() const override; - virtual void shrinkLidSpace() override; + ThreadedCompactableLidSpace(std::shared_ptr<ICompactableLidSpace> target, ISequencedTaskExecutor &executor, + ISequencedTaskExecutor::ExecutorId executorId); + ~ThreadedCompactableLidSpace() override; + void compactLidSpace(uint32_t wantedDocLidLimit) override; + bool canShrinkLidSpace() const override; + size_t getEstimatedShrinkLidSpaceGain() const override; + void shrinkLidSpace() override; }; } -} diff --git a/searchsummary/src/.gitignore b/searchsummary/src/.gitignore index 47011ff3508..2e8e6fd906a 100644 --- a/searchsummary/src/.gitignore +++ b/searchsummary/src/.gitignore @@ -1,4 +1,3 @@ /Makefile.ini /config_command.sh /project.dsw -/searchsummary.mak diff --git a/slobrok/src/.gitignore b/slobrok/src/.gitignore index 42d192f90a7..a39df0815b3 100644 --- a/slobrok/src/.gitignore +++ b/slobrok/src/.gitignore @@ -1,4 +1,3 @@ Makefile.ini config_command.sh project.dsw -/slobrok.mak diff --git a/staging_vespalib/src/.gitignore b/staging_vespalib/src/.gitignore index eadbd940e1c..7871665637b 100644 --- a/staging_vespalib/src/.gitignore +++ b/staging_vespalib/src/.gitignore @@ -1,5 +1,4 @@ *.dsp -*.mak Makefile.ini config_command.sh project.dsw diff --git a/storage/src/.gitignore b/storage/src/.gitignore index f7cecb195ca..4bb2c3395eb 100644 --- a/storage/src/.gitignore +++ b/storage/src/.gitignore @@ -7,4 +7,3 @@ Makefile.ini config_command.sh project.dsw -/storage.mak diff --git a/storage/src/versiontag.mak b/storage/src/versiontag.mak deleted file mode 100644 index b01f54a9446..00000000000 --- a/storage/src/versiontag.mak +++ /dev/null @@ -1,7 +0,0 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -VTAG := $(shell $(VESPALIB_DIR)/bin/getversion -D $(TOP) ) - -ifneq (X$(SPECIFIED_VTAG),XDISABLE) - VTAG += -DV_TAG='"$(SPECIFIED_VTAG)"' -endif diff --git a/storageapi/src/.gitignore b/storageapi/src/.gitignore index 3578c0b3853..a735fdb4395 100644 --- a/storageapi/src/.gitignore +++ b/storageapi/src/.gitignore @@ -3,4 +3,3 @@ Makefile.ini config_command.sh doc project.dsw -/storageapi.mak diff --git a/storageframework/src/.gitignore b/storageframework/src/.gitignore index 57b38cc4f9d..2e8e6fd906a 100644 --- a/storageframework/src/.gitignore +++ b/storageframework/src/.gitignore @@ -1,4 +1,3 @@ /Makefile.ini /config_command.sh /project.dsw -/storageframework.mak diff --git a/storageserver/src/.gitignore b/storageserver/src/.gitignore index 9669be96e4b..4bb2c3395eb 100644 --- a/storageserver/src/.gitignore +++ b/storageserver/src/.gitignore @@ -7,4 +7,3 @@ Makefile.ini config_command.sh project.dsw -/storageserver.mak diff --git a/streamingvisitors/src/.gitignore b/streamingvisitors/src/.gitignore index f7611c3f5a8..a735fdb4395 100644 --- a/streamingvisitors/src/.gitignore +++ b/streamingvisitors/src/.gitignore @@ -3,4 +3,3 @@ Makefile.ini config_command.sh doc project.dsw -/streamingvisitors.mak diff --git a/vbench/src/.gitignore b/vbench/src/.gitignore index 7466c45681b..180d0ec5ea6 100644 --- a/vbench/src/.gitignore +++ b/vbench/src/.gitignore @@ -1,4 +1,3 @@ /Makefile.ini /project.dsw /config_command.sh -/vbench.mak diff --git a/vdslib/src/.gitignore b/vdslib/src/.gitignore index 1ba69e012b7..a39df0815b3 100644 --- a/vdslib/src/.gitignore +++ b/vdslib/src/.gitignore @@ -1,4 +1,3 @@ Makefile.ini config_command.sh project.dsw -/vdslib.mak diff --git a/vdstestlib/src/.gitignore b/vdstestlib/src/.gitignore index 201bafb341f..2e8e6fd906a 100644 --- a/vdstestlib/src/.gitignore +++ b/vdstestlib/src/.gitignore @@ -1,4 +1,3 @@ /Makefile.ini /config_command.sh /project.dsw -/vdstestlib.mak diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/DefaultZtsClient.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/DefaultZtsClient.java index 9d1b16da42a..dfe49d1b407 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/DefaultZtsClient.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/DefaultZtsClient.java @@ -20,6 +20,7 @@ import com.yahoo.vespa.athenz.identity.ServiceIdentityProvider; import com.yahoo.vespa.athenz.tls.Pkcs10Csr; import com.yahoo.vespa.athenz.tls.Pkcs10CsrBuilder; import org.apache.http.HttpResponse; +import org.apache.http.client.config.RequestConfig; import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.client.methods.HttpUriRequest; import org.apache.http.client.methods.RequestBuilder; @@ -246,6 +247,11 @@ public class DefaultZtsClient implements ZtsClient { .setRetryHandler(new DefaultHttpRequestRetryHandler(3, /*requestSentRetryEnabled*/true)) .setUserAgent("vespa-zts-client") .setSSLContext(sslContext) + .setDefaultRequestConfig(RequestConfig.custom() + .setConnectTimeout((int)Duration.ofSeconds(10).toMillis()) + .setConnectionRequestTimeout((int)Duration.ofSeconds(10).toMillis()) + .setSocketTimeout((int)Duration.ofSeconds(20).toMillis()) + .build()) .build(); } diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/ZtsClient.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/ZtsClient.java index 270954c73b2..43378b6507a 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/ZtsClient.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/ZtsClient.java @@ -26,7 +26,7 @@ public interface ZtsClient extends AutoCloseable { */ InstanceIdentity registerInstance(AthenzService providerIdentity, AthenzService instanceIdentity, - String instanceId, + String instanceId, // TODO Remove this parameter (unused/unnecessary) String attestationData, boolean requestServiceToken, Pkcs10Csr csr); diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/SignedIdentityDocument.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/SignedIdentityDocument.java index 60be42544c7..7c64d048944 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/SignedIdentityDocument.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/SignedIdentityDocument.java @@ -74,6 +74,7 @@ public class SignedIdentityDocument { return providerUniqueId; } + @Deprecated public String dnsSuffix() { return dnsSuffix; } @@ -82,6 +83,7 @@ public class SignedIdentityDocument { return providerService; } + @Deprecated public URI ztsEndpoint() { return ztsEndpoint; } diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzCredentialsService.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzCredentialsService.java index e8ef2d9f97e..1136106ce19 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzCredentialsService.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzCredentialsService.java @@ -1,97 +1,105 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.athenz.identityprovider.client; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; import com.yahoo.container.core.identity.IdentityConfig; import com.yahoo.vespa.athenz.api.AthenzService; +import com.yahoo.vespa.athenz.client.zts.DefaultZtsClient; +import com.yahoo.vespa.athenz.client.zts.InstanceIdentity; +import com.yahoo.vespa.athenz.identity.ServiceIdentityProvider; import com.yahoo.vespa.athenz.identityprovider.api.EntityBindingsMapper; +import com.yahoo.vespa.athenz.identityprovider.api.IdentityDocumentClient; import com.yahoo.vespa.athenz.identityprovider.api.SignedIdentityDocument; -import com.yahoo.vespa.athenz.identityprovider.api.bindings.SignedIdentityDocumentEntity; +import com.yahoo.vespa.athenz.tls.AthenzIdentityVerifier; import com.yahoo.vespa.athenz.tls.KeyAlgorithm; import com.yahoo.vespa.athenz.tls.KeyUtils; import com.yahoo.vespa.athenz.tls.Pkcs10Csr; -import com.yahoo.vespa.athenz.tls.Pkcs10CsrUtils; import com.yahoo.vespa.athenz.tls.SslContextBuilder; import javax.net.ssl.SSLContext; import java.io.File; -import java.io.IOException; -import java.io.UncheckedIOException; +import java.net.URI; import java.security.KeyPair; import java.security.PrivateKey; import java.security.cert.X509Certificate; import static com.yahoo.vespa.athenz.tls.KeyStoreType.JKS; +import static java.util.Collections.singleton; /** + * A service that provides method for initially registering the instance and refreshing it. + * * @author bjorncs */ class AthenzCredentialsService { - - private static final ObjectMapper mapper = new ObjectMapper().registerModule(new JavaTimeModule()); - private final IdentityConfig identityConfig; - private final IdentityDocumentClient identityDocumentClient; - private final ZtsClient ztsClient; + private final ServiceIdentityProvider nodeIdentityProvider; private final File trustStoreJks; + private final String hostname; + private final InstanceCsrGenerator instanceCsrGenerator; AthenzCredentialsService(IdentityConfig identityConfig, - IdentityDocumentClient identityDocumentClient, - ZtsClient ztsClient, - File trustStoreJks) { + ServiceIdentityProvider nodeIdentityProvider, + File trustStoreJks, + String hostname) { this.identityConfig = identityConfig; - this.identityDocumentClient = identityDocumentClient; - this.ztsClient = ztsClient; + this.nodeIdentityProvider = nodeIdentityProvider; this.trustStoreJks = trustStoreJks; + this.hostname = hostname; + this.instanceCsrGenerator = new InstanceCsrGenerator(identityConfig.athenzDnsSuffix()); } AthenzCredentials registerInstance() { KeyPair keyPair = KeyUtils.generateKeypair(KeyAlgorithm.RSA); - String rawDocument = identityDocumentClient.getSignedIdentityDocument(); - SignedIdentityDocument document = parseSignedIdentityDocument(rawDocument); - InstanceCsrGenerator instanceCsrGenerator = new InstanceCsrGenerator(document.dnsSuffix()); + IdentityDocumentClient identityDocumentClient = createIdentityDocumentClient(identityConfig, nodeIdentityProvider); + SignedIdentityDocument document = identityDocumentClient.getTenantIdentityDocument(hostname); + AthenzService tenantIdentity = new AthenzService(identityConfig.domain(), identityConfig.service()); Pkcs10Csr csr = instanceCsrGenerator.generateCsr( - new AthenzService(identityConfig.domain(), identityConfig.service()), + tenantIdentity, document.providerUniqueId(), document.identityDocument().ipAddresses(), keyPair); - InstanceRegisterInformation instanceRegisterInformation = - new InstanceRegisterInformation(document.providerService().getFullName(), - identityConfig.domain(), - identityConfig.service(), - rawDocument, - Pkcs10CsrUtils.toPem(csr)); - InstanceIdentity instanceIdentity = ztsClient.sendInstanceRegisterRequest(instanceRegisterInformation, - document.ztsEndpoint()); - return toAthenzCredentials(instanceIdentity, keyPair, document); + + try (com.yahoo.vespa.athenz.client.zts.ZtsClient ztsClient = + new DefaultZtsClient(URI.create(identityConfig.ztsUrl()), nodeIdentityProvider)) { + InstanceIdentity instanceIdentity = + ztsClient.registerInstance( + new AthenzService(identityConfig.configserverIdentityName()), + tenantIdentity, + null, + EntityBindingsMapper.toAttestationData(document), + true, + csr); + return toAthenzCredentials(instanceIdentity, keyPair, document); + } } AthenzCredentials updateCredentials(SignedIdentityDocument document, SSLContext sslContext) { + AthenzService tenantIdentity = new AthenzService(identityConfig.domain(), identityConfig.service()); KeyPair newKeyPair = KeyUtils.generateKeypair(KeyAlgorithm.RSA); - InstanceCsrGenerator instanceCsrGenerator = new InstanceCsrGenerator(document.dnsSuffix()); Pkcs10Csr csr = instanceCsrGenerator.generateCsr( - new AthenzService(identityConfig.domain(), identityConfig.service()), + tenantIdentity, document.providerUniqueId(), document.identityDocument().ipAddresses(), newKeyPair); - InstanceRefreshInformation refreshInfo = new InstanceRefreshInformation(Pkcs10CsrUtils.toPem(csr)); - InstanceIdentity instanceIdentity = - ztsClient.sendInstanceRefreshRequest(document.providerService().getFullName(), - identityConfig.domain(), - identityConfig.service(), - document.providerUniqueId().asDottedString(), - refreshInfo, - document.ztsEndpoint(), - sslContext); - return toAthenzCredentials(instanceIdentity, newKeyPair, document); + + try (com.yahoo.vespa.athenz.client.zts.ZtsClient ztsClient = + new DefaultZtsClient(URI.create(identityConfig.ztsUrl()), tenantIdentity, sslContext)) { + InstanceIdentity instanceIdentity = + ztsClient.refreshInstance( + new AthenzService(identityConfig.configserverIdentityName()), + tenantIdentity, + document.providerUniqueId().asDottedString(), + true, + csr); + return toAthenzCredentials(instanceIdentity, newKeyPair, document); + } } private AthenzCredentials toAthenzCredentials(InstanceIdentity instanceIdentity, KeyPair keyPair, SignedIdentityDocument identityDocument) { - X509Certificate certificate = instanceIdentity.getX509Certificate(); - String serviceToken = instanceIdentity.getServiceToken(); + X509Certificate certificate = instanceIdentity.certificate(); + String serviceToken = instanceIdentity.nToken().get().getRawToken(); SSLContext identitySslContext = createIdentitySslContext(keyPair.getPrivate(), certificate); return new AthenzCredentials(serviceToken, certificate, keyPair, identityDocument, identitySslContext); } @@ -103,11 +111,11 @@ class AthenzCredentialsService { .build(); } - private static SignedIdentityDocument parseSignedIdentityDocument(String rawDocument) { - try { - return EntityBindingsMapper.toSignedIdentityDocument(mapper.readValue(rawDocument, SignedIdentityDocumentEntity.class)); - } catch (IOException e) { - throw new UncheckedIOException(e); - } + private static DefaultIdentityDocumentClient createIdentityDocumentClient(IdentityConfig config, + ServiceIdentityProvider nodeIdentityProvider) { + return new DefaultIdentityDocumentClient( + URI.create(config.loadBalancerAddress()), + nodeIdentityProvider, + new AthenzIdentityVerifier(singleton(new AthenzService(config.configserverIdentityName())))); } } diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzIdentityProviderImpl.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzIdentityProviderImpl.java index 813941ac9b2..ce0743021ff 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzIdentityProviderImpl.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzIdentityProviderImpl.java @@ -16,12 +16,15 @@ import com.yahoo.vespa.athenz.api.AthenzRole; import com.yahoo.vespa.athenz.api.AthenzService; import com.yahoo.vespa.athenz.identity.ServiceIdentityProvider; import com.yahoo.vespa.athenz.identity.ServiceIdentityProviderListenerHelper; +import com.yahoo.vespa.athenz.identity.SiaIdentityProvider; import com.yahoo.vespa.athenz.tls.KeyStoreType; import com.yahoo.vespa.athenz.tls.SslContextBuilder; +import com.yahoo.vespa.athenz.utils.SiaUtils; import com.yahoo.vespa.defaults.Defaults; import javax.net.ssl.SSLContext; import java.io.File; +import java.net.URI; import java.security.PrivateKey; import java.security.cert.X509Certificate; import java.time.Clock; @@ -55,24 +58,26 @@ public final class AthenzIdentityProviderImpl extends AbstractComponent implemen private final Clock clock; private final AthenzService identity; private final ServiceIdentityProviderListenerHelper listenerHelper; + private final String dnsSuffix; + private final URI ztsEndpoint; private final LoadingCache<AthenzRole, SSLContext> roleSslContextCache; private final static Duration roleSslContextExpiry = Duration.ofHours(24); - // TODO IdentityConfig should contain ZTS uri and dns suffix @Inject public AthenzIdentityProviderImpl(IdentityConfig config, Metric metric) { this(config, metric, new AthenzCredentialsService(config, - new IdentityDocumentClient(config.loadBalancerAddress()), - new ZtsClient(), - getDefaultTrustStoreLocation()), + createNodeIdentityProvider(config), + getDefaultTrustStoreLocation(), + Defaults.getDefaults().vespaHostname()), new ScheduledThreadPoolExecutor(1), Clock.systemUTC()); } // Test only + AthenzIdentityProviderImpl(IdentityConfig config, Metric metric, AthenzCredentialsService athenzCredentialsService, @@ -84,6 +89,8 @@ public final class AthenzIdentityProviderImpl extends AbstractComponent implemen this.clock = clock; this.identity = new AthenzService(config.domain(), config.service()); this.listenerHelper = new ServiceIdentityProviderListenerHelper(this.identity); + this.dnsSuffix = config.athenzDnsSuffix(); + this.ztsEndpoint = URI.create(config.ztsUrl()); registerInstance(); roleSslContextCache = CacheBuilder.newBuilder() .refreshAfterWrite(roleSslContextExpiry.dividedBy(2).toMinutes(), TimeUnit.MINUTES) @@ -150,8 +157,8 @@ public final class AthenzIdentityProviderImpl extends AbstractComponent implemen PrivateKey privateKey = credentials.getKeyPair().getPrivate(); X509Certificate roleCertificate = ztsClient.getRoleCertificate( role, - credentials.getIdentityDocument().dnsSuffix(), - credentials.getIdentityDocument().ztsEndpoint(), + dnsSuffix, + ztsEndpoint, identity, privateKey, credentials.getIdentitySslContext()); @@ -166,7 +173,7 @@ public final class AthenzIdentityProviderImpl extends AbstractComponent implemen return ztsClient .getRoleToken( new AthenzDomain(domain), - credentials.getIdentityDocument().ztsEndpoint(), + ztsEndpoint, credentials.getIdentitySslContext()) .getRawToken(); } @@ -177,7 +184,7 @@ public final class AthenzIdentityProviderImpl extends AbstractComponent implemen .getRoleToken( new AthenzDomain(domain), role, - credentials.getIdentityDocument().ztsEndpoint(), + ztsEndpoint, credentials.getIdentitySslContext()) .getRawToken(); } @@ -193,6 +200,11 @@ public final class AthenzIdentityProviderImpl extends AbstractComponent implemen } } + private static SiaIdentityProvider createNodeIdentityProvider(IdentityConfig config) { + return new SiaIdentityProvider( + new AthenzService(config.nodeIdentityName()), SiaUtils.DEFAULT_SIA_DIRECTORY, getDefaultTrustStoreLocation()); + } + private static File getDefaultTrustStoreLocation() { return new File(Defaults.getDefaults().underVespaHome("share/ssl/certs/yahoo_certificate_bundle.jks")); } diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/DefaultIdentityDocumentClient.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/DefaultIdentityDocumentClient.java index b9aba6e66b0..c99e40732bb 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/DefaultIdentityDocumentClient.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/DefaultIdentityDocumentClient.java @@ -8,6 +8,7 @@ import com.yahoo.vespa.athenz.identityprovider.api.EntityBindingsMapper; import com.yahoo.vespa.athenz.identityprovider.api.IdentityDocumentClient; import com.yahoo.vespa.athenz.identityprovider.api.SignedIdentityDocument; import com.yahoo.vespa.athenz.identityprovider.api.bindings.SignedIdentityDocumentEntity; +import org.apache.http.client.config.RequestConfig; import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.client.methods.HttpUriRequest; import org.apache.http.client.methods.RequestBuilder; @@ -22,6 +23,7 @@ import javax.net.ssl.SSLContext; import java.io.IOException; import java.io.UncheckedIOException; import java.net.URI; +import java.time.Duration; import java.util.function.Supplier; /** @@ -102,6 +104,11 @@ public class DefaultIdentityDocumentClient implements IdentityDocumentClient { .setSSLContext(sslContext) .setSSLHostnameVerifier(hostnameVerifier) .setUserAgent("default-identity-document-client") + .setDefaultRequestConfig(RequestConfig.custom() + .setConnectTimeout((int)Duration.ofSeconds(10).toMillis()) + .setConnectionRequestTimeout((int)Duration.ofSeconds(10).toMillis()) + .setSocketTimeout((int)Duration.ofSeconds(20).toMillis()) + .build()) .build(); } diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/IdentityDocumentClient.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/IdentityDocumentClient.java deleted file mode 100644 index dfc89431ce4..00000000000 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/IdentityDocumentClient.java +++ /dev/null @@ -1,83 +0,0 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.athenz.identityprovider.client; - -import com.yahoo.vespa.defaults.Defaults; -import org.apache.http.client.methods.CloseableHttpResponse; -import org.apache.http.client.methods.HttpGet; -import org.apache.http.client.utils.URIBuilder; -import org.apache.http.conn.ssl.NoopHostnameVerifier; -import org.apache.http.conn.ssl.SSLConnectionSocketFactory; -import org.apache.http.conn.ssl.TrustSelfSignedStrategy; -import org.apache.http.impl.client.CloseableHttpClient; -import org.apache.http.impl.client.HttpClientBuilder; -import org.apache.http.ssl.SSLContextBuilder; -import org.apache.http.util.EntityUtils; -import org.eclipse.jetty.http.HttpStatus; - -import java.io.IOException; -import java.net.URI; -import java.net.URISyntaxException; -import java.security.GeneralSecurityException; - -/** - * @author mortent - * @author bjorncs - */ -public class IdentityDocumentClient { - - private final URI identityDocumentApiUri; - - public IdentityDocumentClient(String loadBalancerName) { - this.identityDocumentApiUri = createIdentityDocumentApiUri(loadBalancerName); - } - - /** - * Get signed identity document from config server - */ - public String getSignedIdentityDocument() { - try (CloseableHttpClient httpClient = createHttpClient()) { - CloseableHttpResponse idDocResponse = httpClient.execute(new HttpGet(identityDocumentApiUri)); - String responseContent = EntityUtils.toString(idDocResponse.getEntity()); - if (HttpStatus.isSuccess(idDocResponse.getStatusLine().getStatusCode())) { - return responseContent; - } else { - // TODO make sure we have retried a few times (AND logged) before giving up - throw new RuntimeException( - "Failed to initialize Athenz instance provider: " + - idDocResponse.getStatusLine() + ": " + responseContent); - } - } catch (IOException e) { - throw new RuntimeException("Failed getting signed identity document", e); - } - } - - // TODO Use client side auth to establish trusted secure channel - // TODO Validate TLS certifcate of config server - private static CloseableHttpClient createHttpClient() { - try { - SSLContextBuilder sslContextBuilder = new SSLContextBuilder(); - sslContextBuilder.loadTrustMaterial(null, new TrustSelfSignedStrategy()); - SSLConnectionSocketFactory sslSocketFactory = - new SSLConnectionSocketFactory(sslContextBuilder.build(), - NoopHostnameVerifier.INSTANCE); - return HttpClientBuilder.create().setSSLSocketFactory(sslSocketFactory).setUserAgent("identity-document-client").build(); - } catch (GeneralSecurityException e) { - throw new RuntimeException(e); - } - } - - private static URI createIdentityDocumentApiUri(String loadBalancerName) { - try { - // TODO Figure out a proper way of determining the hostname matching what's registred in node-repository - return new URIBuilder() - .setScheme("https") - .setHost(loadBalancerName) - .setPort(4443) - .setPath("/athenz/v1/provider/identity-document/tenant/" + Defaults.getDefaults().vespaHostname()) - .build(); - } catch (URISyntaxException e) { - throw new RuntimeException(e); - } - } - -} diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/InstanceIdentity.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/InstanceIdentity.java deleted file mode 100644 index 48200599149..00000000000 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/InstanceIdentity.java +++ /dev/null @@ -1,49 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.athenz.identityprovider.client; - -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.core.JsonParser; -import com.fasterxml.jackson.databind.DeserializationContext; -import com.fasterxml.jackson.databind.JsonDeserializer; -import com.fasterxml.jackson.databind.annotation.JsonDeserialize; -import com.yahoo.vespa.athenz.tls.X509CertificateUtils; - -import java.io.IOException; -import java.security.cert.X509Certificate; - -/** - * Used for deserializing response from ZTS - * - * @author mortent - */ -@JsonIgnoreProperties(ignoreUnknown = true) -@JsonInclude(JsonInclude.Include.NON_NULL) -public class InstanceIdentity { - @JsonProperty("x509Certificate") private final X509Certificate x509Certificate; - @JsonProperty("serviceToken") private final String serviceToken; - - public InstanceIdentity(@JsonProperty("x509Certificate") @JsonDeserialize(using = X509CertificateDeserializer.class) - X509Certificate x509Certificate, - @JsonProperty("serviceToken") String serviceToken) { - this.x509Certificate = x509Certificate; - this.serviceToken = serviceToken; - } - - public X509Certificate getX509Certificate() { - return x509Certificate; - } - - public String getServiceToken() { - return serviceToken; - } - - public static class X509CertificateDeserializer extends JsonDeserializer<X509Certificate> { - @Override - public X509Certificate deserialize(JsonParser parser, DeserializationContext context) throws IOException { - return X509CertificateUtils.fromPem(parser.getValueAsString()); - } - } - -} diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/InstanceRefreshInformation.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/InstanceRefreshInformation.java deleted file mode 100644 index dd35cb7e401..00000000000 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/InstanceRefreshInformation.java +++ /dev/null @@ -1,23 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.athenz.identityprovider.client; - -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonProperty; - -/** - * @author bjorncs - */ -@JsonIgnoreProperties(ignoreUnknown = true) -@JsonInclude(JsonInclude.Include.NON_NULL) -public class InstanceRefreshInformation { - - @JsonProperty("csr") - private final String csr; - @JsonProperty("token") - private final boolean requestServiceToken = true; - - public InstanceRefreshInformation(String csr) { - this.csr = csr; - } -} diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/InstanceRegisterInformation.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/InstanceRegisterInformation.java deleted file mode 100644 index cdf47ad8624..00000000000 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/InstanceRegisterInformation.java +++ /dev/null @@ -1,38 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.athenz.identityprovider.client; - -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonProperty; - -/** - * Used for serializing request to ZTS - * - * @author mortent - */ -@JsonIgnoreProperties(ignoreUnknown = true) -@JsonInclude(JsonInclude.Include.NON_NULL) -public class InstanceRegisterInformation { - @JsonProperty("provider") - private final String provider; - @JsonProperty("domain") - private final String domain; - @JsonProperty("service") - private final String service; - @JsonProperty("attestationData") - private final String attestationData; - @JsonProperty("ssh") - private final String ssh = null; // Not needed - @JsonProperty("csr") - private final String csr; - @JsonProperty("token") - private final boolean token = true; - - public InstanceRegisterInformation(String provider, String domain, String service, String attestationData, String csr) { - this.provider = provider; - this.domain = domain; - this.service = service; - this.attestationData = attestationData; - this.csr = csr; - } -} diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/ZtsClient.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/ZtsClient.java index afdccac62cf..a3ec55eb815 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/ZtsClient.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/ZtsClient.java @@ -1,8 +1,6 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.athenz.identityprovider.client; -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.ObjectMapper; import com.yahoo.athenz.zts.RoleCertificateRequest; import com.yahoo.athenz.zts.RoleToken; import com.yahoo.athenz.zts.ZTSClient; @@ -10,22 +8,10 @@ import com.yahoo.vespa.athenz.api.AthenzDomain; import com.yahoo.vespa.athenz.api.AthenzRole; import com.yahoo.vespa.athenz.api.AthenzService; import com.yahoo.vespa.athenz.api.ZToken; +import com.yahoo.vespa.athenz.client.zts.DefaultZtsClient; import com.yahoo.vespa.athenz.tls.X509CertificateUtils; -import org.apache.http.client.HttpRequestRetryHandler; -import org.apache.http.client.methods.CloseableHttpResponse; -import org.apache.http.client.methods.HttpUriRequest; -import org.apache.http.client.methods.RequestBuilder; -import org.apache.http.entity.ContentType; -import org.apache.http.entity.StringEntity; -import org.apache.http.impl.client.CloseableHttpClient; -import org.apache.http.impl.client.DefaultHttpRequestRetryHandler; -import org.apache.http.impl.client.HttpClientBuilder; -import org.apache.http.util.EntityUtils; -import org.eclipse.jetty.http.HttpStatus; import javax.net.ssl.SSLContext; -import java.io.IOException; -import java.io.UncheckedIOException; import java.net.URI; import java.security.PrivateKey; import java.security.cert.X509Certificate; @@ -34,54 +20,11 @@ import java.time.Duration; /** * @author mortent * @author bjorncs + * @deprecated Will be replaced by {@link DefaultZtsClient} once role token/certificate caching is ready. */ +@Deprecated class ZtsClient { - private static final String INSTANCE_API_PATH = "/zts/v1/instance"; - - private final ObjectMapper objectMapper = new ObjectMapper(); - private final HttpRequestRetryHandler retryHandler = new DefaultHttpRequestRetryHandler(3, /*requestSentRetryEnabled*/true); - - /** - * Send instance register request to ZTS, get InstanceIdentity - */ - InstanceIdentity sendInstanceRegisterRequest(InstanceRegisterInformation instanceRegisterInformation, - URI uri) { - try(CloseableHttpClient client = HttpClientBuilder.create().setRetryHandler(retryHandler).build()) { - HttpUriRequest postRequest = RequestBuilder.post() - .setUri(uri.resolve(INSTANCE_API_PATH)) - .setEntity(toJsonStringEntity(instanceRegisterInformation)) - .build(); - return getInstanceIdentity(client, postRequest); - } catch (IOException e) { - throw new UncheckedIOException(e); - } - } - - InstanceIdentity sendInstanceRefreshRequest(String providerService, - String instanceDomain, - String instanceServiceName, - String instanceId, - InstanceRefreshInformation instanceRefreshInformation, - URI ztsEndpoint, - SSLContext sslContext) { - try (CloseableHttpClient client = createHttpClientWithTlsAuth(sslContext, retryHandler)) { - URI uri = ztsEndpoint - .resolve(INSTANCE_API_PATH + '/') - .resolve(providerService + '/') - .resolve(instanceDomain + '/') - .resolve(instanceServiceName + '/') - .resolve(instanceId); - HttpUriRequest postRequest = RequestBuilder.post() - .setUri(uri) - .setEntity(toJsonStringEntity(instanceRefreshInformation)) - .build(); - return getInstanceIdentity(client, postRequest); - } catch (IOException e) { - throw new UncheckedIOException(e); - } - } - ZToken getRoleToken(AthenzDomain domain, URI ztsEndpoint, SSLContext sslContext) { @@ -118,28 +61,4 @@ class ZtsClient { return X509CertificateUtils.fromPem(pemCert.token); } - private InstanceIdentity getInstanceIdentity(CloseableHttpClient client, HttpUriRequest postRequest) - throws IOException { - try (CloseableHttpResponse response = client.execute(postRequest)) { - if(HttpStatus.isSuccess(response.getStatusLine().getStatusCode())) { - return objectMapper.readValue(response.getEntity().getContent(), InstanceIdentity.class); - } else { - String message = EntityUtils.toString(response.getEntity()); - throw new RuntimeException(String.format("Unable to get identity. http code/message: %d/%s", - response.getStatusLine().getStatusCode(), message)); - } - } - } - - private StringEntity toJsonStringEntity(Object value) throws JsonProcessingException { - return new StringEntity(objectMapper.writeValueAsString(value), ContentType.APPLICATION_JSON); - } - - private static CloseableHttpClient createHttpClientWithTlsAuth(SSLContext sslContext, - HttpRequestRetryHandler retryHandler) { - return HttpClientBuilder.create() - .setRetryHandler(retryHandler) - .setSSLContext(sslContext) - .build(); - } } diff --git a/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzIdentityProviderImplTest.java b/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzIdentityProviderImplTest.java index 7ad465a7d80..48781aad651 100644 --- a/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzIdentityProviderImplTest.java +++ b/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzIdentityProviderImplTest.java @@ -1,42 +1,22 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.athenz.identityprovider.client; -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; import com.yahoo.container.core.identity.IdentityConfig; import com.yahoo.container.jdisc.athenz.AthenzIdentityProviderException; import com.yahoo.jdisc.Metric; import com.yahoo.test.ManualClock; -import com.yahoo.vespa.athenz.api.AthenzService; -import com.yahoo.vespa.athenz.identityprovider.api.EntityBindingsMapper; -import com.yahoo.vespa.athenz.identityprovider.api.IdentityDocument; -import com.yahoo.vespa.athenz.identityprovider.api.IdentityType; -import com.yahoo.vespa.athenz.identityprovider.api.SignedIdentityDocument; -import com.yahoo.vespa.athenz.identityprovider.api.VespaUniqueInstanceId; -import com.yahoo.vespa.athenz.tls.KeyStoreBuilder; -import com.yahoo.vespa.athenz.tls.KeyStoreUtils; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; -import org.mockito.invocation.InvocationOnMock; -import org.mockito.stubbing.Answer; -import java.io.File; -import java.io.IOException; -import java.net.URI; -import java.security.KeyStore; import java.security.cert.X509Certificate; import java.time.Duration; import java.time.Instant; -import java.util.Collections; import java.util.Date; import java.util.concurrent.ScheduledExecutorService; import java.util.function.Supplier; -import static com.yahoo.vespa.athenz.tls.KeyStoreType.JKS; import static org.mockito.Matchers.any; -import static org.mockito.Matchers.anyString; import static org.mockito.Matchers.eq; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; @@ -55,7 +35,13 @@ public class AthenzIdentityProviderImplTest { private static final IdentityConfig IDENTITY_CONFIG = new IdentityConfig(new IdentityConfig.Builder() - .service("tenantService").domain("tenantDomain").loadBalancerAddress("cfg").ztsUrl("https:localhost:4443/zts/v1").athenzDnsSuffix("vespa.cloud")); + .service("tenantService") + .domain("tenantDomain") + .nodeIdentityName("vespa.tenant") + .configserverIdentityName("vespa.configserver") + .loadBalancerAddress("cfg") + .ztsUrl("https:localhost:4443/zts/v1") + .athenzDnsSuffix("dev-us-north-1.vespa.cloud")); @Test(expected = AthenzIdentityProviderException.class) public void component_creation_fails_when_credentials_not_found() { @@ -67,30 +53,24 @@ public class AthenzIdentityProviderImplTest { } @Test - public void metrics_updated_on_refresh() throws IOException { - IdentityDocumentClient identityDocumentClient = mock(IdentityDocumentClient.class); - ZtsClient ztsClient = mock(ZtsClient.class); + public void metrics_updated_on_refresh() { ManualClock clock = new ManualClock(Instant.EPOCH); Metric metric = mock(Metric.class); - when(identityDocumentClient.getSignedIdentityDocument()).thenReturn(getIdentityDocument()); - when(ztsClient.sendInstanceRegisterRequest(any(), any())).then(new Answer<InstanceIdentity>() { - @Override - public InstanceIdentity answer(InvocationOnMock invocationOnMock) throws Throwable { - return new InstanceIdentity(getCertificate(getExpirationSupplier(clock)), "TOKEN"); - } - }); + AthenzCredentialsService athenzCredentialsService = mock(AthenzCredentialsService.class); - when(ztsClient.sendInstanceRefreshRequest(anyString(), anyString(), anyString(), anyString(), any(), any(), any())) + X509Certificate certificate = getCertificate(getExpirationSupplier(clock)); + + when(athenzCredentialsService.registerInstance()) + .thenReturn(new AthenzCredentials(null, certificate, null, null, null)); + + when(athenzCredentialsService.updateCredentials(any(), any())) .thenThrow(new RuntimeException("#1")) .thenThrow(new RuntimeException("#2")) - .thenReturn(new InstanceIdentity(getCertificate(getExpirationSupplier(clock)), "TOKEN")); - - AthenzCredentialsService credentialService = - new AthenzCredentialsService(IDENTITY_CONFIG, identityDocumentClient, ztsClient, createDummyTrustStore()); + .thenReturn(new AthenzCredentials(null, certificate, null, null, null)); AthenzIdentityProviderImpl identityProvider = - new AthenzIdentityProviderImpl(IDENTITY_CONFIG, metric, credentialService, mock(ScheduledExecutorService.class), clock); + new AthenzIdentityProviderImpl(IDENTITY_CONFIG, metric, athenzCredentialsService, mock(ScheduledExecutorService.class), clock); identityProvider.reportMetrics(); verify(metric).set(eq(AthenzIdentityProviderImpl.CERTIFICATE_EXPIRY_METRIC_NAME), eq(certificateValidity.getSeconds()), any()); @@ -125,31 +105,4 @@ public class AthenzIdentityProviderImplTest { return x509Certificate; } - private File createDummyTrustStore() throws IOException { - File file = tempDir.newFile(); - KeyStore keyStore = KeyStoreBuilder.withType(JKS).build(); - KeyStoreUtils.writeKeyStoreToFile(keyStore, file); - return file; - } - - private static String getIdentityDocument() throws JsonProcessingException { - VespaUniqueInstanceId instanceId = new VespaUniqueInstanceId(0, "default", "default", "application", "tenant", "us-north-1", "dev", IdentityType.TENANT); - SignedIdentityDocument signedIdentityDocument = new SignedIdentityDocument( - new IdentityDocument(instanceId, "localhost", "x.y.com", Instant.EPOCH, Collections.emptySet()), - "dummysignature", - 0, - instanceId, - "dev-us-north-1.vespa.cloud", - new AthenzService("vespa.vespa.provider_dev_us-north-1"), - URI.create("https://zts:4443/zts/v1"), - 1, - "localhost", - "x.y.com", - Instant.EPOCH, - Collections.emptySet(), - IdentityType.TENANT); - - return new ObjectMapper().registerModule(new JavaTimeModule()) - .writeValueAsString(EntityBindingsMapper.toSignedIdentityDocumentEntity(signedIdentityDocument)); - } } diff --git a/vespalib/src/.gitignore b/vespalib/src/.gitignore index 8859e7233e9..d185ea18fd0 100644 --- a/vespalib/src/.gitignore +++ b/vespalib/src/.gitignore @@ -1,5 +1,4 @@ *.dsp -*.mak Makefile.ini config_command.sh project.dsw diff --git a/vespalog/src/.gitignore b/vespalog/src/.gitignore index 41357d5435e..e03155ca194 100644 --- a/vespalog/src/.gitignore +++ b/vespalog/src/.gitignore @@ -1,5 +1,4 @@ Makefile.ini config.h config_command.sh -vespalog.mak project.dsw diff --git a/vespamalloc/src/.gitignore b/vespamalloc/src/.gitignore index dd21c2da121..a39df0815b3 100644 --- a/vespamalloc/src/.gitignore +++ b/vespamalloc/src/.gitignore @@ -1,4 +1,3 @@ Makefile.ini config_command.sh project.dsw -vespamalloc.mak diff --git a/vsm/src/.gitignore b/vsm/src/.gitignore index 8859e7233e9..d185ea18fd0 100644 --- a/vsm/src/.gitignore +++ b/vsm/src/.gitignore @@ -1,5 +1,4 @@ *.dsp -*.mak Makefile.ini config_command.sh project.dsw |