diff options
59 files changed, 458 insertions, 455 deletions
diff --git a/config-model-api/src/main/java/com/yahoo/config/application/api/ApplicationPackage.java b/config-model-api/src/main/java/com/yahoo/config/application/api/ApplicationPackage.java index cd825767565..5646b62c900 100644 --- a/config-model-api/src/main/java/com/yahoo/config/application/api/ApplicationPackage.java +++ b/config-model-api/src/main/java/com/yahoo/config/application/api/ApplicationPackage.java @@ -270,8 +270,19 @@ public interface ApplicationPackage { * * @return A new application package instance pointing to a new location */ - default ApplicationPackage preprocess(ZoneId zone, RuleConfigDeriver ruleConfigDeriver, DeployLogger logger) throws IOException, TransformerException, ParserConfigurationException, SAXException { + default ApplicationPackage preprocess(ZoneId zone, RuleConfigDeriver ruleConfigDeriver, DeployLogger logger) + throws IOException, TransformerException, ParserConfigurationException, SAXException { throw new UnsupportedOperationException("This application package does not support preprocessing"); } + /** + * @deprecated pass a ZoneId as first parameter instead + */ + // TODO: Remove on Vespa 7 + @Deprecated + default ApplicationPackage preprocess(Zone zone, RuleConfigDeriver ruleConfigDeriver, DeployLogger logger) + throws IOException, TransformerException, ParserConfigurationException, SAXException { + return preprocess(zone.id(), ruleConfigDeriver, logger); + } + } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/application/ApplicationConvergenceChecker.java b/configserver/src/main/java/com/yahoo/vespa/config/server/application/ApplicationConvergenceChecker.java index 39cd4629ff0..925f8324b30 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/application/ApplicationConvergenceChecker.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/application/ApplicationConvergenceChecker.java @@ -35,7 +35,6 @@ public class ApplicationConvergenceChecker extends AbstractComponent { private final static Set<String> serviceTypesToCheck = new HashSet<>(Arrays.asList( "container", - "container-clustercontroller", "qrserver", "docprocservice", "searchnode", diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/JobStatus.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/JobStatus.java index ceb04d88026..a7940076277 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/JobStatus.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/JobStatus.java @@ -11,14 +11,14 @@ import java.util.Optional; /** * The last known build status of a particular deployment job for a particular application. * This is immutable. - * + * * @author bratseth * @author mpolden */ public class JobStatus { - + private final DeploymentJobs.JobType type; - + private final Optional<JobRun> lastTriggered; private final Optional<JobRun> lastCompleted; private final Optional<JobRun> firstFailing; @@ -42,7 +42,7 @@ public class JobStatus { this.type = type; this.jobError = jobError; - + // Never say we triggered component because we don't: this.lastTriggered = type == DeploymentJobs.JobType.component ? Optional.empty() : lastTriggered; this.lastCompleted = lastCompleted; @@ -52,7 +52,7 @@ public class JobStatus { /** Returns an empty job status */ public static JobStatus initial(DeploymentJobs.JobType type) { - return new JobStatus(type, Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty()); + return new JobStatus(type, Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty()); } public JobStatus withTriggering(Version version, Optional<ApplicationRevision> revision, @@ -89,13 +89,13 @@ public class JobStatus { Optional<JobRun> firstFailing = this.firstFailing; if (jobError.isPresent() && ! this.firstFailing.isPresent()) firstFailing = Optional.of(thisCompletion); - + Optional<JobRun> lastSuccess = this.lastSuccess; if ( ! jobError.isPresent()) { lastSuccess = Optional.of(thisCompletion); firstFailing = Optional.empty(); } - + return new JobStatus(type, jobError, lastTriggered, Optional.of(thisCompletion), firstFailing, lastSuccess); } @@ -105,7 +105,7 @@ public class JobStatus { public boolean isSuccess() { return lastCompleted().isPresent() && ! jobError.isPresent(); } - + /** Returns true if last triggered is newer than last completed and was started after timeoutLimit */ public boolean isRunning(Instant timeoutLimit) { if ( ! lastTriggered.isPresent()) return false; @@ -114,6 +114,11 @@ public class JobStatus { return ! lastTriggered.get().at().isBefore(lastCompleted.get().at()); } + /** Returns true if this is running and has been so since before the given limit */ + public boolean isHanging(Instant timeoutLimit) { + return isRunning(Instant.MIN) && lastTriggered.get().at().isBefore(timeoutLimit.plusMillis(1)); + } + /** The error of the last completion, or empty if the last run succeeded */ public Optional<DeploymentJobs.JobError> jobError() { return jobError; } @@ -140,10 +145,10 @@ public class JobStatus { ", first failing: " + firstFailing.map(JobRun::toString).orElse("(not failing)") + ", lastSuccess: " + lastSuccess.map(JobRun::toString).orElse("(never)") + "]"; } - + @Override public int hashCode() { return Objects.hash(type, jobError, lastTriggered, lastCompleted, firstFailing, lastSuccess); } - + @Override public boolean equals(Object o) { if (o == this) return true; @@ -159,15 +164,15 @@ public class JobStatus { /** Information about a particular triggering or completion of a run of a job. This is immutable. */ public static class JobRun { - + private final long id; private final Version version; private final Optional<ApplicationRevision> revision; private final boolean upgrade; private final String reason; private final Instant at; - - public JobRun(long id, Version version, Optional<ApplicationRevision> revision, + + public JobRun(long id, Version version, Optional<ApplicationRevision> revision, boolean upgrade, String reason, Instant at) { Objects.requireNonNull(version, "version cannot be null"); Objects.requireNonNull(revision, "revision cannot be null"); @@ -188,16 +193,16 @@ public class JobStatus { // TODO: Fix how this is set, and add an applicationChange() method as well, in the same vein. /** Returns whether this job run was a Vespa upgrade */ public boolean upgrade() { return upgrade; } - + /** Returns the Vespa version used on this run */ public Version version() { return version; } - + /** Returns the application revision used for this run, or empty when not known */ public Optional<ApplicationRevision> revision() { return revision; } - + /** Returns a human-readable reason for this particular job run */ public String reason() { return reason; } - + /** Returns the time if this triggering or completion */ public Instant at() { return at; } @@ -218,7 +223,7 @@ public class JobStatus { public int hashCode() { return Objects.hash(version, revision, upgrade, at); } - + @Override public boolean equals(Object o) { if (this == o) return true; @@ -234,7 +239,7 @@ public class JobStatus { @Override public String toString() { return "job run " + id + " of version " + (upgrade() ? "upgrade " : "") + version + " " + revision + " at " + at; } - + } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java index f0c950b024b..192901165be 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java @@ -32,9 +32,9 @@ import java.util.logging.Logger; /** * Responsible for scheduling deployment jobs in a build system and keeping * Application.deploying() in sync with what is scheduled. - * + * * This class is multithread safe. - * + * * @author bratseth * @author mpolden */ @@ -60,7 +60,7 @@ public class DeploymentTrigger { this.order = new DeploymentOrder(controller); this.jobTimeout = controller.system().equals(SystemName.main) ? Duration.ofHours(12) : Duration.ofHours(1); } - + /** Returns the time in the past before which jobs are at this moment considered unresponsive */ public Instant jobTimeoutLimit() { return clock.instant().minus(jobTimeout); } @@ -70,10 +70,10 @@ public class DeploymentTrigger { //--- Start of methods which triggers deployment jobs ------------------------- - /** + /** * Called each time a job completes (successfully or not) to cause triggering of one or more follow-up jobs * (which may possibly the same job once over). - * + * * @param report information about the job that just completed */ public void triggerFromCompletion(JobReport report) { @@ -143,10 +143,11 @@ public class DeploymentTrigger { JobStatus systemTestStatus = application.deploymentJobs().jobStatus().get(JobType.systemTest); if (application.deploying().get() instanceof Change.VersionChange) { Version target = ((Change.VersionChange) application.deploying().get()).version(); - if (systemTestStatus == null + if (systemTestStatus == null || ! systemTestStatus.lastTriggered().isPresent() || ! systemTestStatus.isSuccess() - || ! systemTestStatus.lastTriggered().get().version().equals(target)) { + || ! systemTestStatus.lastTriggered().get().version().equals(target) + || systemTestStatus.isHanging(jobTimeoutLimit())) { application = trigger(JobType.systemTest, application, false, "Upgrade to " + target); controller.applications().store(application); } @@ -170,7 +171,7 @@ public class DeploymentTrigger { List<JobType> nextToTrigger = new ArrayList<>(); for (JobType nextJobType : order.nextAfter(jobType, application)) { JobStatus nextStatus = application.deploymentJobs().jobStatus().get(nextJobType); - if (changesAvailable(application, jobStatus, nextStatus)) + if (changesAvailable(application, jobStatus, nextStatus) || nextStatus.isHanging(jobTimeoutLimit())) nextToTrigger.add(nextJobType); } // Trigger them in parallel @@ -209,10 +210,10 @@ public class DeploymentTrigger { return true; return false; } - + /** * Triggers a change of this application - * + * * @param applicationId the application to trigger * @throws IllegalArgumentException if this application already have an ongoing change */ @@ -267,7 +268,7 @@ public class DeploymentTrigger { } /** - * Trigger a job for an application + * Trigger a job for an application * * @param jobType the type of the job to trigger, or null to trigger nothing * @param application the application to trigger the job for @@ -289,7 +290,7 @@ public class DeploymentTrigger { /** * Trigger a job for an application, if allowed - * + * * @param jobType the type of the job to trigger, or null to trigger nothing * @param application the application to trigger the job for * @param first whether to trigger the job before other jobs @@ -323,7 +324,7 @@ public class DeploymentTrigger { /** Returns true if the given proposed job triggering should be effected */ private boolean allowedTriggering(JobType jobType, LockedApplication application) { - // Note: We could make a more fine-grained and more correct determination about whether to block + // Note: We could make a more fine-grained and more correct determination about whether to block // by instead basing the decision on what is currently deployed in the zone. However, // this leads to some additional corner cases, and the possibility of blocking an application // fix to a version upgrade, so not doing it now @@ -341,7 +342,7 @@ public class DeploymentTrigger { return true; } - + private boolean isRunningProductionJob(Application application) { return JobList.from(application) .production() @@ -364,7 +365,7 @@ public class DeploymentTrigger { if (existingDeployment == null) return false; return existingDeployment.version().isAfter(version); } - + private boolean acceptNewRevisionNow(LockedApplication application) { if ( ! application.deploying().isPresent()) return true; @@ -377,5 +378,5 @@ public class DeploymentTrigger { // Otherwise, the application is currently upgrading, without failures, and we should wait with the revision. return false; } - + } diff --git a/document/src/vespa/document/select/context.cpp b/document/src/vespa/document/select/context.cpp index 6d9e0df157b..3a728db33f8 100644 --- a/document/src/vespa/document/select/context.cpp +++ b/document/src/vespa/document/select/context.cpp @@ -38,10 +38,14 @@ Context::~Context() { } std::unique_ptr<Value> Context::getValue(const vespalib::string & value) const { - VariableMap::const_iterator iter = _variables->find(value); - - if (iter != _variables->end()) { - return std::make_unique<FloatValue>(iter->second); + if (_variables) { + VariableMap::const_iterator iter = _variables->find(value); + + if (iter != _variables->end()) { + return std::make_unique<FloatValue>(iter->second); + } else { + return std::make_unique<FloatValue>(0.0); + } } else { return std::make_unique<FloatValue>(0.0); } diff --git a/eval/src/tests/tensor/dense_dot_product_function/dense_dot_product_function_test.cpp b/eval/src/tests/tensor/dense_dot_product_function/dense_dot_product_function_test.cpp index ca77997bac7..0b8b98fc617 100644 --- a/eval/src/tests/tensor/dense_dot_product_function/dense_dot_product_function_test.cpp +++ b/eval/src/tests/tensor/dense_dot_product_function/dense_dot_product_function_test.cpp @@ -1,8 +1,5 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/log/log.h> -LOG_SETUP("dense_dot_product_function_test"); - #include <vespa/vespalib/testkit/test_kit.h> #include <vespa/eval/eval/tensor_function.h> #include <vespa/eval/tensor/dense/dense_dot_product_function.h> @@ -12,16 +9,13 @@ LOG_SETUP("dense_dot_product_function_test"); #include <vespa/vespalib/util/stringfmt.h> #include <vespa/vespalib/util/stash.h> +#include <vespa/log/log.h> +LOG_SETUP("dense_dot_product_function_test"); + using namespace vespalib; using namespace vespalib::eval; using namespace vespalib::tensor; -ValueType -makeType(size_t numCells) -{ - return ValueType::tensor_type({{"x", numCells}}); -} - tensor::Tensor::UP makeTensor(size_t numCells, double cellBias) { diff --git a/eval/src/tests/tensor/dense_tensor_builder/dense_tensor_builder_test.cpp b/eval/src/tests/tensor/dense_tensor_builder/dense_tensor_builder_test.cpp index 6f3cdd5f93f..61efdbe6d22 100644 --- a/eval/src/tests/tensor/dense_tensor_builder/dense_tensor_builder_test.cpp +++ b/eval/src/tests/tensor/dense_tensor_builder/dense_tensor_builder_test.cpp @@ -147,7 +147,7 @@ TEST_F("require that builder can be re-used", Fixture) } void -assertTensorCell(const std::vector<size_t> &expAddress, +assertTensorCell(const DenseTensor::Address &expAddress, double expCell, const DenseTensor::CellsIterator &itr) { diff --git a/eval/src/vespa/eval/eval/simple_tensor.cpp b/eval/src/vespa/eval/eval/simple_tensor.cpp index 0e58d292334..1836f2088f3 100644 --- a/eval/src/vespa/eval/eval/simple_tensor.cpp +++ b/eval/src/vespa/eval/eval/simple_tensor.cpp @@ -57,14 +57,14 @@ Address select(const Address &a, const Address &b, const IndexList &selector) { return result; } -size_t get_dimension_size(const ValueType &type, size_t dim_idx) { +size_t get_dimension_size(const ValueType &type, ValueType::Dimension::size_type dim_idx) { if (dim_idx == ValueType::Dimension::npos) { return 1; } return type.dimensions()[dim_idx].size; } -size_t get_dimension_index(const Address &addr, size_t dim_idx) { +size_t get_dimension_index(const Address &addr, ValueType::Dimension::size_type dim_idx) { if (dim_idx == ValueType::Dimension::npos) { return 0; } diff --git a/eval/src/vespa/eval/eval/value_type.cpp b/eval/src/vespa/eval/eval/value_type.cpp index 03e6d2bbcdf..1c4973a78ca 100644 --- a/eval/src/vespa/eval/eval/value_type.cpp +++ b/eval/src/vespa/eval/eval/value_type.cpp @@ -101,9 +101,9 @@ struct Renamer { } // namespace vespalib::tensor::<unnamed> -constexpr size_t ValueType::Dimension::npos; +constexpr ValueType::Dimension::size_type ValueType::Dimension::npos; -ValueType::~ValueType() { } +ValueType::~ValueType() = default; bool ValueType::is_sparse() const { diff --git a/eval/src/vespa/eval/eval/value_type.h b/eval/src/vespa/eval/eval/value_type.h index e304f51436f..a4762acd4c0 100644 --- a/eval/src/vespa/eval/eval/value_type.h +++ b/eval/src/vespa/eval/eval/value_type.h @@ -18,12 +18,13 @@ class ValueType public: enum class Type { ANY, ERROR, DOUBLE, TENSOR }; struct Dimension { - static constexpr size_t npos = -1; + using size_type = uint32_t; + static constexpr size_type npos = -1; vespalib::string name; - size_t size; + size_type size; Dimension(const vespalib::string &name_in) : name(name_in), size(npos) {} - Dimension(const vespalib::string &name_in, size_t size_in) + Dimension(const vespalib::string &name_in, size_type size_in) : name(name_in), size(size_in) {} bool operator==(const Dimension &rhs) const { return ((name == rhs.name) && (size == rhs.size)); diff --git a/eval/src/vespa/eval/tensor/dense/dense_dot_product_function.cpp b/eval/src/vespa/eval/tensor/dense/dense_dot_product_function.cpp index 992f2eae750..fdd0cd6638f 100644 --- a/eval/src/vespa/eval/tensor/dense/dense_dot_product_function.cpp +++ b/eval/src/vespa/eval/tensor/dense/dense_dot_product_function.cpp @@ -6,8 +6,7 @@ #include <vespa/eval/eval/value.h> #include <vespa/eval/tensor/tensor.h> -namespace vespalib { -namespace tensor { +namespace vespalib::tensor { using CellsRef = DenseTensorView::CellsRef; @@ -39,5 +38,5 @@ DenseDotProductFunction::eval(ConstArrayRef<eval::Value::CREF> params, Stash &st return stash.create<eval::DoubleValue>(result); } -} // namespace tensor -} // namespace vespalib +} + diff --git a/eval/src/vespa/eval/tensor/dense/dense_dot_product_function.h b/eval/src/vespa/eval/tensor/dense/dense_dot_product_function.h index 8ad57d69524..288f2afd084 100644 --- a/eval/src/vespa/eval/tensor/dense/dense_dot_product_function.h +++ b/eval/src/vespa/eval/tensor/dense/dense_dot_product_function.h @@ -5,8 +5,7 @@ #include <vespa/eval/eval/tensor_function.h> #include <vespa/vespalib/hwaccelrated/iaccelrated.h> -namespace vespalib { -namespace tensor { +namespace vespalib::tensor { /** * Tensor function for a dot product between two 1-dimensional dense tensors. @@ -27,5 +26,5 @@ public: const eval::Value &eval(ConstArrayRef<eval::Value::CREF> params, Stash &stash) const override; }; -} // namespace tensor -} // namespace vespalib +} + diff --git a/eval/src/vespa/eval/tensor/dense/dense_tensor.cpp b/eval/src/vespa/eval/tensor/dense/dense_tensor.cpp index 5d7e0c83267..9693e89bb75 100644 --- a/eval/src/vespa/eval/tensor/dense/dense_tensor.cpp +++ b/eval/src/vespa/eval/tensor/dense/dense_tensor.cpp @@ -4,12 +4,10 @@ #include <vespa/vespalib/util/stringfmt.h> #include <vespa/vespalib/util/exceptions.h> #include <vespa/eval/eval/operation.h> -#include <sstream> using vespalib::eval::TensorSpec; -namespace vespalib { -namespace tensor { +namespace vespalib::tensor { namespace { @@ -84,5 +82,5 @@ DenseTensor::operator==(const DenseTensor &rhs) const (_cells == rhs._cells); } -} // namespace vespalib::tensor -} // namespace vespalib +} + diff --git a/eval/src/vespa/eval/tensor/dense/dense_tensor.h b/eval/src/vespa/eval/tensor/dense/dense_tensor.h index 1b97438272e..c45d3c7ccb6 100644 --- a/eval/src/vespa/eval/tensor/dense/dense_tensor.h +++ b/eval/src/vespa/eval/tensor/dense/dense_tensor.h @@ -8,8 +8,7 @@ #include "dense_tensor_cells_iterator.h" #include "dense_tensor_view.h" -namespace vespalib { -namespace tensor { +namespace vespalib::tensor { /** * A dense tensor where all dimensions are indexed. @@ -29,16 +28,13 @@ private: public: DenseTensor(); ~DenseTensor() {} - DenseTensor(const eval::ValueType &type_in, - const Cells &cells_in); - DenseTensor(const eval::ValueType &type_in, - Cells &&cells_in); - DenseTensor(eval::ValueType &&type_in, - Cells &&cells_in); + DenseTensor(const eval::ValueType &type_in, const Cells &cells_in); + DenseTensor(const eval::ValueType &type_in, Cells &&cells_in); + DenseTensor(eval::ValueType &&type_in, Cells &&cells_in); bool operator==(const DenseTensor &rhs) const; const Cells &cells() const { return _cells; } }; -} // namespace vespalib::tensor -} // namespace vespalib +} + diff --git a/eval/src/vespa/eval/tensor/dense/dense_tensor_address_combiner.cpp b/eval/src/vespa/eval/tensor/dense/dense_tensor_address_combiner.cpp index 3e9f4f619f0..ef2a56d4582 100644 --- a/eval/src/vespa/eval/tensor/dense/dense_tensor_address_combiner.cpp +++ b/eval/src/vespa/eval/tensor/dense/dense_tensor_address_combiner.cpp @@ -4,33 +4,7 @@ #include <vespa/vespalib/util/exceptions.h> #include <cassert> -namespace vespalib { -namespace tensor { - -using Address = DenseTensorAddressCombiner::Address; - -namespace { - -class AddressReader -{ -private: - const Address &_address; - size_t _idx; - -public: - AddressReader(const Address &address) - : _address(address), - _idx(0) - {} - size_t nextLabel() { - return _address[_idx++]; - } - bool valid() { - return _idx < _address.size(); - } -}; - -} +namespace vespalib::tensor { DenseTensorAddressCombiner::~DenseTensorAddressCombiner() { } @@ -57,35 +31,7 @@ DenseTensorAddressCombiner::DenseTensorAddressCombiner(const eval::ValueType &lh _ops.push_back(AddressOp::RHS); ++rhsItr; } -} - -bool -DenseTensorAddressCombiner::combine(const CellsIterator &lhsItr, - const CellsIterator &rhsItr) -{ - _combinedAddress.clear(); - AddressReader lhsReader(lhsItr.address()); - AddressReader rhsReader(rhsItr.address()); - for (const auto &op : _ops) { - switch (op) { - case AddressOp::LHS: - _combinedAddress.emplace_back(lhsReader.nextLabel()); - break; - case AddressOp::RHS: - _combinedAddress.emplace_back(rhsReader.nextLabel()); - break; - case AddressOp::BOTH: - size_t lhsLabel = lhsReader.nextLabel(); - size_t rhsLabel = rhsReader.nextLabel(); - if (lhsLabel != rhsLabel) { - return false; - } - _combinedAddress.emplace_back(lhsLabel); - } - } - assert(!lhsReader.valid()); - assert(!rhsReader.valid()); - return true; + _combinedAddress.resize(_ops.size()); } eval::ValueType @@ -120,5 +66,4 @@ DenseTensorAddressCombiner::combineDimensions(const eval::ValueType &lhs, eval::ValueType::tensor_type(std::move(result))); } -} // namespace vespalib::tensor -} // namespace vespalib +} diff --git a/eval/src/vespa/eval/tensor/dense/dense_tensor_address_combiner.h b/eval/src/vespa/eval/tensor/dense/dense_tensor_address_combiner.h index 30bfd740fdd..37fad083dc1 100644 --- a/eval/src/vespa/eval/tensor/dense/dense_tensor_address_combiner.h +++ b/eval/src/vespa/eval/tensor/dense/dense_tensor_address_combiner.h @@ -7,8 +7,7 @@ #include <vespa/eval/tensor/types.h> #include <vespa/eval/eval/value_type.h> -namespace vespalib { -namespace tensor { +namespace vespalib::tensor { /** @@ -19,32 +18,57 @@ namespace tensor { class DenseTensorAddressCombiner { public: - using Address = std::vector<size_t>; + using Address = DenseTensorCellsIterator::Address; private: - enum class AddressOp { - LHS, - RHS, - BOTH - }; + enum class AddressOp { LHS, RHS, BOTH }; using CellsIterator = DenseTensorCellsIterator; std::vector<AddressOp> _ops; Address _combinedAddress; + class AddressReader + { + private: + const Address &_address; + uint32_t _idx; + public: + AddressReader(const Address &address) : _address(address), _idx(0) {} + Address::value_type nextLabel() { return _address[_idx++]; } + }; public: - DenseTensorAddressCombiner(const eval::ValueType &lhs, - const eval::ValueType &rhs); + DenseTensorAddressCombiner(const eval::ValueType &lhs, const eval::ValueType &rhs); ~DenseTensorAddressCombiner(); - bool combine(const CellsIterator &lhsItr, - const CellsIterator &rhsItr); const Address &address() const { return _combinedAddress; } - static eval::ValueType combineDimensions(const eval::ValueType &lhs, const eval::ValueType &rhs); + bool combine(const CellsIterator &lhsItr, const CellsIterator &rhsItr) { + uint32_t index(0); + AddressReader lhsReader(lhsItr.address()); + AddressReader rhsReader(rhsItr.address()); + for (const auto &op : _ops) { + switch (op) { + case AddressOp::LHS: + _combinedAddress[index] = lhsReader.nextLabel(); + break; + case AddressOp::RHS: + _combinedAddress[index] = rhsReader.nextLabel(); + break; + case AddressOp::BOTH: + Address::value_type lhsLabel = lhsReader.nextLabel(); + Address::value_type rhsLabel = rhsReader.nextLabel(); + if (lhsLabel != rhsLabel) { + return false; + } + _combinedAddress[index] = lhsLabel; + } + index++; + } + return true; + } + static eval::ValueType combineDimensions(const eval::ValueType &lhs, const eval::ValueType &rhs); }; -} // namespace vespalib::tensor -} // namespace vespalib +} diff --git a/eval/src/vespa/eval/tensor/dense/dense_tensor_apply.h b/eval/src/vespa/eval/tensor/dense/dense_tensor_apply.h index 36432c420f5..49e075f6999 100644 --- a/eval/src/vespa/eval/tensor/dense/dense_tensor_apply.h +++ b/eval/src/vespa/eval/tensor/dense/dense_tensor_apply.h @@ -2,13 +2,12 @@ #pragma once -namespace vespalib { -namespace tensor { +namespace vespalib::tensor { + class Tensor; + class DenseTensor; +} -class Tensor; -class DenseTensor; - -namespace dense { +namespace vespalib::tensor::dense { /** * Creates a new tensor using all combinations of input tensor cells with matching @@ -22,7 +21,4 @@ template <typename Function> std::unique_ptr<Tensor> apply(const DenseTensorView &lhs, const DenseTensorView &rhs, Function &&func); -} // namespace vespalib::tensor::dense -} // namespace vespalib::tensor -} // namespace vespalib - +} diff --git a/eval/src/vespa/eval/tensor/dense/dense_tensor_apply.hpp b/eval/src/vespa/eval/tensor/dense/dense_tensor_apply.hpp index 65fee767690..dc47d02d47c 100644 --- a/eval/src/vespa/eval/tensor/dense/dense_tensor_apply.hpp +++ b/eval/src/vespa/eval/tensor/dense/dense_tensor_apply.hpp @@ -6,9 +6,7 @@ #include "dense_tensor_address_combiner.h" #include "direct_dense_tensor_builder.h" -namespace vespalib { -namespace tensor { -namespace dense { +namespace vespalib::tensor::dense { template <typename Function> std::unique_ptr<Tensor> @@ -42,6 +40,4 @@ apply(const DenseTensorView &lhs, const Tensor &rhs, Function &&func) return Tensor::UP(); } -} // namespace vespalib::tensor::dense -} // namespace vespalib::tensor -} // namespace vespalib +} diff --git a/eval/src/vespa/eval/tensor/dense/dense_tensor_builder.cpp b/eval/src/vespa/eval/tensor/dense/dense_tensor_builder.cpp index 0b66dd51206..5d52e5f6e0e 100644 --- a/eval/src/vespa/eval/tensor/dense/dense_tensor_builder.cpp +++ b/eval/src/vespa/eval/tensor/dense/dense_tensor_builder.cpp @@ -83,7 +83,7 @@ DenseTensorBuilder::calculateCellAddress() const auto &dim = _dimensions[i]; if (label == UNDEFINED_LABEL) { throw IllegalArgumentException(make_string("Label for dimension '%s' is undefined. " - "Expected a value in the range [0, %zu>", + "Expected a value in the range [0, %u>", dim.name.c_str(), dim.size)); } result += (label * multiplier); diff --git a/eval/src/vespa/eval/tensor/dense/dense_tensor_builder.h b/eval/src/vespa/eval/tensor/dense/dense_tensor_builder.h index 765ed57393a..3969a9335b8 100644 --- a/eval/src/vespa/eval/tensor/dense/dense_tensor_builder.h +++ b/eval/src/vespa/eval/tensor/dense/dense_tensor_builder.h @@ -6,8 +6,7 @@ #include <vespa/vespalib/stllike/hash_map.h> #include <vespa/eval/tensor/tensor_builder.h> -namespace vespalib { -namespace tensor { +namespace vespalib::tensor { /** * A builder of for dense tensors. @@ -38,5 +37,5 @@ public: Tensor::UP build(); }; -} // namespace vespalib::tensor -} // namespace vespalib +} + diff --git a/eval/src/vespa/eval/tensor/dense/dense_tensor_cells_iterator.cpp b/eval/src/vespa/eval/tensor/dense/dense_tensor_cells_iterator.cpp index 59b4646a22b..d20c5124330 100644 --- a/eval/src/vespa/eval/tensor/dense/dense_tensor_cells_iterator.cpp +++ b/eval/src/vespa/eval/tensor/dense/dense_tensor_cells_iterator.cpp @@ -2,23 +2,14 @@ #include "dense_tensor_cells_iterator.h" -namespace vespalib { -namespace tensor { +namespace vespalib::tensor { -void -DenseTensorCellsIterator::next() -{ - ++_cellIdx; - if (valid()) { - for (int64_t i = (_address.size() - 1); i >= 0; --i) { - _address[i] = (_address[i] + 1) % _type.dimensions()[i].size; - if (_address[i] != 0) { - // Outer dimension labels can only be increased when this label wraps around. - break; - } - } - } -} +DenseTensorCellsIterator::DenseTensorCellsIterator(const eval::ValueType &type_in, CellsRef cells) + : _type(type_in), + _cells(cells), + _cellIdx(0), + _address(type_in.dimensions().size(), 0) +{} +DenseTensorCellsIterator::~DenseTensorCellsIterator() = default; -} // namespace vespalib::tensor -} // namespace vespalib +} diff --git a/eval/src/vespa/eval/tensor/dense/dense_tensor_cells_iterator.h b/eval/src/vespa/eval/tensor/dense/dense_tensor_cells_iterator.h index f77517bfdc5..fcffecef764 100644 --- a/eval/src/vespa/eval/tensor/dense/dense_tensor_cells_iterator.h +++ b/eval/src/vespa/eval/tensor/dense/dense_tensor_cells_iterator.h @@ -8,34 +8,41 @@ #include <vespa/eval/tensor/tensor.h> #include <vespa/vespalib/util/arrayref.h> -namespace vespalib { -namespace tensor { +namespace vespalib::tensor { /** * Utility class to iterate over cells in a dense tensor. */ class DenseTensorCellsIterator { +public: + using size_type = eval::ValueType::Dimension::size_type; + using Address = std::vector<size_type>; private: using CellsRef = vespalib::ConstArrayRef<double>; const eval::ValueType &_type; CellsRef _cells; - size_t _cellIdx; - std::vector<size_t> _address; - + size_t _cellIdx; + Address _address; public: - DenseTensorCellsIterator(const eval::ValueType &type_in, CellsRef cells) - : _type(type_in), - _cells(cells), - _cellIdx(0), - _address(type_in.dimensions().size(), 0) - {} + DenseTensorCellsIterator(const eval::ValueType &type_in, CellsRef cells); + ~DenseTensorCellsIterator(); + void next() { + ++_cellIdx; + for (int64_t i = (_address.size() - 1); i >= 0; --i) { + _address[i]++; + if (__builtin_expect((_address[i] != _type.dimensions()[i].size), true)) { + // Outer dimension labels can only be increased when this label wraps around. + break; + } else { + _address[i] = 0; + } + } + } bool valid() const { return _cellIdx < _cells.size(); } - void next(); double cell() const { return _cells[_cellIdx]; } - const std::vector<size_t> &address() const { return _address; } + const Address &address() const { return _address; } const eval::ValueType &fast_type() const { return _type; } }; -} // namespace vespalib::tensor -} // namespace vespalib +} diff --git a/eval/src/vespa/eval/tensor/dense/dense_tensor_function_compiler.cpp b/eval/src/vespa/eval/tensor/dense/dense_tensor_function_compiler.cpp index 1268a46b8e5..22e2a3fb78c 100644 --- a/eval/src/vespa/eval/tensor/dense/dense_tensor_function_compiler.cpp +++ b/eval/src/vespa/eval/tensor/dense/dense_tensor_function_compiler.cpp @@ -11,8 +11,7 @@ using namespace vespalib::eval; using namespace vespalib::eval::tensor_function; using namespace vespalib::eval::operation; -namespace vespalib { -namespace tensor { +namespace vespalib::tensor { namespace { @@ -89,5 +88,5 @@ DenseTensorFunctionCompiler::compile(const eval::tensor_function::Node &expr, St return InnerProductFunctionCompiler::compile(expr, stash); } -} // namespace tensor -} // namespace vespalib +} + diff --git a/eval/src/vespa/eval/tensor/dense/dense_tensor_function_compiler.h b/eval/src/vespa/eval/tensor/dense/dense_tensor_function_compiler.h index d5ba4e4f7a7..61c3af079e3 100644 --- a/eval/src/vespa/eval/tensor/dense/dense_tensor_function_compiler.h +++ b/eval/src/vespa/eval/tensor/dense/dense_tensor_function_compiler.h @@ -4,11 +4,9 @@ #include <vespa/eval/eval/tensor_function.h> -namespace vespalib { +namespace vespalib { class Stash; } -class Stash; - -namespace tensor { +namespace vespalib::tensor { /** * Class that recognizes calculations over dense tensors (in tensor function intermediate representation) @@ -19,5 +17,5 @@ struct DenseTensorFunctionCompiler static const eval::TensorFunction &compile(const eval::tensor_function::Node &expr, Stash &stash); }; -} // namespace tensor -} // namespace vespalib +} + diff --git a/eval/src/vespa/eval/tensor/dense/dense_tensor_reduce.h b/eval/src/vespa/eval/tensor/dense/dense_tensor_reduce.h index d8f47d2234c..fb054318985 100644 --- a/eval/src/vespa/eval/tensor/dense/dense_tensor_reduce.h +++ b/eval/src/vespa/eval/tensor/dense/dense_tensor_reduce.h @@ -4,9 +4,7 @@ #include "dense_tensor.h" -namespace vespalib { -namespace tensor { -namespace dense { +namespace vespalib::tensor::dense { /** * Returns a tensor with the given dimension(s) removed and the cell values in that dimension(s) @@ -16,6 +14,5 @@ template<typename Function> std::unique_ptr<Tensor> reduce(const DenseTensorView &tensor, const std::vector<vespalib::string> &dimensions, Function &&func); -} // namespace dense -} // namespace tensor -} // namespace vespalib +} + diff --git a/eval/src/vespa/eval/tensor/dense/dense_tensor_view.cpp b/eval/src/vespa/eval/tensor/dense/dense_tensor_view.cpp index 30c9f17348e..74c8981168d 100644 --- a/eval/src/vespa/eval/tensor/dense/dense_tensor_view.cpp +++ b/eval/src/vespa/eval/tensor/dense/dense_tensor_view.cpp @@ -13,8 +13,7 @@ using vespalib::eval::TensorSpec; -namespace vespalib { -namespace tensor { +namespace vespalib::tensor { namespace { @@ -228,7 +227,7 @@ DenseTensorView::accept(TensorVisitor &visitor) const addressBuilder.clear(); auto rawIndex = iterator.address().begin(); for (const auto &dimension : _typeRef.dimensions()) { - label = vespalib::make_string("%zu", *rawIndex); + label = vespalib::make_string("%u", *rawIndex); addressBuilder.add(dimension.name, label); ++rawIndex; } @@ -264,5 +263,4 @@ DenseTensorView::reduce(join_fun_t op, op); } -} // namespace vespalib::tensor -} // namespace vespalib +} diff --git a/eval/src/vespa/eval/tensor/dense/dense_tensor_view.h b/eval/src/vespa/eval/tensor/dense/dense_tensor_view.h index 5a59594667d..fd95c8555f4 100644 --- a/eval/src/vespa/eval/tensor/dense/dense_tensor_view.h +++ b/eval/src/vespa/eval/tensor/dense/dense_tensor_view.h @@ -7,8 +7,7 @@ #include <vespa/eval/eval/value_type.h> #include "dense_tensor_cells_iterator.h" -namespace vespalib { -namespace tensor { +namespace vespalib::tensor { class DenseTensor; @@ -22,6 +21,7 @@ public: using Cells = std::vector<double>; using CellsRef = ConstArrayRef<double>; using CellsIterator = DenseTensorCellsIterator; + using Address = std::vector<eval::ValueType::Dimension::size_type>; private: const eval::ValueType &_typeRef; @@ -61,5 +61,5 @@ public: virtual void accept(TensorVisitor &visitor) const override; }; -} // namespace vespalib::tensor -} // namespace vespalib +} + diff --git a/eval/src/vespa/eval/tensor/dense/dense_xw_product_function.cpp b/eval/src/vespa/eval/tensor/dense/dense_xw_product_function.cpp index 45de00dc7fe..1ab78b8ee30 100644 --- a/eval/src/vespa/eval/tensor/dense/dense_xw_product_function.cpp +++ b/eval/src/vespa/eval/tensor/dense/dense_xw_product_function.cpp @@ -8,8 +8,7 @@ #include <vespa/vespalib/util/exceptions.h> #include <assert.h> -namespace vespalib { -namespace tensor { +namespace vespalib::tensor { DenseXWProductFunction::DenseXWProductFunction(const eval::ValueType &resultType, size_t vectorId, @@ -87,5 +86,5 @@ DenseXWProductFunction::eval(ConstArrayRef<eval::Value::CREF> params, Stash &sta return stash.create<DenseTensorView>(_resultType, outputCells); } -} // namespace tensor -} // namespace vespalib +} + diff --git a/eval/src/vespa/eval/tensor/dense/dense_xw_product_function.h b/eval/src/vespa/eval/tensor/dense/dense_xw_product_function.h index db006100e5a..151f1f13800 100644 --- a/eval/src/vespa/eval/tensor/dense/dense_xw_product_function.h +++ b/eval/src/vespa/eval/tensor/dense/dense_xw_product_function.h @@ -6,8 +6,7 @@ #include "dense_tensor_view.h" #include <vespa/vespalib/hwaccelrated/iaccelrated.h> -namespace vespalib { -namespace tensor { +namespace vespalib::tensor { using XWInput = DenseTensorView::CellsRef; using XWOutput = ArrayRef<double>; @@ -49,5 +48,5 @@ public: const eval::Value &eval(ConstArrayRef<eval::Value::CREF> params, Stash &stash) const override; }; -} // namespace tensor -} // namespace vespalib +} + diff --git a/eval/src/vespa/eval/tensor/dense/direct_dense_tensor_builder.cpp b/eval/src/vespa/eval/tensor/dense/direct_dense_tensor_builder.cpp index f73d123d4bd..27d72e18f96 100644 --- a/eval/src/vespa/eval/tensor/dense/direct_dense_tensor_builder.cpp +++ b/eval/src/vespa/eval/tensor/dense/direct_dense_tensor_builder.cpp @@ -3,8 +3,7 @@ #include "direct_dense_tensor_builder.h" #include <cassert> -namespace vespalib { -namespace tensor { +namespace vespalib::tensor { using Address = DirectDenseTensorBuilder::Address; using eval::ValueType; @@ -35,7 +34,7 @@ calculateCellAddress(const Address &address, const ValueType &type) } -DirectDenseTensorBuilder::~DirectDenseTensorBuilder() { } +DirectDenseTensorBuilder::~DirectDenseTensorBuilder() = default; DirectDenseTensorBuilder::DirectDenseTensorBuilder(const ValueType &type_in) : _type(type_in), @@ -57,5 +56,5 @@ DirectDenseTensorBuilder::build() return std::make_unique<DenseTensor>(std::move(_type), std::move(_cells)); } -} // namespace tensor -} // namesapce vespalib +} + diff --git a/eval/src/vespa/eval/tensor/dense/direct_dense_tensor_builder.h b/eval/src/vespa/eval/tensor/dense/direct_dense_tensor_builder.h index 5e0368e8e69..865decd9fb8 100644 --- a/eval/src/vespa/eval/tensor/dense/direct_dense_tensor_builder.h +++ b/eval/src/vespa/eval/tensor/dense/direct_dense_tensor_builder.h @@ -4,8 +4,7 @@ #include "dense_tensor.h" -namespace vespalib { -namespace tensor { +namespace vespalib::tensor { /** * Class for building a dense tensor by inserting cell values directly into underlying array of cells. @@ -14,7 +13,7 @@ class DirectDenseTensorBuilder { public: using Cells = DenseTensor::Cells; - using Address = std::vector<size_t>; + using Address = DenseTensor::Address; private: eval::ValueType _type; @@ -27,5 +26,5 @@ public: Tensor::UP build(); }; -} // namespace tensor -} // namespace vespalib +} + diff --git a/eval/src/vespa/eval/tensor/dense/mutable_dense_tensor_view.cpp b/eval/src/vespa/eval/tensor/dense/mutable_dense_tensor_view.cpp index 71b7824ee5d..e3b4c8dee42 100644 --- a/eval/src/vespa/eval/tensor/dense/mutable_dense_tensor_view.cpp +++ b/eval/src/vespa/eval/tensor/dense/mutable_dense_tensor_view.cpp @@ -4,8 +4,7 @@ using vespalib::eval::ValueType; -namespace vespalib { -namespace tensor { +namespace vespalib::tensor { MutableDenseTensorView::MutableValueType::MutableValueType(ValueType type_in) : _type(type_in) @@ -19,7 +18,7 @@ MutableDenseTensorView::MutableValueType::MutableValueType(ValueType type_in) } } -MutableDenseTensorView::MutableValueType::~MutableValueType() {} +MutableDenseTensorView::MutableValueType::~MutableValueType() = default; MutableDenseTensorView::MutableDenseTensorView(ValueType type_in) : DenseTensorView(_concreteType.fast_type(), CellsRef()), @@ -33,5 +32,5 @@ MutableDenseTensorView::MutableDenseTensorView(ValueType type_in, CellsRef cells { } -} // namespace tensor -} // namespace vespalib +} + diff --git a/eval/src/vespa/eval/tensor/dense/mutable_dense_tensor_view.h b/eval/src/vespa/eval/tensor/dense/mutable_dense_tensor_view.h index 7eee3a9483c..b68a1594905 100644 --- a/eval/src/vespa/eval/tensor/dense/mutable_dense_tensor_view.h +++ b/eval/src/vespa/eval/tensor/dense/mutable_dense_tensor_view.h @@ -5,8 +5,7 @@ #include "dense_tensor_view.h" #include <cassert> -namespace vespalib { -namespace tensor { +namespace vespalib::tensor { /** * A mutable view to a dense tensor where all dimensions are indexed. @@ -18,7 +17,7 @@ private: { private: eval::ValueType _type; - std::vector<size_t *> _unboundDimSizes; + std::vector<eval::ValueType::Dimension::size_type *> _unboundDimSizes; public: MutableValueType(eval::ValueType type_in); @@ -55,5 +54,5 @@ public: } }; -} // namespace vespalib::tensor -} // namespace vespalib +} + diff --git a/eval/src/vespa/eval/tensor/sparse/sparse_tensor.cpp b/eval/src/vespa/eval/tensor/sparse/sparse_tensor.cpp index b02de7dc310..1aa05bf4f61 100644 --- a/eval/src/vespa/eval/tensor/sparse/sparse_tensor.cpp +++ b/eval/src/vespa/eval/tensor/sparse/sparse_tensor.cpp @@ -33,8 +33,7 @@ copyCells(Cells &cells, const Cells &cells_in, Stash &stash) } -SparseTensor::SparseTensor(const eval::ValueType &type_in, - const Cells &cells_in) +SparseTensor::SparseTensor(const eval::ValueType &type_in, const Cells &cells_in) : _type(type_in), _cells(), _stash(STASH_CHUNK_SIZE) @@ -43,14 +42,13 @@ SparseTensor::SparseTensor(const eval::ValueType &type_in, } -SparseTensor::SparseTensor(eval::ValueType &&type_in, - Cells &&cells_in, Stash &&stash_in) +SparseTensor::SparseTensor(eval::ValueType &&type_in, Cells &&cells_in, Stash &&stash_in) : _type(std::move(type_in)), _cells(std::move(cells_in)), _stash(std::move(stash_in)) -{ -} +{ } +SparseTensor::~SparseTensor() = default; bool SparseTensor::operator==(const SparseTensor &rhs) const diff --git a/eval/src/vespa/eval/tensor/sparse/sparse_tensor.h b/eval/src/vespa/eval/tensor/sparse/sparse_tensor.h index ef0827ce8ac..2715e606729 100644 --- a/eval/src/vespa/eval/tensor/sparse/sparse_tensor.h +++ b/eval/src/vespa/eval/tensor/sparse/sparse_tensor.h @@ -21,7 +21,7 @@ namespace vespalib::tensor { class SparseTensor : public Tensor { public: - using Cells = vespalib::hash_map<SparseTensorAddressRef, double>; + using Cells = hash_map<SparseTensorAddressRef, double>; static constexpr size_t STASH_CHUNK_SIZE = 16384u; @@ -33,6 +33,7 @@ private: public: explicit SparseTensor(const eval::ValueType &type_in, const Cells &cells_in); SparseTensor(eval::ValueType &&type_in, Cells &&cells_in, Stash &&stash_in); + ~SparseTensor() override; const Cells &cells() const { return _cells; } const eval::ValueType &fast_type() const { return _type; } bool operator==(const SparseTensor &rhs) const; diff --git a/eval/src/vespa/eval/tensor/sparse/sparse_tensor_address_combiner.h b/eval/src/vespa/eval/tensor/sparse/sparse_tensor_address_combiner.h index 491d5c9be8b..8e792cb100a 100644 --- a/eval/src/vespa/eval/tensor/sparse/sparse_tensor_address_combiner.h +++ b/eval/src/vespa/eval/tensor/sparse/sparse_tensor_address_combiner.h @@ -5,9 +5,8 @@ #include "sparse_tensor_address_builder.h" #include <vespa/eval/tensor/types.h> -namespace vespalib { -namespace eval { class ValueType; } -namespace tensor::sparse { +namespace vespalib::eval { class ValueType; } +namespace vespalib::tensor::sparse { /** * Combine two tensor addresses to a new tensor address. Common dimensions @@ -15,12 +14,7 @@ namespace tensor::sparse { */ class TensorAddressCombiner : public SparseTensorAddressBuilder { - enum class AddressOp - { - LHS, - RHS, - BOTH - }; + enum class AddressOp { LHS, RHS, BOTH }; std::vector<AddressOp> _ops; @@ -33,6 +27,5 @@ public: size_t numDimensions() const { return _ops.size(); } }; +} -} // namespace vespalib::tensor::sparse -} // namespace vespalib diff --git a/eval/src/vespa/eval/tensor/sparse/sparse_tensor_address_decoder.h b/eval/src/vespa/eval/tensor/sparse/sparse_tensor_address_decoder.h index 3a0502aee5b..2fbd9932009 100644 --- a/eval/src/vespa/eval/tensor/sparse/sparse_tensor_address_decoder.h +++ b/eval/src/vespa/eval/tensor/sparse/sparse_tensor_address_decoder.h @@ -5,10 +5,7 @@ #include <vespa/vespalib/stllike/string.h> #include "sparse_tensor_address_ref.h" -namespace vespalib { - - -namespace tensor { +namespace vespalib::tensor { /** * A decoder for a serialized tensor address, with only labels present. @@ -40,5 +37,5 @@ public: }; -} // namespace vespalib::tensor -} // namespace vespalib +} + diff --git a/eval/src/vespa/eval/tensor/sparse/sparse_tensor_address_padder.h b/eval/src/vespa/eval/tensor/sparse/sparse_tensor_address_padder.h index 506f8b29593..29e10c778ba 100644 --- a/eval/src/vespa/eval/tensor/sparse/sparse_tensor_address_padder.h +++ b/eval/src/vespa/eval/tensor/sparse/sparse_tensor_address_padder.h @@ -6,8 +6,7 @@ #include "sparse_tensor_address_decoder.h" #include <cassert> -namespace vespalib { -namespace tensor { +namespace vespalib::tensor { /** @@ -16,11 +15,7 @@ namespace tensor { */ class SparseTensorAddressPadder : public SparseTensorAddressBuilder { - enum class PadOp - { - PAD, - COPY - }; + enum class PadOp { PAD, COPY }; std::vector<PadOp> _padOps; @@ -67,6 +62,5 @@ public: } }; +} -} // namespace vespalib::tensor -} // namespace vespalib diff --git a/eval/src/vespa/eval/tensor/sparse/sparse_tensor_address_reducer.cpp b/eval/src/vespa/eval/tensor/sparse/sparse_tensor_address_reducer.cpp index 7da5bd8d61a..fbd0034bc14 100644 --- a/eval/src/vespa/eval/tensor/sparse/sparse_tensor_address_reducer.cpp +++ b/eval/src/vespa/eval/tensor/sparse/sparse_tensor_address_reducer.cpp @@ -4,20 +4,16 @@ #include <vespa/eval/eval/value_type.h> #include <vespa/vespalib/stllike/hash_set.hpp> -namespace vespalib { -namespace tensor { -namespace sparse { +namespace vespalib::tensor::sparse { TensorAddressReducer::TensorAddressReducer(const eval::ValueType &type, - const std::vector<vespalib::string> & - removeDimensions) + const std::vector<vespalib::string> & removeDimensions) : SparseTensorAddressBuilder(), _ops() { - TensorDimensionsSet removeSet(removeDimensions.cbegin(), - removeDimensions.cend()); + TensorDimensionsSet removeSet(removeDimensions.cbegin(), removeDimensions.cend()); _ops.reserve(type.dimensions().size()); - for (auto &dim : type.dimensions()) { + for (const auto &dim : type.dimensions()) { if (removeSet.find(dim.name) != removeSet.end()) { _ops.push_back(AddressOp::REMOVE); } else { @@ -26,10 +22,7 @@ TensorAddressReducer::TensorAddressReducer(const eval::ValueType &type, } } -TensorAddressReducer::~TensorAddressReducer() -{ +TensorAddressReducer::~TensorAddressReducer() = default; + } -} // namespace vespalib::tensor::sparse -} // namespace vespalib::tensor -} // namespace vespalib diff --git a/eval/src/vespa/eval/tensor/sparse/sparse_tensor_address_reducer.h b/eval/src/vespa/eval/tensor/sparse/sparse_tensor_address_reducer.h index c40d34d9a53..a2034d3be49 100644 --- a/eval/src/vespa/eval/tensor/sparse/sparse_tensor_address_reducer.h +++ b/eval/src/vespa/eval/tensor/sparse/sparse_tensor_address_reducer.h @@ -7,21 +7,15 @@ #include "sparse_tensor_address_decoder.h" #include <cassert> -namespace vespalib { -namespace eval { class ValueType; } -namespace tensor { -namespace sparse { +namespace vespalib::eval { class ValueType; } +namespace vespalib::tensor::sparse { /** * Reduce sparse tensor address by removing one or more dimensions. */ class TensorAddressReducer : public SparseTensorAddressBuilder { - enum AddressOp - { - REMOVE, - COPY - }; + enum AddressOp { REMOVE, COPY }; using AddressOps = std::vector<AddressOp>; @@ -50,7 +44,5 @@ public: } }; +} -} // namespace vespalib::tensor::sparse -} // namespace vespalib::tensor -} // namespace vespalib diff --git a/eval/src/vespa/eval/tensor/sparse/sparse_tensor_apply.h b/eval/src/vespa/eval/tensor/sparse/sparse_tensor_apply.h index 92345b260fd..ec6edf2d847 100644 --- a/eval/src/vespa/eval/tensor/sparse/sparse_tensor_apply.h +++ b/eval/src/vespa/eval/tensor/sparse/sparse_tensor_apply.h @@ -3,9 +3,11 @@ #pragma once namespace vespalib::tensor { -class Tensor; -class SparseTensor; -namespace sparse { + class Tensor; + class SparseTensor; +} + +namespace vespalib::tensor::sparse { /** * Create new tensor using all combinations of input tensor cells with matching @@ -16,6 +18,5 @@ template <typename Function> std::unique_ptr<Tensor> apply(const SparseTensor &lhs, const SparseTensor &rhs, Function &&func); +} -} // namespace vespalib::tensor::sparse -} // namespace vespalib::tensor diff --git a/eval/src/vespa/eval/tensor/sparse/sparse_tensor_builder.cpp b/eval/src/vespa/eval/tensor/sparse/sparse_tensor_builder.cpp index dacf0c27593..9c3b13f6260 100644 --- a/eval/src/vespa/eval/tensor/sparse/sparse_tensor_builder.cpp +++ b/eval/src/vespa/eval/tensor/sparse/sparse_tensor_builder.cpp @@ -3,8 +3,7 @@ #include "sparse_tensor_builder.h" #include <cassert> -namespace vespalib { -namespace tensor { +namespace vespalib::tensor { SparseTensorBuilder::SparseTensorBuilder() : TensorBuilder(), @@ -19,10 +18,7 @@ SparseTensorBuilder::SparseTensorBuilder() { } -SparseTensorBuilder::~SparseTensorBuilder() -{ -} - +SparseTensorBuilder::~SparseTensorBuilder() = default; void SparseTensorBuilder::makeType() @@ -103,6 +99,5 @@ SparseTensorBuilder::build() return ret; } +} -} // namespace vespalib::tensor -} // namespace vespalib diff --git a/eval/src/vespa/eval/tensor/sparse/sparse_tensor_builder.h b/eval/src/vespa/eval/tensor/sparse/sparse_tensor_builder.h index af1566d46c5..ea5f607ff7e 100644 --- a/eval/src/vespa/eval/tensor/sparse/sparse_tensor_builder.h +++ b/eval/src/vespa/eval/tensor/sparse/sparse_tensor_builder.h @@ -10,8 +10,7 @@ #include <vespa/vespalib/stllike/hash_map.h> #include <vespa/vespalib/util/stash.h> -namespace vespalib { -namespace tensor { +namespace vespalib::tensor { /** * A builder of sparse tensors. @@ -30,17 +29,13 @@ class SparseTensorBuilder : public TensorBuilder void makeType(); public: SparseTensorBuilder(); - virtual ~SparseTensorBuilder(); + ~SparseTensorBuilder() override; - virtual Dimension - define_dimension(const vespalib::string &dimension) override; - virtual TensorBuilder & - add_label(Dimension dimension, - const vespalib::string &label) override; - virtual TensorBuilder &add_cell(double value) override; - - virtual Tensor::UP build() override; + Dimension define_dimension(const vespalib::string &dimension) override; + TensorBuilder & add_label(Dimension dimension, const vespalib::string &label) override; + TensorBuilder &add_cell(double value) override; + Tensor::UP build() override; }; -} // namespace vespalib::tensor -} // namespace vespalib +} + diff --git a/eval/src/vespa/eval/tensor/sparse/sparse_tensor_unsorted_address_builder.cpp b/eval/src/vespa/eval/tensor/sparse/sparse_tensor_unsorted_address_builder.cpp index 1e112cbaa6e..866956dd23e 100644 --- a/eval/src/vespa/eval/tensor/sparse/sparse_tensor_unsorted_address_builder.cpp +++ b/eval/src/vespa/eval/tensor/sparse/sparse_tensor_unsorted_address_builder.cpp @@ -14,12 +14,10 @@ SparseTensorUnsortedAddressBuilder::SparseTensorUnsortedAddressBuilder() { } -SparseTensorUnsortedAddressBuilder::~SparseTensorUnsortedAddressBuilder() { -} +SparseTensorUnsortedAddressBuilder::~SparseTensorUnsortedAddressBuilder() = default; void -SparseTensorUnsortedAddressBuilder::buildTo(SparseTensorAddressBuilder & - builder, +SparseTensorUnsortedAddressBuilder::buildTo(SparseTensorAddressBuilder & builder, const eval::ValueType &type) { const char *base = &_elementStrings[0]; @@ -47,3 +45,4 @@ SparseTensorUnsortedAddressBuilder::buildTo(SparseTensorAddressBuilder & } } + diff --git a/eval/src/vespa/eval/tensor/sparse/sparse_tensor_unsorted_address_builder.h b/eval/src/vespa/eval/tensor/sparse/sparse_tensor_unsorted_address_builder.h index 24519e924d9..681bdabc5eb 100644 --- a/eval/src/vespa/eval/tensor/sparse/sparse_tensor_unsorted_address_builder.h +++ b/eval/src/vespa/eval/tensor/sparse/sparse_tensor_unsorted_address_builder.h @@ -6,9 +6,8 @@ #include <vector> #include <vespa/eval/tensor/types.h> -namespace vespalib { -namespace eval { class ValueType; } -namespace tensor { +namespace vespalib::eval { class ValueType; } +namespace vespalib::tensor { class SparseTensorAddressBuilder; @@ -73,11 +72,9 @@ public: * Sort the stored tensor address and pass it over to a strict * tensor address builder in sorted order. */ - void buildTo(SparseTensorAddressBuilder &builder, - const eval::ValueType &type); + void buildTo(SparseTensorAddressBuilder &builder, const eval::ValueType &type); void clear() { _elementStrings.clear(); _elements.clear(); } }; +} -} // namespace vespalib::tensor -} // namespace vespalib diff --git a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileDistributionRpcServer.java b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileDistributionRpcServer.java index 2e58455bc39..b2d1af15867 100644 --- a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileDistributionRpcServer.java +++ b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileDistributionRpcServer.java @@ -80,13 +80,13 @@ public class FileDistributionRpcServer { try { if (pathToFile.isPresent()) { req.returnValues().add(new StringValue(pathToFile.get().getAbsolutePath())); - log.log(LogLevel.INFO, "File reference '" + fileReference.value() + "' available at " + pathToFile.get()); + log.log(LogLevel.DEBUG, "File reference '" + fileReference.value() + "' available at " + pathToFile.get()); } else { log.log(LogLevel.INFO, "File reference '" + fileReference.value() + "' not found, returning error"); req.setError(fileReferenceDoesNotExists, "File reference '" + fileReference.value() + "' not found"); } } catch (Throwable e) { - log.log(LogLevel.WARNING, "File reference '" + fileReference.value() + "' got exeption: " + e.getMessage()); + log.log(LogLevel.WARNING, "File reference '" + fileReference.value() + "' got exception: " + e.getMessage()); req.setError(fileReferenceInternalError, "File reference '" + fileReference.value() + "' removed"); } req.returnRequest(); @@ -123,5 +123,4 @@ public class FileDistributionRpcServer { req.returnValues().add(new Int32Value(0)); } - } diff --git a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileDownloader.java b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileDownloader.java index 727786cdc78..5de006cd17c 100644 --- a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileDownloader.java +++ b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileDownloader.java @@ -107,7 +107,7 @@ public class FileDownloader { } else if (!file.canRead()) { throw new RuntimeException("File with reference '" + fileReference.value() + "'exists, but unable to read it"); } else { - fileReferenceDownloader.setDownloadStatus(fileReference.value(), 100.0); + fileReferenceDownloader.setDownloadStatus(fileReference, 100.0); return Optional.of(file); } } diff --git a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReceiver.java b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReceiver.java index d57ce4ca5de..70f22296bc1 100644 --- a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReceiver.java +++ b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReceiver.java @@ -85,7 +85,7 @@ public class FileReceiver { try { inprogressFile = Files.createTempFile(tmpDirectory.toPath(), fileName, ".inprogress").toFile(); } catch (IOException e) { - String msg = "Failed creating tempfile for inprogress file for(" + fileName + ") in '" + fileReferenceDir.toPath() + "': "; + String msg = "Failed creating temp file for inprogress file for(" + fileName + ") in '" + fileReferenceDir.toPath() + "': "; log.log(LogLevel.ERROR, msg + e.getMessage(), e); throw new RuntimeException(msg, e); } diff --git a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceDownloader.java b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceDownloader.java index 509231ba7ff..031506487a8 100644 --- a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceDownloader.java +++ b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceDownloader.java @@ -65,7 +65,7 @@ public class FileReferenceDownloader { Thread.sleep(10); } } - catch (InterruptedException e) {} + catch (InterruptedException e) { /* ignored */} } if ( !downloadStarted) { @@ -107,7 +107,7 @@ public class FileReferenceDownloader { if (validateResponse(request)) { log.log(LogLevel.DEBUG, "Request callback, OK. Req: " + request + "\nSpec: " + connection); if (request.returnValues().get(0).asInt32() == 0) { - log.log(LogLevel.INFO, "Found file reference '" + fileReference.value() + "' available at " + connection.getAddress()); + log.log(LogLevel.DEBUG, "Found file reference '" + fileReference.value() + "' available at " + connection.getAddress()); return true; } else { log.log(LogLevel.INFO, "File reference '" + fileReference.value() + "' not found for " + connection.getAddress()); @@ -169,10 +169,6 @@ public class FileReferenceDownloader { return status; } - void setDownloadStatus(String file, double completeness) { - setDownloadStatus(new FileReference(file), completeness); - } - void setDownloadStatus(FileReference fileReference, double completeness) { synchronized (downloads) { downloadStatus.put(fileReference, completeness); diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/util/ConfigServerHttpRequestExecutor.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/util/ConfigServerHttpRequestExecutor.java index 3576f37eb9a..94ad94d9a65 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/util/ConfigServerHttpRequestExecutor.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/util/ConfigServerHttpRequestExecutor.java @@ -43,7 +43,6 @@ import java.util.Optional; */ public class ConfigServerHttpRequestExecutor { private static final PrefixLogger NODE_ADMIN_LOGGER = PrefixLogger.getNodeAdminLogger(ConfigServerHttpRequestExecutor.class); - private static final int MAX_LOOPS = 2; private final ObjectMapper mapper = new ObjectMapper(); private final CloseableHttpClient client; @@ -108,43 +107,41 @@ public class ConfigServerHttpRequestExecutor { private <T> T tryAllConfigServers(CreateRequest requestFactory, Class<T> wantedReturnType) { Exception lastException = null; - for (int loopRetry = 0; loopRetry < MAX_LOOPS; loopRetry++) { - for (URI configServer : configServerHosts) { - final CloseableHttpResponse response; - try { - response = client.execute(requestFactory.createRequest(configServer)); - } catch (Exception e) { - // Failure to communicate with a config server is not abnormal, as they are - // upgraded at the same time as Docker hosts. - if (e.getMessage().indexOf("(Connection refused)") > 0) { - NODE_ADMIN_LOGGER.info("Connection refused to " + configServer + " (upgrading?), will try next"); - } else { - NODE_ADMIN_LOGGER.warning("Failed to communicate with " + configServer + ", will try next: " + e.getMessage()); - } - lastException = e; + for (URI configServer : configServerHosts) { + final CloseableHttpResponse response; + try { + response = client.execute(requestFactory.createRequest(configServer)); + } catch (Exception e) { + // Failure to communicate with a config server is not abnormal, as they are + // upgraded at the same time as Docker hosts. + if (e.getMessage().indexOf("(Connection refused)") > 0) { + NODE_ADMIN_LOGGER.info("Connection refused to " + configServer + " (upgrading?), will try next"); + } else { + NODE_ADMIN_LOGGER.warning("Failed to communicate with " + configServer + ", will try next: " + e.getMessage()); + } + lastException = e; + continue; + } + + try { + Optional<HttpException> retryableException = HttpException.handleStatusCode( + response.getStatusLine().getStatusCode(), + "Config server " + configServer); + if (retryableException.isPresent()) { + lastException = retryableException.get(); continue; } try { - Optional<HttpException> retryableException = HttpException.handleStatusCode( - response.getStatusLine().getStatusCode(), - "Config server " + configServer); - if (retryableException.isPresent()) { - lastException = retryableException.get(); - continue; - } - - try { - return mapper.readValue(response.getEntity().getContent(), wantedReturnType); - } catch (IOException e) { - throw new RuntimeException("Response didn't contain nodes element, failed parsing?", e); - } - } finally { - try { - response.close(); - } catch (IOException e) { - NODE_ADMIN_LOGGER.warning("Ignoring exception from closing response", e); - } + return mapper.readValue(response.getEntity().getContent(), wantedReturnType); + } catch (IOException e) { + throw new RuntimeException("Response didn't contain nodes element, failed parsing?", e); + } + } finally { + try { + response.close(); + } catch (IOException e) { + NODE_ADMIN_LOGGER.warning("Ignoring exception from closing response", e); } } } diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/util/ConfigServerHttpRequestExecutorTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/util/ConfigServerHttpRequestExecutorTest.java index 67cd2c79034..799f8a72fd9 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/util/ConfigServerHttpRequestExecutorTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/util/ConfigServerHttpRequestExecutorTest.java @@ -117,8 +117,7 @@ public class ConfigServerHttpRequestExecutorTest { } String[] log = mockLog.toString().split(" "); - assertThat(log, arrayContainingInAnyOrder("GET http://host1:666/path", "GET http://host2:666/path", - "GET http://host1:666/path", "GET http://host2:666/path")); + assertThat(log, arrayContainingInAnyOrder("GET http://host1:666/path", "GET http://host2:666/path")); } @Test @@ -134,7 +133,6 @@ public class ConfigServerHttpRequestExecutorTest { String[] log = mockLog.toString().split(" "); assertThat(log, arrayContainingInAnyOrder( - "GET http://host1:666/path", "GET http://host2:666/path", "GET http://host1:666/path", "GET http://host2:666/path")); } diff --git a/storage/src/vespa/storage/common/storagecomponent.cpp b/storage/src/vespa/storage/common/storagecomponent.cpp index bf387240dc5..1d6b563f6eb 100644 --- a/storage/src/vespa/storage/common/storagecomponent.cpp +++ b/storage/src/vespa/storage/common/storagecomponent.cpp @@ -28,14 +28,14 @@ StorageComponent::setNodeInfo(vespalib::stringref clusterName, void StorageComponent::setDocumentTypeRepo(DocumentTypeRepoSP repo) { - std::lock_guard<std::mutex> guard(_lock); + std::lock_guard guard(_lock); _docTypeRepo = repo; } void StorageComponent::setLoadTypes(LoadTypeSetSP loadTypes) { - std::lock_guard<std::mutex> guard(_lock); + std::lock_guard guard(_lock); _loadTypes = loadTypes; } @@ -57,14 +57,21 @@ StorageComponent::setBucketIdFactory(const document::BucketIdFactory& factory) void StorageComponent::setDistribution(DistributionSP distribution) { - std::lock_guard<std::mutex> guard(_lock); + std::lock_guard guard(_lock); _distribution = distribution; } void +StorageComponent::enableMultipleBucketSpaces(bool value) +{ + std::lock_guard guard(_lock); + _enableMultipleBucketSpaces = value; +} + +void StorageComponent::setNodeStateUpdater(NodeStateUpdater& updater) { - std::lock_guard<std::mutex> guard(_lock); + std::lock_guard guard(_lock); if (_nodeStateUpdater != 0) { throw vespalib::IllegalStateException( "Node state updater is already set", VESPA_STRLOC); @@ -76,10 +83,16 @@ StorageComponent::StorageComponent(StorageComponentRegister& compReg, vespalib::stringref name) : Component(compReg, name), _clusterName(), - _nodeType(0), + _nodeType(nullptr), _index(0), + _docTypeRepo(), + _loadTypes(), _priorityMapper(new PriorityMapper), - _nodeStateUpdater(0) + _bucketIdFactory(), + _distribution(), + _nodeStateUpdater(nullptr), + _lock(), + _enableMultipleBucketSpaces(false) { compReg.registerStorageComponent(*this); } @@ -87,7 +100,7 @@ StorageComponent::StorageComponent(StorageComponentRegister& compReg, NodeStateUpdater& StorageComponent::getStateUpdater() const { - std::lock_guard<std::mutex> guard(_lock); + std::lock_guard guard(_lock); if (_nodeStateUpdater == 0) { throw vespalib::IllegalStateException( "Component need node state updater at this time, but it has " @@ -114,22 +127,29 @@ StorageComponent::getPriority(const documentapi::LoadType& lt) const StorageComponent::DocumentTypeRepoSP StorageComponent::getTypeRepo() const { - std::lock_guard<std::mutex> guard(_lock); + std::lock_guard guard(_lock); return _docTypeRepo; } StorageComponent::LoadTypeSetSP StorageComponent::getLoadTypes() const { - std::lock_guard<std::mutex> guard(_lock); + std::lock_guard guard(_lock); return _loadTypes; } StorageComponent::DistributionSP StorageComponent::getDistribution() const { - std::lock_guard<std::mutex> guard(_lock); + std::lock_guard guard(_lock); return _distribution; } +bool +StorageComponent::enableMultipleBucketSpaces() const +{ + std::lock_guard guard(_lock); + return _enableMultipleBucketSpaces; +} + } // storage diff --git a/storage/src/vespa/storage/common/storagecomponent.h b/storage/src/vespa/storage/common/storagecomponent.h index d469540b55f..e136d991ac5 100644 --- a/storage/src/vespa/storage/common/storagecomponent.h +++ b/storage/src/vespa/storage/common/storagecomponent.h @@ -37,10 +37,9 @@ #include <vespa/vdslib/state/node.h> #include <mutex> -namespace vespa { namespace config { namespace content { namespace core { -namespace internal { +namespace vespa::config::content::core::internal { class InternalStorPrioritymappingType; -} } } } } +} namespace document { class DocumentTypeRepo; } @@ -59,11 +58,11 @@ class StorageComponentRegister; class StorageComponent : public framework::Component { public: - typedef std::unique_ptr<StorageComponent> UP; - typedef vespa::config::content::core::internal::InternalStorPrioritymappingType PriorityConfig; - typedef std::shared_ptr<document::DocumentTypeRepo> DocumentTypeRepoSP; - typedef std::shared_ptr<documentapi::LoadTypeSet> LoadTypeSetSP; - typedef std::shared_ptr<lib::Distribution> DistributionSP; + using UP = std::unique_ptr<StorageComponent>; + using PriorityConfig = vespa::config::content::core::internal::InternalStorPrioritymappingType; + using DocumentTypeRepoSP = std::shared_ptr<document::DocumentTypeRepo>; + using LoadTypeSetSP = std::shared_ptr<documentapi::LoadTypeSet>; + using DistributionSP = std::shared_ptr<lib::Distribution>; /** * Node type is supposed to be set immediately, and never be updated. @@ -84,6 +83,7 @@ public: void setPriorityConfig(const PriorityConfig&); void setBucketIdFactory(const document::BucketIdFactory&); void setDistribution(DistributionSP); + void enableMultipleBucketSpaces(bool value); StorageComponent(StorageComponentRegister&, vespalib::stringref name); virtual ~StorageComponent(); @@ -102,6 +102,7 @@ public: uint8_t getPriority(const documentapi::LoadType&) const; DistributionSP getDistribution() const; NodeStateUpdater& getStateUpdater() const; + bool enableMultipleBucketSpaces() const; private: vespalib::string _clusterName; @@ -114,6 +115,7 @@ private: DistributionSP _distribution; NodeStateUpdater* _nodeStateUpdater; mutable std::mutex _lock; + bool _enableMultipleBucketSpaces; }; struct StorageComponentRegister : public virtual framework::ComponentRegister diff --git a/storage/src/vespa/storage/frameworkimpl/component/storagecomponentregisterimpl.cpp b/storage/src/vespa/storage/frameworkimpl/component/storagecomponentregisterimpl.cpp index d2cb8fa4380..2f9430c49bb 100644 --- a/storage/src/vespa/storage/frameworkimpl/component/storagecomponentregisterimpl.cpp +++ b/storage/src/vespa/storage/frameworkimpl/component/storagecomponentregisterimpl.cpp @@ -10,11 +10,20 @@ LOG_SETUP(".storage.component.register"); namespace storage { StorageComponentRegisterImpl::StorageComponentRegisterImpl() - : _nodeType(0), + : _componentLock(), + _components(), + _clusterName(), + _nodeType(nullptr), _index(0xffff), + _docTypeRepo(), _loadTypes(new documentapi::LoadTypeSet), - _nodeStateUpdater(0) -{ } + _priorityConfig(), + _bucketIdFactory(), + _distribution(), + _nodeStateUpdater(nullptr), + _bucketSpacesConfig() +{ +} StorageComponentRegisterImpl::~StorageComponentRegisterImpl() { } @@ -33,6 +42,7 @@ StorageComponentRegisterImpl::registerStorageComponent(StorageComponent& smc) smc.setPriorityConfig(_priorityConfig); smc.setBucketIdFactory(_bucketIdFactory); smc.setDistribution(_distribution); + smc.enableMultipleBucketSpaces(_bucketSpacesConfig.enableMultipleBucketSpaces); } void @@ -115,4 +125,14 @@ StorageComponentRegisterImpl::setDistribution(lib::Distribution::SP distribution } } +void +StorageComponentRegisterImpl::setBucketSpacesConfig(const BucketspacesConfig& config) +{ + vespalib::LockGuard lock(_componentLock); + _bucketSpacesConfig = config; + for (size_t i = 0; i < _components.size(); ++i) { + _components[i]->enableMultipleBucketSpaces(config.enableMultipleBucketSpaces); + } +} + } // storage diff --git a/storage/src/vespa/storage/frameworkimpl/component/storagecomponentregisterimpl.h b/storage/src/vespa/storage/frameworkimpl/component/storagecomponentregisterimpl.h index 49387e2c2b5..afd9f11a88b 100644 --- a/storage/src/vespa/storage/frameworkimpl/component/storagecomponentregisterimpl.h +++ b/storage/src/vespa/storage/frameworkimpl/component/storagecomponentregisterimpl.h @@ -11,6 +11,7 @@ #include <vespa/document/repo/documenttyperepo.h> #include <vespa/documentapi/loadtypes/loadtypeset.h> #include <vespa/storage/common/storagecomponent.h> +#include <vespa/storage/config/config-bucketspaces.h> #include <vespa/storage/config/config-stor-prioritymapping.h> #include <vespa/storageframework/defaultimplementation/component/componentregisterimpl.h> #include <vespa/vdslib/distribution/distribution.h> @@ -21,9 +22,9 @@ class StorageComponentRegisterImpl : public virtual StorageComponentRegister, public virtual framework::defaultimplementation::ComponentRegisterImpl { - typedef framework::defaultimplementation::ComponentRegisterImpl CompRegImpl; - typedef StorageComponent::PriorityConfig PriorityConfig; - //CompRegImpl _compReg; + using PriorityConfig = StorageComponent::PriorityConfig; + using BucketspacesConfig = vespa::config::content::core::internal::InternalBucketspacesType; + vespalib::Lock _componentLock; std::vector<StorageComponent*> _components; vespalib::string _clusterName; @@ -35,6 +36,7 @@ class StorageComponentRegisterImpl document::BucketIdFactory _bucketIdFactory; lib::Distribution::SP _distribution; NodeStateUpdater* _nodeStateUpdater; + BucketspacesConfig _bucketSpacesConfig; public: typedef std::unique_ptr<StorageComponentRegisterImpl> UP; @@ -64,6 +66,7 @@ public: virtual void setPriorityConfig(const PriorityConfig&); virtual void setBucketIdFactory(const document::BucketIdFactory&); virtual void setDistribution(lib::Distribution::SP); + virtual void setBucketSpacesConfig(const BucketspacesConfig&); }; diff --git a/storage/src/vespa/storage/storageserver/storagenode.cpp b/storage/src/vespa/storage/storageserver/storagenode.cpp index ba1556bd3b9..d60f46e5a07 100644 --- a/storage/src/vespa/storage/storageserver/storagenode.cpp +++ b/storage/src/vespa/storage/storageserver/storagenode.cpp @@ -76,12 +76,38 @@ StorageNode::StorageNode( std::unique_ptr<HostInfo> hostInfo, RunMode mode) : _singleThreadedDebugMode(mode == SINGLE_THREADED_TEST_MODE), + _configFetcher(), _hostInfo(std::move(hostInfo)), _context(context), _generationFetcher(generationFetcher), + _rootFolder(), _attemptedStopped(false), + _pidFile(), + _statusWebServer(), + _metrics(), + _metricManager(), + _deadLockDetector(), + _statusMetrics(), + _stateReporter(), + _stateManager(), + _chain(), + _configLock(), + _initial_config_mutex(), + _serverConfig(), + _clusterConfig(), + _distributionConfig(), + _priorityConfig(), + _doctypesConfig(), + _bucketSpacesConfig(), + _newServerConfig(), + _newClusterConfig(), + _newDistributionConfig(), + _newPriorityConfig(), + _newDoctypesConfig(), + _newBucketSpacesConfig(), + _component(), _configUri(configUri), - _communicationManager(0) + _communicationManager(nullptr) { } @@ -93,6 +119,7 @@ StorageNode::subscribeToConfigs() _configFetcher->subscribe<UpgradingConfig>(_configUri.getConfigId(), this); _configFetcher->subscribe<StorServerConfig>(_configUri.getConfigId(), this); _configFetcher->subscribe<StorPrioritymappingConfig>(_configUri.getConfigId(), this); + _configFetcher->subscribe<BucketspacesConfig>(_configUri.getConfigId(), this); _configFetcher->start(); @@ -101,6 +128,7 @@ StorageNode::subscribeToConfigs() _clusterConfig = std::move(_newClusterConfig); _distributionConfig = std::move(_newDistributionConfig); _priorityConfig = std::move(_newPriorityConfig); + _bucketSpacesConfig = std::move(_newBucketSpacesConfig); } void @@ -127,6 +155,7 @@ StorageNode::initialize() _context.getComponentRegister().setBucketIdFactory(document::BucketIdFactory()); _context.getComponentRegister().setDistribution(make_shared<lib::Distribution>(*_distributionConfig)); _context.getComponentRegister().setPriorityConfig(*_priorityConfig); + _context.getComponentRegister().setBucketSpacesConfig(*_bucketSpacesConfig); _metrics.reset(new StorageMetricSet); _component.reset(new StorageComponent(_context.getComponentRegister(), "storagenode")); @@ -315,6 +344,11 @@ StorageNode::handleLiveConfigUpdate(const InitialGuard & initGuard) _priorityConfig = std::move(_newPriorityConfig); _context.getComponentRegister().setPriorityConfig(*_priorityConfig); } + if (_newBucketSpacesConfig) { + _bucketSpacesConfig = std::move(_newBucketSpacesConfig); + _context.getComponentRegister().setBucketSpacesConfig(*_bucketSpacesConfig); + // TODO: Add new bucket space resolver to document api converter + } } void @@ -430,7 +464,7 @@ void StorageNode::configure(std::unique_ptr<StorServerConfig> config) // updates { vespalib::LockGuard configLockGuard(_configLock); - _newServerConfig.reset(config.release()); + _newServerConfig = std::move(config); } if (_serverConfig) { InitialGuard concurrent_config_guard(_initial_config_mutex); @@ -447,7 +481,7 @@ StorageNode::configure(std::unique_ptr<UpgradingConfig> config) // updates { vespalib::LockGuard configLockGuard(_configLock); - _newClusterConfig.reset(config.release()); + _newClusterConfig = std::move(config); } if (_clusterConfig) { InitialGuard concurrent_config_guard(_initial_config_mutex); @@ -464,7 +498,7 @@ StorageNode::configure(std::unique_ptr<StorDistributionConfig> config) // updates { vespalib::LockGuard configLockGuard(_configLock); - _newDistributionConfig.reset(config.release()); + _newDistributionConfig = std::move(config); } if (_distributionConfig) { InitialGuard concurrent_config_guard(_initial_config_mutex); @@ -477,7 +511,7 @@ StorageNode::configure(std::unique_ptr<StorPrioritymappingConfig> config) { { vespalib::LockGuard configLockGuard(_configLock); - _newPriorityConfig.reset(config.release()); + _newPriorityConfig = std::move(config); } if (_priorityConfig) { InitialGuard concurrent_config_guard(_initial_config_mutex); @@ -485,15 +519,16 @@ StorageNode::configure(std::unique_ptr<StorPrioritymappingConfig> config) } } -void StorageNode::configure(std::unique_ptr<document::DocumenttypesConfig> config, - bool hasChanged, int64_t generation) +void +StorageNode::configure(std::unique_ptr<document::DocumenttypesConfig> config, + bool hasChanged, int64_t generation) { (void) generation; if (!hasChanged) return; { vespalib::LockGuard configLockGuard(_configLock); - _newDoctypesConfig.reset(config.release()); + _newDoctypesConfig = std::move(config); } if (_doctypesConfig) { InitialGuard concurrent_config_guard(_initial_config_mutex); @@ -501,6 +536,19 @@ void StorageNode::configure(std::unique_ptr<document::DocumenttypesConfig> confi } } +void +StorageNode::configure(std::unique_ptr<BucketspacesConfig> config) +{ + { + vespalib::LockGuard configLockGuard(_configLock); + _newBucketSpacesConfig = std::move(config); + } + if (_bucketSpacesConfig) { + InitialGuard concurrent_config_guard(_initial_config_mutex); + handleLiveConfigUpdate(concurrent_config_guard); + } +} + bool StorageNode::attemptedStopped() const { diff --git a/storage/src/vespa/storage/storageserver/storagenode.h b/storage/src/vespa/storage/storageserver/storagenode.h index e9d3004be68..a07d1c0c534 100644 --- a/storage/src/vespa/storage/storageserver/storagenode.h +++ b/storage/src/vespa/storage/storageserver/storagenode.h @@ -12,20 +12,19 @@ #pragma once -#include <vespa/storage/storageutil/resumeguard.h> -#include <vespa/storage/common/doneinitializehandler.h> -#include <vespa/storageframework/generic/metric/metricupdatehook.h> -#include <vespa/storageframework/defaultimplementation/component/componentregisterimpl.h> - -#include <vespa/config/subscription/configuri.h> -#include <vespa/config/helper/ifetchercallback.h> +#include <vespa/config-stor-distribution.h> +#include <vespa/config-upgrading.h> #include <vespa/config/helper/configfetcher.h> - +#include <vespa/config/helper/ifetchercallback.h> +#include <vespa/config/subscription/configuri.h> +#include <vespa/document/config/config-documenttypes.h> +#include <vespa/storage/common/doneinitializehandler.h> +#include <vespa/storage/config/config-bucketspaces.h> #include <vespa/storage/config/config-stor-prioritymapping.h> #include <vespa/storage/config/config-stor-server.h> -#include <vespa/document/config/config-documenttypes.h> -#include <vespa/config-upgrading.h> -#include <vespa/config-stor-distribution.h> +#include <vespa/storage/storageutil/resumeguard.h> +#include <vespa/storageframework/defaultimplementation/component/componentregisterimpl.h> +#include <vespa/storageframework/generic/metric/metricupdatehook.h> #include <mutex> namespace document { class DocumentTypeRepo; } @@ -54,6 +53,7 @@ class StorageNode : private config::IFetcherCallback<vespa::config::content::cor private config::IFetcherCallback<vespa::config::content::StorDistributionConfig>, private config::IFetcherCallback<vespa::config::content::UpgradingConfig>, private config::IFetcherCallback<vespa::config::content::core::StorPrioritymappingConfig>, + private config::IFetcherCallback<vespa::config::content::core::BucketspacesConfig>, private framework::MetricUpdateHook, private DoneInitializeHandler, private framework::defaultimplementation::ShutdownListener @@ -101,6 +101,7 @@ protected: using UpgradingConfig = vespa::config::content::UpgradingConfig; using StorDistributionConfig = vespa::config::content::StorDistributionConfig; using StorPrioritymappingConfig = vespa::config::content::core::StorPrioritymappingConfig; + using BucketspacesConfig = vespa::config::content::core::BucketspacesConfig; private: bool _singleThreadedDebugMode; // Subscriptions to config @@ -137,6 +138,7 @@ private: void configure(std::unique_ptr<StorPrioritymappingConfig>) override; virtual void configure(std::unique_ptr<document::DocumenttypesConfig> config, bool hasChanged, int64_t generation); + void configure(std::unique_ptr<BucketspacesConfig>) override; void updateUpgradeFlag(const UpgradingConfig&); protected: @@ -151,12 +153,14 @@ protected: std::unique_ptr<StorDistributionConfig> _distributionConfig; std::unique_ptr<StorPrioritymappingConfig> _priorityConfig; std::unique_ptr<document::DocumenttypesConfig> _doctypesConfig; + std::unique_ptr<BucketspacesConfig> _bucketSpacesConfig; // New configs gotten that has yet to have been handled std::unique_ptr<StorServerConfig> _newServerConfig; std::unique_ptr<UpgradingConfig> _newClusterConfig; std::unique_ptr<StorDistributionConfig> _newDistributionConfig; std::unique_ptr<StorPrioritymappingConfig> _newPriorityConfig; std::unique_ptr<document::DocumenttypesConfig> _newDoctypesConfig; + std::unique_ptr<BucketspacesConfig> _newBucketSpacesConfig; std::unique_ptr<StorageComponent> _component; config::ConfigUri _configUri; CommunicationManager* _communicationManager; diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/operationProcessor/OperationProcessor.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/operationProcessor/OperationProcessor.java index 0a9fe72552c..5907694f55a 100644 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/operationProcessor/OperationProcessor.java +++ b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/operationProcessor/OperationProcessor.java @@ -177,10 +177,14 @@ public class OperationProcessor { docSendInfoByOperationId.remove(endpointResult.getOperationId()); String documentId = documentSendInfo.getDocument().getDocumentId(); - inflightDocumentIds.remove(documentId); - + /** + * If we got a pending operation against this document + * dont't remove it from inflightDocuments and send blocked document operation + */ List<Document> blockedDocuments = blockedDocumentsByDocumentId.get(documentId); - if (! blockedDocuments.isEmpty()) { + if (blockedDocuments.isEmpty()) { + inflightDocumentIds.remove(documentId); + } else { sendToClusters(blockedDocuments.remove(0)); } return result; diff --git a/vespalib/src/vespa/vespalib/stllike/hash_map.hpp b/vespalib/src/vespa/vespalib/stllike/hash_map.hpp index 359ba235a36..40ef90826b6 100644 --- a/vespalib/src/vespa/vespalib/stllike/hash_map.hpp +++ b/vespalib/src/vespa/vespalib/stllike/hash_map.hpp @@ -64,12 +64,18 @@ hash_map<K, V, H, EQ, M>::getMemoryUsed() const } -#define VESPALIB_HASH_MAP_INSTANTIATE_H(K, V, H) \ - template class vespalib::hash_map<K, V, H>; \ - template class vespalib::hashtable<K, std::pair<K,V>, H, std::equal_to<K>, std::_Select1st<std::pair<K,V>>>; \ - template vespalib::hashtable<K, std::pair<K,V>, H, std::equal_to<K>, std::_Select1st<std::pair<K,V>>>::insert_result \ - vespalib::hashtable<K, std::pair<K,V>, H, std::equal_to<K>, std::_Select1st<std::pair<K,V>>>::insert(std::pair<K,V> &&); \ + +#define VESPALIB_HASH_MAP_INSTANTIATE_H_E_M(K, V, H, E, M) \ + template class vespalib::hash_map<K, V, H, E, M>; \ + template class vespalib::hashtable<K, std::pair<K,V>, H, E, std::_Select1st<std::pair<K,V>>, M>; \ + template vespalib::hashtable<K, std::pair<K,V>, H, E, std::_Select1st<std::pair<K,V>>, M>::insert_result \ + vespalib::hashtable<K, std::pair<K,V>, H, E, std::_Select1st<std::pair<K,V>>, M>::insert(std::pair<K,V> &&); \ template class vespalib::Array<vespalib::hash_node<std::pair<K,V>>>; +#define VESPALIB_HASH_MAP_INSTANTIATE_H_E(K, V, H, E) \ + VESPALIB_HASH_MAP_INSTANTIATE_H_E_M(K, V, H, E, vespalib::hashtable_base::prime_modulator) + +#define VESPALIB_HASH_MAP_INSTANTIATE_H(K, V, H) VESPALIB_HASH_MAP_INSTANTIATE_H_E(K, V, H, std::equal_to<K>) + #define VESPALIB_HASH_MAP_INSTANTIATE(K, V) VESPALIB_HASH_MAP_INSTANTIATE_H(K, V, vespalib::hash<K>) |