diff options
15 files changed, 61 insertions, 42 deletions
diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java index 0a96b3c1030..f15d700a398 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java @@ -77,7 +77,7 @@ public interface ModelContext { @ModelFeatureFlag(owners = {"baldersheim"}) default boolean skipCommunicationManagerThread() { throw new UnsupportedOperationException("TODO specify default value"); } @ModelFeatureFlag(owners = {"baldersheim"}) default boolean skipMbusRequestThread() { throw new UnsupportedOperationException("TODO specify default value"); } @ModelFeatureFlag(owners = {"baldersheim"}) default boolean skipMbusReplyThread() { throw new UnsupportedOperationException("TODO specify default value"); } - @ModelFeatureFlag(owners = {"tokle"}) default boolean useAccessControlTlsHandshakeClientAuth() { return false; } + @ModelFeatureFlag(owners = {"tokle"}) default boolean useAccessControlTlsHandshakeClientAuth() { return true; } @ModelFeatureFlag(owners = {"baldersheim"}) default boolean useAsyncMessageHandlingOnSchedule() { throw new UnsupportedOperationException("TODO specify default value"); } @ModelFeatureFlag(owners = {"baldersheim"}) default double feedConcurrency() { throw new UnsupportedOperationException("TODO specify default value"); } @ModelFeatureFlag(owners = {"baldersheim"}) default boolean useBucketExecutorForLidSpaceCompact() { throw new UnsupportedOperationException("TODO specify default value"); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java index d03246b4e8c..d713ae2e016 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java @@ -435,8 +435,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { } EndpointCertificateSecrets endpointCertificateSecrets = deployState.endpointCertificateSecrets().get(); - boolean enforceHandshakeClientAuth = context.properties().featureFlags().useAccessControlTlsHandshakeClientAuth() && - cluster.getHttp().getAccessControl() + boolean enforceHandshakeClientAuth = cluster.getHttp().getAccessControl() .map(accessControl -> accessControl.clientAuthentication) .map(clientAuth -> clientAuth.equals(AccessControl.ClientAuthentication.need)) .orElse(false); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java index a1a8548f5f1..b4dd81ad7aa 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java @@ -166,7 +166,6 @@ public class ModelContextImpl implements ModelContext { private final boolean skipCommunicationManagerThread; private final boolean skipMbusRequestThread; private final boolean skipMbusReplyThread; - private final boolean useAccessControlTlsHandshakeClientAuth; private final boolean useAsyncMessageHandlingOnSchedule; private final double feedConcurrency; private final boolean useBucketExecutorForLidSpaceCompact; @@ -191,7 +190,6 @@ public class ModelContextImpl implements ModelContext { this.skipCommunicationManagerThread = flagValue(source, appId, Flags.SKIP_COMMUNICATIONMANAGER_THREAD); this.skipMbusRequestThread = flagValue(source, appId, Flags.SKIP_MBUS_REQUEST_THREAD); this.skipMbusReplyThread = flagValue(source, appId, Flags.SKIP_MBUS_REPLY_THREAD); - this.useAccessControlTlsHandshakeClientAuth = flagValue(source, appId, Flags.USE_ACCESS_CONTROL_CLIENT_AUTHENTICATION); this.useAsyncMessageHandlingOnSchedule = flagValue(source, appId, Flags.USE_ASYNC_MESSAGE_HANDLING_ON_SCHEDULE); this.feedConcurrency = flagValue(source, appId, Flags.FEED_CONCURRENCY); this.useBucketExecutorForLidSpaceCompact = flagValue(source, appId, Flags.USE_BUCKET_EXECUTOR_FOR_LID_SPACE_COMPACT); @@ -216,7 +214,6 @@ public class ModelContextImpl implements ModelContext { @Override public boolean skipCommunicationManagerThread() { return skipCommunicationManagerThread; } @Override public boolean skipMbusRequestThread() { return skipMbusRequestThread; } @Override public boolean skipMbusReplyThread() { return skipMbusReplyThread; } - @Override public boolean useAccessControlTlsHandshakeClientAuth() { return useAccessControlTlsHandshakeClientAuth; } @Override public boolean useAsyncMessageHandlingOnSchedule() { return useAsyncMessageHandlingOnSchedule; } @Override public double feedConcurrency() { return feedConcurrency; } @Override public boolean useBucketExecutorForLidSpaceCompact() { return useBucketExecutorForLidSpaceCompact; } diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index 4576b546d2e..00d6662e84a 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -151,13 +151,6 @@ public class Flags { APPLICATION_ID ); - public static final UnboundBooleanFlag USE_ACCESS_CONTROL_CLIENT_AUTHENTICATION = defineFeatureFlag( - "use-access-control-client-authentication", false, - List.of("tokle"), "2020-12-02", "2021-04-01", - "Whether application container should set up client authentication on default port based on access control element", - "Takes effect on next internal redeployment", - APPLICATION_ID); - public static final UnboundBooleanFlag USE_ASYNC_MESSAGE_HANDLING_ON_SCHEDULE = defineFeatureFlag( "async-message-handling-on-schedule", false, List.of("baldersheim"), "2020-12-02", "2022-01-01", diff --git a/searchlib/src/vespa/searchlib/attribute/attribute_blueprint_factory.cpp b/searchlib/src/vespa/searchlib/attribute/attribute_blueprint_factory.cpp index 1edcd67f5cb..449d5ac24ca 100644 --- a/searchlib/src/vespa/searchlib/attribute/attribute_blueprint_factory.cpp +++ b/searchlib/src/vespa/searchlib/attribute/attribute_blueprint_factory.cpp @@ -80,6 +80,7 @@ using search::queryeval::Searchable; using search::queryeval::SimpleLeafBlueprint; using search::queryeval::WeightedSetTermBlueprint; using search::tensor::DenseTensorAttribute; +using search::tensor::ITensorAttribute; using vespalib::geo::ZCurve; using vespalib::make_string; using vespalib::string; @@ -725,17 +726,14 @@ public: setResult(std::make_unique<queryeval::EmptyBlueprint>(_field)); } void visit(query::NearestNeighborTerm &n) override { - if (_attr.asTensorAttribute() == nullptr) { + const ITensorAttribute *tensor_attr = _attr.asTensorAttribute(); + if (tensor_attr == nullptr) { return fail_nearest_neighbor_term(n, "Attribute is not a tensor"); } - const auto* dense_attr_tensor = dynamic_cast<const DenseTensorAttribute*>(_attr.asTensorAttribute()); - if (dense_attr_tensor == nullptr) { - return fail_nearest_neighbor_term(n, make_string("Attribute is not a dense tensor (type=%s)", - _attr.asTensorAttribute()->getTensorType().to_spec().c_str())); - } - if (dense_attr_tensor->getTensorType().dimensions().size() != 1) { - return fail_nearest_neighbor_term(n, make_string("Attribute tensor type (%s) is not of order 1", - dense_attr_tensor->getTensorType().to_spec().c_str())); + const auto & ta_type = tensor_attr->getTensorType(); + if ((! ta_type.is_dense()) || (ta_type.dimensions().size() != 1)) { + return fail_nearest_neighbor_term(n, make_string("Attribute tensor type (%s) is not a dense tensor of order 1", + ta_type.to_spec().c_str())); } auto query_tensor = getRequestContext().get_query_tensor(n.get_query_tensor_name()); if (query_tensor.get() == nullptr) { @@ -746,11 +744,15 @@ public: return fail_nearest_neighbor_term(n, make_string("Query tensor is not a dense tensor (type=%s)", qt_type.to_spec().c_str())); } - if (!is_compatible_for_nearest_neighbor(dense_attr_tensor->getTensorType(), qt_type)) { + if (!is_compatible_for_nearest_neighbor(ta_type, qt_type)) { return fail_nearest_neighbor_term(n, make_string("Attribute tensor type (%s) and query tensor type (%s) are not compatible", - dense_attr_tensor->getTensorType().to_spec().c_str(), qt_type.to_spec().c_str())); + ta_type.to_spec().c_str(), qt_type.to_spec().c_str())); + } + if (tensor_attr->supports_extract_cells_ref() == false) { + return fail_nearest_neighbor_term(n, make_string("Attribute does not support access to tensor data (type=%s)", + ta_type.to_spec().c_str())); } - setResult(std::make_unique<queryeval::NearestNeighborBlueprint>(_field, *dense_attr_tensor, + setResult(std::make_unique<queryeval::NearestNeighborBlueprint>(_field, *tensor_attr, std::move(query_tensor), n.get_target_num_hits(), n.get_allow_approximate(), diff --git a/searchlib/src/vespa/searchlib/attribute/postinglistsearchcontext.cpp b/searchlib/src/vespa/searchlib/attribute/postinglistsearchcontext.cpp index 576506aeea9..b7517094b19 100644 --- a/searchlib/src/vespa/searchlib/attribute/postinglistsearchcontext.cpp +++ b/searchlib/src/vespa/searchlib/attribute/postinglistsearchcontext.cpp @@ -57,6 +57,10 @@ void PostingListSearchContext::lookupRange(const vespalib::datastore::EntryComparator &low, const vespalib::datastore::EntryComparator &high) { + if (!_dictionary.get_has_btree_dictionary()) { + _uniqueValues = 2; // Avoid zero and single value optimizations, use filtering + return; + } _lowerDictItr.lower_bound(_frozenDictionary.getRoot(), EnumIndex(), low); _upperDictItr = _lowerDictItr; if (_upperDictItr.valid() && !high.less(EnumIndex(), _upperDictItr.getKey())) { diff --git a/searchlib/src/vespa/searchlib/attribute/postinglistsearchcontext.h b/searchlib/src/vespa/searchlib/attribute/postinglistsearchcontext.h index 3777f3cf4ea..22e9987aa9e 100644 --- a/searchlib/src/vespa/searchlib/attribute/postinglistsearchcontext.h +++ b/searchlib/src/vespa/searchlib/attribute/postinglistsearchcontext.h @@ -80,6 +80,9 @@ protected: } virtual bool fallbackToFiltering() const { + if (_uniqueValues >= 2 && !_dictionary.get_has_btree_dictionary()) { + return true; // force filtering for range search + } uint32_t numHits = calculateApproxNumHits(); // numHits > 1000: make sure that posting lists are unit tested. return (numHits > 1000) && @@ -216,7 +219,7 @@ private: bool fallbackToFiltering() const override { return (this->getRangeLimit() != 0) - ? false + ? (this->_uniqueValues >= 2 && !this->_dictionary.get_has_btree_dictionary()) : Parent::fallbackToFiltering(); } unsigned int approximateHits() const override { @@ -339,6 +342,11 @@ getIterators(bool shouldApplyRangeLimit) auto compHigh = _enumStore.make_comparator(capped.upper()); this->lookupRange(compLow, compHigh); + if (!this->_dictionary.get_has_btree_dictionary()) { + _low = capped.lower(); + _high = capped.upper(); + return; + } if (shouldApplyRangeLimit) { this->applyRangeLimit(this->getRangeLimit()); } diff --git a/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_blueprint.cpp b/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_blueprint.cpp index 4d69bebe065..4ec23b993b6 100644 --- a/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_blueprint.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_blueprint.cpp @@ -46,7 +46,7 @@ struct ConvertCellsSelector } // namespace <unnamed> NearestNeighborBlueprint::NearestNeighborBlueprint(const queryeval::FieldSpec& field, - const tensor::DenseTensorAttribute& attr_tensor, + const tensor::ITensorAttribute& attr_tensor, std::unique_ptr<Value> query_tensor, uint32_t target_num_hits, bool approximate, uint32_t explore_additional_hits, double distance_threshold, double brute_force_limit) @@ -68,7 +68,7 @@ NearestNeighborBlueprint::NearestNeighborBlueprint(const queryeval::FieldSpec& f using MyTypify = vespalib::eval::TypifyCellType; auto fixup_fun = vespalib::typify_invoke<2,MyTypify,ConvertCellsSelector>(lct, rct); fixup_fun(_query_tensor, _attr_tensor.getTensorType()); - _fallback_dist_fun = search::tensor::make_distance_function(_attr_tensor.getConfig().distance_metric(), rct); + _fallback_dist_fun = search::tensor::make_distance_function(_attr_tensor.distance_metric(), rct); _dist_fun = _fallback_dist_fun.get(); assert(_dist_fun); auto nns_index = _attr_tensor.nearest_neighbor_index(); @@ -80,7 +80,7 @@ NearestNeighborBlueprint::NearestNeighborBlueprint(const queryeval::FieldSpec& f _distance_threshold = _dist_fun->convert_threshold(distance_threshold); _distance_heap.set_distance_threshold(_distance_threshold); } - uint32_t est_hits = _attr_tensor.getNumDocs(); + uint32_t est_hits = _attr_tensor.get_num_docs(); setEstimate(HitEstimate(est_hits, false)); set_want_global_filter(true); } @@ -97,7 +97,7 @@ NearestNeighborBlueprint::set_global_filter(const GlobalFilter &global_filter) (nns_index ? "nns_index" : "no_index"), (_global_filter->has_filter() ? "has_filter" : "no_filter")); if (_approximate && nns_index) { - uint32_t est_hits = _attr_tensor.getNumDocs(); + uint32_t est_hits = _attr_tensor.get_num_docs(); if (_global_filter->has_filter()) { uint32_t max_hits = _global_filter->filter()->countTrueBits(); LOG(debug, "set_global_filter getNumDocs: %u / max_hits %u", est_hits, max_hits); diff --git a/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_blueprint.h b/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_blueprint.h index aad43c923a2..827dafacf57 100644 --- a/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_blueprint.h +++ b/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_blueprint.h @@ -6,7 +6,7 @@ #include <vespa/searchlib/tensor/distance_function.h> #include <vespa/searchlib/tensor/nearest_neighbor_index.h> -namespace search::tensor { class DenseTensorAttribute; } +namespace search::tensor { class ITensorAttribute; } namespace vespalib::eval { struct Value; } namespace search::queryeval { @@ -19,7 +19,7 @@ namespace search::queryeval { */ class NearestNeighborBlueprint : public ComplexLeafBlueprint { private: - const tensor::DenseTensorAttribute& _attr_tensor; + const tensor::ITensorAttribute& _attr_tensor; std::unique_ptr<vespalib::eval::Value> _query_tensor; uint32_t _target_num_hits; bool _approximate; @@ -35,7 +35,7 @@ private: void perform_top_k(); public: NearestNeighborBlueprint(const queryeval::FieldSpec& field, - const tensor::DenseTensorAttribute& attr_tensor, + const tensor::ITensorAttribute& attr_tensor, std::unique_ptr<vespalib::eval::Value> query_tensor, uint32_t target_num_hits, bool approximate, uint32_t explore_additional_hits, double distance_threshold, @@ -43,7 +43,7 @@ public: NearestNeighborBlueprint(const NearestNeighborBlueprint&) = delete; NearestNeighborBlueprint& operator=(const NearestNeighborBlueprint&) = delete; ~NearestNeighborBlueprint(); - const tensor::DenseTensorAttribute& get_attribute_tensor() const { return _attr_tensor; } + const tensor::ITensorAttribute& get_attribute_tensor() const { return _attr_tensor; } const vespalib::eval::Value& get_query_tensor() const { return *_query_tensor; } uint32_t get_target_num_hits() const { return _target_num_hits; } void set_global_filter(const GlobalFilter &global_filter) override; diff --git a/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_iterator.cpp b/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_iterator.cpp index 52814bb2631..d85da49c2f7 100644 --- a/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_iterator.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_iterator.cpp @@ -3,7 +3,7 @@ #include "nearest_neighbor_iterator.h" #include <vespa/searchlib/common/bitvector.h> -using search::tensor::DenseTensorAttribute; +using search::tensor::ITensorAttribute; using vespalib::ConstArrayRef; using vespalib::eval::TypedCells; using vespalib::eval::CellType; @@ -109,7 +109,7 @@ NearestNeighborIterator::create( bool strict, fef::TermFieldMatchData &tfmd, const vespalib::eval::Value &queryTensor, - const search::tensor::DenseTensorAttribute &tensorAttribute, + const search::tensor::ITensorAttribute &tensorAttribute, NearestNeighborDistanceHeap &distanceHeap, const search::BitVector *filter, const search::tensor::DistanceFunction *dist_fun) diff --git a/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_iterator.h b/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_iterator.h index eec45ea1af0..64ac91e6db5 100644 --- a/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_iterator.h +++ b/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_iterator.h @@ -6,7 +6,7 @@ #include "nearest_neighbor_distance_heap.h" #include <vespa/eval/eval/value.h> #include <vespa/searchlib/fef/termfieldmatchdata.h> -#include <vespa/searchlib/tensor/dense_tensor_attribute.h> +#include <vespa/searchlib/tensor/i_tensor_attribute.h> #include <vespa/searchlib/tensor/distance_function.h> #include <vespa/vespalib/util/priority_queue.h> #include <cmath> @@ -16,20 +16,20 @@ namespace search::queryeval { class NearestNeighborIterator : public SearchIterator { public: - using DenseTensorAttribute = search::tensor::DenseTensorAttribute; + using ITensorAttribute = search::tensor::ITensorAttribute; using Value = vespalib::eval::Value; struct Params { fef::TermFieldMatchData &tfmd; const Value &queryTensor; - const DenseTensorAttribute &tensorAttribute; + const ITensorAttribute &tensorAttribute; NearestNeighborDistanceHeap &distanceHeap; const search::BitVector *filter; const search::tensor::DistanceFunction *distanceFunction; Params(fef::TermFieldMatchData &tfmd_in, const Value &queryTensor_in, - const DenseTensorAttribute &tensorAttribute_in, + const ITensorAttribute &tensorAttribute_in, NearestNeighborDistanceHeap &distanceHeap_in, const search::BitVector *filter_in, const search::tensor::DistanceFunction *distanceFunction_in) @@ -50,7 +50,7 @@ public: bool strict, fef::TermFieldMatchData &tfmd, const Value &queryTensor, - const search::tensor::DenseTensorAttribute &tensorAttribute, + const search::tensor::ITensorAttribute &tensorAttribute, NearestNeighborDistanceHeap &distanceHeap, const search::BitVector *filter, const search::tensor::DistanceFunction *dist_fun); diff --git a/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute.h b/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute.h index 55e7b8cb464..097c3c93dad 100644 --- a/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute.h +++ b/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute.h @@ -48,7 +48,7 @@ public: // Implements DocVectorAccess vespalib::eval::TypedCells get_vector(uint32_t docid) const override; - const NearestNeighborIndex* nearest_neighbor_index() const { return _index.get(); } + const NearestNeighborIndex* nearest_neighbor_index() const override { return _index.get(); } }; } diff --git a/searchlib/src/vespa/searchlib/tensor/i_tensor_attribute.h b/searchlib/src/vespa/searchlib/tensor/i_tensor_attribute.h index 360250c869e..c2896b6a12e 100644 --- a/searchlib/src/vespa/searchlib/tensor/i_tensor_attribute.h +++ b/searchlib/src/vespa/searchlib/tensor/i_tensor_attribute.h @@ -4,12 +4,15 @@ #include <memory> #include <vespa/eval/eval/typed_cells.h> +#include <vespa/searchcommon/attribute/distance_metric.h> namespace vespalib::eval { class ValueType; struct Value; } namespace vespalib::slime { struct Inserter; } namespace search::tensor { +class NearestNeighborIndex; + /** * Interface for tensor attribute used by feature executors to get information. */ @@ -26,6 +29,11 @@ public: virtual const vespalib::eval::ValueType & getTensorType() const = 0; + virtual const NearestNeighborIndex* nearest_neighbor_index() const { return nullptr; } + using DistanceMetric = search::attribute::DistanceMetric; + virtual DistanceMetric distance_metric() const = 0; + virtual uint32_t get_num_docs() const = 0; + /** * Gets custom state for this tensor attribute by inserting it into the given Slime inserter. * This function is only called by the writer thread or when the writer thread is blocked. diff --git a/searchlib/src/vespa/searchlib/tensor/imported_tensor_attribute_vector_read_guard.h b/searchlib/src/vespa/searchlib/tensor/imported_tensor_attribute_vector_read_guard.h index c55a922487f..a88f0a5de76 100644 --- a/searchlib/src/vespa/searchlib/tensor/imported_tensor_attribute_vector_read_guard.h +++ b/searchlib/src/vespa/searchlib/tensor/imported_tensor_attribute_vector_read_guard.h @@ -36,6 +36,9 @@ public: const vespalib::eval::Value& get_tensor_ref(uint32_t docid) const override; bool supports_extract_cells_ref() const override { return _target_tensor_attribute.supports_extract_cells_ref(); } bool supports_get_tensor_ref() const override { return _target_tensor_attribute.supports_get_tensor_ref(); } + DistanceMetric distance_metric() const override { return _target_tensor_attribute.distance_metric(); } + uint32_t get_num_docs() const override { return getNumDocs(); } + const vespalib::eval::ValueType &getTensorType() const override; void get_state(const vespalib::slime::Inserter& inserter) const override; }; diff --git a/searchlib/src/vespa/searchlib/tensor/tensor_attribute.h b/searchlib/src/vespa/searchlib/tensor/tensor_attribute.h index 9d92e226139..601e19e54d1 100644 --- a/searchlib/src/vespa/searchlib/tensor/tensor_attribute.h +++ b/searchlib/src/vespa/searchlib/tensor/tensor_attribute.h @@ -62,6 +62,11 @@ public: virtual void update_tensor(DocId docId, const document::TensorUpdate &update, bool create_empty_if_non_existing); + DistanceMetric distance_metric() const override { + return getConfig().distance_metric(); + } + uint32_t get_num_docs() const override { return getNumDocs(); } + /** * Performs the prepare step in a two-phase operation to set a tensor for a document. * |