diff options
26 files changed, 457 insertions, 125 deletions
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/changemanagement/ChangeManagementApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/changemanagement/ChangeManagementApiHandler.java index 6d19316efd7..a72e0fdfe52 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/changemanagement/ChangeManagementApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/changemanagement/ChangeManagementApiHandler.java @@ -27,6 +27,8 @@ import com.yahoo.vespa.hosted.controller.restapi.ErrorResponses; import com.yahoo.yolean.Exceptions; import java.io.IOException; +import java.time.ZoneOffset; +import java.time.format.DateTimeFormatter; import java.util.ArrayList; import java.util.List; import java.util.Optional; @@ -36,6 +38,7 @@ public class ChangeManagementApiHandler extends AuditLoggingRequestHandler { private final ChangeManagementAssessor assessor; private final Controller controller; + private static final DateTimeFormatter formatter = DateTimeFormatter.ISO_DATE_TIME.withZone(ZoneOffset.UTC); public ChangeManagementApiHandler(ThreadedHttpRequestHandler.Context ctx, Controller controller) { super(ctx, controller.auditLogger()); @@ -164,7 +167,7 @@ public class ChangeManagementApiHandler extends AuditLoggingRequestHandler { Cursor assessmentCursor = root.setObject("assessment"); // Updated gives clue to if the assessment is old - assessmentCursor.setString("updated", "2021-03-12:12:12:12Z"); + assessmentCursor.setString("updated", formatter.format(controller.clock().instant())); // Assessment on the cluster level Cursor clustersCursor = assessmentCursor.setArray("clusters"); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/changemanagement/responses/initial.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/changemanagement/responses/initial.json index 1fb8ad8be17..d9ffa9feaf9 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/changemanagement/responses/initial.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/changemanagement/responses/initial.json @@ -1,6 +1,6 @@ { "assessment": { - "updated": "2021-03-12:12:12:12Z", + "updated": "2020-09-13T12:26:40Z", "clusters": [ { "app": "mytenant:myapp:default", diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index 6ba0f394d38..b359108ed1f 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -379,7 +379,7 @@ public class Flags { ); public static final UnboundBooleanFlag ENABLE_CROWDSTRIKE = defineFeatureFlag( - "enable-crowdstrike", true, List.of("andreer"), "2023-04-13", "2023-07-25", + "enable-crowdstrike", true, List.of("andreer"), "2023-04-13", "2023-07-31", "Whether to enable CrowdStrike.", "Takes effect on next host admin tick", HOSTNAME); @@ -396,7 +396,7 @@ public class Flags { HOSTNAME, APPLICATION_ID, VESPA_VERSION); public static final UnboundBooleanFlag RANDOMIZED_ENDPOINT_NAMES = defineFeatureFlag( - "randomized-endpoint-names", false, List.of("andreer"), "2023-04-26", "2023-06-30", + "randomized-endpoint-names", false, List.of("andreer"), "2023-04-26", "2023-07-30", "Whether to use randomized endpoint names", "Takes effect on application deployment", APPLICATION_ID); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirer.java index cb0a8005e87..4f2201adba0 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirer.java @@ -76,7 +76,7 @@ public class FailedExpirer extends NodeRepositoryMaintainer { isExpired, (node, lock) -> recycle(node, List.of(), allNodes).get()); - nodeRepository.nodes().performOnRecursively(allNodes.nodeType(NodeType.host), + nodeRepository.nodes().performOnRecursively(allNodes.nodeType(NodeType.host).matching(isExpired), nodes -> isExpired.test(nodes.parent().node()), nodes -> recycle(nodes.parent().node(), nodes.children().stream().map(NodeMutex::node).toList(), diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/HostResumeProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/HostResumeProvisioner.java index fe89ba17469..039e40a3204 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/HostResumeProvisioner.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/HostResumeProvisioner.java @@ -7,6 +7,7 @@ import com.yahoo.jdisc.Metric; import com.yahoo.transaction.Mutex; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeList; +import com.yahoo.vespa.hosted.provision.NodeMutex; import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.node.Agent; import com.yahoo.vespa.hosted.provision.node.IP; @@ -17,6 +18,8 @@ import com.yahoo.yolean.Exceptions; import javax.naming.NamingException; import java.time.Duration; +import java.util.ArrayList; +import java.util.List; import java.util.logging.Level; import java.util.logging.Logger; @@ -77,7 +80,14 @@ public class HostResumeProvisioner extends NodeRepositoryMaintainer { if (hostIpConfig.isEmpty()) return; hostIpConfig.asMap().forEach((hostname, ipConfig) -> verifyDns(hostname, host.type(), host.cloudAccount(), ipConfig)); - nodeRepository().nodes().setIpConfig(hostIpConfig); + + nodeRepository().nodes().performOnRecursively(NodeList.of(host), __ -> true, nodes -> { + List<Node> updated = new ArrayList<>(); + for (NodeMutex node : nodes.nodes().nodes()) + updated.add(nodeRepository().nodes().write(node.node().with(hostIpConfig.require(node.node().hostname())), node)); + + return updated; + }); } /** Verify DNS configuration of given node */ diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Nodes.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Nodes.java index 490e7b9ac33..7ac027afbf8 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Nodes.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Nodes.java @@ -164,8 +164,7 @@ public class Nodes { * with the history of that node. */ public List<Node> addNodes(List<Node> nodes, Agent agent) { - try (NodeMutexes existingNodesLocks = lockAndGetAll(nodes, Optional.empty()); // Locks for any existing nodes we may remove. - Mutex allocationLock = lockUnallocated()) { + try (Mutex allocationLock = lockUnallocated()) { List<Node> nodesToAdd = new ArrayList<>(); List<Node> nodesToRemove = new ArrayList<>(); for (int i = 0; i < nodes.size(); i++) { @@ -378,17 +377,6 @@ public class Nodes { } } - /** Update IP config for nodes in given config */ - public void setIpConfig(HostIpConfig hostIpConfig) { - // Ideally this should hold the unallocated lock over the entire method, but unallocated lock must be taken - // after the application lock, making this impossible - Predicate<Node> nodeInConfig = (node) -> hostIpConfig.contains(node.hostname()); - performOn(nodeInConfig, (node, lock) -> { - IP.Config ipConfig = hostIpConfig.require(node.hostname()); - return write(node.with(ipConfig), lock); - }); - } - /** * Parks this node and returns it in its new state. * @@ -746,7 +734,7 @@ public class Nodes { public List<Node> performOn(NodeList nodes, Predicate<Node> filter, BiFunction<Node, Mutex, Node> action) { List<Node> resultingNodes = new ArrayList<>(); - nodes.stream().collect(groupingBy(Nodes::applicationIdForLock)) + nodes.matching(filter).stream().collect(groupingBy(Nodes::applicationIdForLock)) .forEach((applicationId, nodeList) -> { // Grouped only to reduce number of lock acquire/release cycles. try (NodeMutexes locked = lockAndGetAll(nodeList, Optional.empty())) { for (NodeMutex node : locked.nodes()) @@ -984,7 +972,8 @@ public class Nodes { for (NodeMutex node : outOfOrder) unlocked.add(node.node()); outOfOrder.clear(); - Mutex lock = lock(next, budget.timeLeftOrThrow()); + boolean nextLockSameAsPrevious = ! locked.isEmpty() && applicationIdForLock(locked.last().node()).equals(applicationIdForLock(next)); + Mutex lock = nextLockSameAsPrevious ? () -> { } : lock(next, budget.timeLeftOrThrow()); try { Optional<Node> fresh = node(next.hostname()); if (fresh.isEmpty()) { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/InfraDeployerImpl.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/InfraDeployerImpl.java index 35b2fef2c78..1f424a1e1d5 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/InfraDeployerImpl.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/InfraDeployerImpl.java @@ -89,23 +89,20 @@ public class InfraDeployerImpl implements InfraDeployer { public void prepare() { if (prepared) return; - try (Mutex lock = nodeRepository.applications().lock(application.getApplicationId())) { - NodeType nodeType = application.getCapacity().type(); - Version targetVersion = infrastructureVersions.getTargetVersionFor(nodeType); - hostSpecs = provisioner.prepare(application.getApplicationId(), - application.getClusterSpecWithVersion(targetVersion), - application.getCapacity(), - logger::log); - - prepared = true; - } + NodeType nodeType = application.getCapacity().type(); + Version targetVersion = infrastructureVersions.getTargetVersionFor(nodeType); + hostSpecs = provisioner.prepare(application.getApplicationId(), + application.getClusterSpecWithVersion(targetVersion), + application.getCapacity(), + logger::log); + + prepared = true; } @Override public long activate() { + prepare(); try (var lock = provisioner.lock(application.getApplicationId())) { - prepare(); - if (hostSpecs.isEmpty()) { logger.log(Level.FINE, () -> "No nodes to provision for " + application.getCapacity().type() + ", removing application"); removeApplication(application.getApplicationId()); diff --git a/searchcore/src/vespa/searchcore/proton/matching/matcher.cpp b/searchcore/src/vespa/searchcore/proton/matching/matcher.cpp index 28da6013edb..800d0d0aed8 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/matcher.cpp +++ b/searchcore/src/vespa/searchcore/proton/matching/matcher.cpp @@ -86,9 +86,9 @@ private: }; bool -willNotNeedRanking(const SearchRequest & request, const GroupingContext & groupingContext) { - return (!groupingContext.needRanking() && (request.maxhits == 0)) - || (!request.sortSpec.empty() && (request.sortSpec.find("[rank]") == vespalib::string::npos)); +willNeedRanking(const SearchRequest & request, const GroupingContext & groupingContext) { + return (groupingContext.needRanking() || (request.maxhits != 0)) + && (request.sortSpec.empty() || (request.sortSpec.find("[rank]") != vespalib::string::npos)); } SearchReply::UP @@ -244,7 +244,7 @@ Matcher::match(const SearchRequest &request, vespalib::ThreadBundle &threadBundl MatchParams params(searchContext.getDocIdLimit(), heapSize, arraySize, rank_score_drop_limit, request.offset, request.maxhits, !_rankSetup->getSecondPhaseRank().empty(), - !willNotNeedRanking(request, groupingContext)); + willNeedRanking(request, groupingContext)); ResultProcessor rp(attrContext, metaStore, sessionMgr, groupingContext, sessionId, request.sortSpec, params.offset, params.hits); diff --git a/searchlib/abi-spec.json b/searchlib/abi-spec.json index 30f2cb5c6ea..7d6f2f8790c 100644 --- a/searchlib/abi-spec.json +++ b/searchlib/abi-spec.json @@ -947,6 +947,7 @@ "public final com.yahoo.searchlib.rankingexpression.rule.TensorFunctionNode tensorL1Normalize()", "public final com.yahoo.searchlib.rankingexpression.rule.TensorFunctionNode tensorL2Normalize()", "public final com.yahoo.searchlib.rankingexpression.rule.TensorFunctionNode tensorEuclideanDistance()", + "public final com.yahoo.searchlib.rankingexpression.rule.TensorFunctionNode tensorCosineSimilarity()", "public final com.yahoo.searchlib.rankingexpression.rule.TensorFunctionNode tensorMatmul()", "public final com.yahoo.searchlib.rankingexpression.rule.TensorFunctionNode tensorSoftmax()", "public final com.yahoo.searchlib.rankingexpression.rule.TensorFunctionNode tensorXwPlusB()", @@ -1100,6 +1101,7 @@ "public static final int L1_NORMALIZE", "public static final int L2_NORMALIZE", "public static final int EUCLIDEAN_DISTANCE", + "public static final int COSINE_SIMILARITY", "public static final int MATMUL", "public static final int SOFTMAX", "public static final int XW_PLUS_B", diff --git a/searchlib/src/main/javacc/RankingExpressionParser.jj b/searchlib/src/main/javacc/RankingExpressionParser.jj index 744e629893e..41647a5ef5b 100755 --- a/searchlib/src/main/javacc/RankingExpressionParser.jj +++ b/searchlib/src/main/javacc/RankingExpressionParser.jj @@ -139,6 +139,7 @@ TOKEN : <L1_NORMALIZE: "l1_normalize"> | <L2_NORMALIZE: "l2_normalize"> | <EUCLIDEAN_DISTANCE: "euclidean_distance"> | + <COSINE_SIMILARITY: "cosine_similarity"> | <MATMUL: "matmul"> | <SOFTMAX: "softmax"> | <XW_PLUS_B: "xw_plus_b"> | @@ -381,6 +382,7 @@ TensorFunctionNode tensorFunction() : tensorExpression = tensorL1Normalize() | tensorExpression = tensorL2Normalize() | tensorExpression = tensorEuclideanDistance() | + tensorExpression = tensorCosineSimilarity() | tensorExpression = tensorMatmul() | tensorExpression = tensorSoftmax() | tensorExpression = tensorXwPlusB() | @@ -558,6 +560,18 @@ TensorFunctionNode tensorEuclideanDistance() : dimension)); } } +TensorFunctionNode tensorCosineSimilarity() : +{ + ExpressionNode tensor1, tensor2; + String dimension; +} +{ + <COSINE_SIMILARITY> <LBRACE> tensor1 = expression() <COMMA> tensor2 = expression() <COMMA> dimension = identifier() <RBRACE> + { return new TensorFunctionNode(new CosineSimilarity(TensorFunctionNode.wrap(tensor1), + TensorFunctionNode.wrap(tensor2), + dimension)); } +} + TensorFunctionNode tensorMatmul() : { ExpressionNode tensor1, tensor2; @@ -716,6 +730,7 @@ String tensorFunctionName() : ( <L1_NORMALIZE> { return token.image; } ) | ( <L2_NORMALIZE> { return token.image; } ) | ( <EUCLIDEAN_DISTANCE> { return token.image; } ) | + ( <COSINE_SIMILARITY> { return token.image; } ) | ( <MATMUL> { return token.image; } ) | ( <SOFTMAX> { return token.image; } ) | ( <XW_PLUS_B> { return token.image; } ) | diff --git a/searchlib/src/tests/tensor/tensor_buffer_type_mapper/tensor_buffer_type_mapper_test.cpp b/searchlib/src/tests/tensor/tensor_buffer_type_mapper/tensor_buffer_type_mapper_test.cpp index fc574ba9b2c..6f25b9e07c5 100644 --- a/searchlib/src/tests/tensor/tensor_buffer_type_mapper/tensor_buffer_type_mapper_test.cpp +++ b/searchlib/src/tests/tensor/tensor_buffer_type_mapper/tensor_buffer_type_mapper_test.cpp @@ -3,10 +3,13 @@ #include <vespa/searchlib/tensor/tensor_buffer_type_mapper.h> #include <vespa/searchlib/tensor/tensor_buffer_operations.h> #include <vespa/eval/eval/value_type.h> +#include <vespa/vespalib/datastore/array_store_config.h> #include <vespa/vespalib/gtest/gtest.h> +#include <limits> using search::tensor::TensorBufferOperations; using search::tensor::TensorBufferTypeMapper; +using vespalib::datastore::ArrayStoreConfig; using vespalib::eval::ValueType; const vespalib::string tensor_type_sparse_spec("tensor(x{})"); @@ -15,18 +18,26 @@ const vespalib::string tensor_type_2d_mixed_spec("tensor(x{},y[2])"); const vespalib::string float_tensor_type_spec("tensor<float>(y{})"); const vespalib::string tensor_type_dense_spec("tensor(x[2])"); -constexpr double grow_factor = 1.03; +namespace { + +constexpr double default_grow_factor = 1.03; +constexpr size_t default_max_buffer_size = ArrayStoreConfig::default_max_buffer_size; +constexpr size_t max_max_buffer_size = std::numeric_limits<uint32_t>::max(); + +} struct TestParam { vespalib::string _name; std::vector<size_t> _array_sizes; std::vector<size_t> _large_array_sizes; + std::vector<uint32_t> _type_id_caps; vespalib::string _tensor_type_spec; - TestParam(vespalib::string name, std::vector<size_t> array_sizes, std::vector<size_t> large_array_sizes, const vespalib::string& tensor_type_spec) + TestParam(vespalib::string name, std::vector<size_t> array_sizes, std::vector<size_t> large_array_sizes, std::vector<uint32_t> type_id_caps, const vespalib::string& tensor_type_spec) : _name(std::move(name)), _array_sizes(std::move(array_sizes)), _large_array_sizes(std::move(large_array_sizes)), + _type_id_caps(type_id_caps), _tensor_type_spec(tensor_type_spec) { } @@ -61,7 +72,7 @@ TensorBufferTypeMapperTest::TensorBufferTypeMapperTest() : testing::TestWithParam<TestParam>(), _tensor_type(ValueType::from_spec(GetParam()._tensor_type_spec)), _ops(_tensor_type), - _mapper(GetParam()._array_sizes.size(), grow_factor, &_ops) + _mapper(GetParam()._array_sizes.size(), default_grow_factor, default_max_buffer_size, &_ops) { } @@ -73,7 +84,7 @@ TensorBufferTypeMapperTest::get_array_sizes() uint32_t max_small_subspaces_type_id = GetParam()._array_sizes.size(); std::vector<size_t> array_sizes; for (uint32_t type_id = 1; type_id <= max_small_subspaces_type_id; ++type_id) { - auto num_subspaces = type_id - 1; + auto num_subspaces = _tensor_type.is_dense() ? 1 : (type_id - 1); array_sizes.emplace_back(_mapper.get_array_size(type_id)); EXPECT_EQ(_ops.get_buffer_size(num_subspaces), array_sizes.back()); } @@ -85,10 +96,13 @@ TensorBufferTypeMapperTest::get_large_array_sizes() { auto& large_array_sizes = GetParam()._large_array_sizes; uint32_t max_large = large_array_sizes.size(); - TensorBufferTypeMapper mapper(max_large * 100, grow_factor, &_ops); + TensorBufferTypeMapper mapper(max_large * 100, default_grow_factor, default_max_buffer_size, &_ops); std::vector<size_t> result; for (uint32_t i = 0; i < max_large; ++i) { uint32_t type_id = (i + 1) * 100; + if (type_id > mapper.get_max_type_id(max_large * 100)) { + break; + } auto array_size = mapper.get_array_size(type_id); result.emplace_back(array_size); EXPECT_EQ(type_id, mapper.get_type_id(array_size)); @@ -128,11 +142,11 @@ TensorBufferTypeMapperTest::select_type_ids() INSTANTIATE_TEST_SUITE_P(TensorBufferTypeMapperMultiTest, TensorBufferTypeMapperTest, - testing::Values(TestParam("1d", {8, 16, 32, 40, 64}, {2768, 49712, 950768, 18268976, 351101184}, tensor_type_sparse_spec), - TestParam("1dfloat", {4, 12, 20, 28, 36}, {2688, 48896, 937248, 18009808, 346121248}, float_tensor_type_spec), - TestParam("2d", {8, 24, 40, 56, 80}, {2416, 41392, 790112, 15179616, 291726288}, tensor_type_2d_spec), - TestParam("2dmixed", {8, 24, 48, 64, 96}, {3008, 51728, 987632, 18974512, 364657856}, tensor_type_2d_mixed_spec), - TestParam("dense", {8, 24}, {}, tensor_type_dense_spec)), + testing::Values(TestParam("1d", {8, 16, 32, 40, 64}, {2768, 49712, 950768, 18268976, 351101184}, {27, 30, 514, 584}, tensor_type_sparse_spec), + TestParam("1dfloat", {4, 12, 20, 28, 36}, {2688, 48896, 937248, 18009808, 346121248}, {27, 30, 514, 585}, float_tensor_type_spec), + TestParam("2d", {8, 24, 40, 56, 80}, {2416, 41392, 790112, 15179616, 291726288}, {26, 29, 520, 590}, tensor_type_2d_spec), + TestParam("2dmixed", {8, 24, 48, 64, 96}, {3008, 51728, 987632, 18974512, 364657856}, {26, 29, 513, 583}, tensor_type_2d_mixed_spec), + TestParam("dense", {24}, {}, {1, 1, 1, 1}, tensor_type_dense_spec)), testing::PrintToStringParamName()); TEST_P(TensorBufferTypeMapperTest, array_sizes_are_calculated) @@ -150,10 +164,19 @@ TEST_P(TensorBufferTypeMapperTest, large_arrays_grows_exponentially) EXPECT_EQ(GetParam()._large_array_sizes, get_large_array_sizes()); } -TEST_P(TensorBufferTypeMapperTest, avoid_array_size_overflow) +TEST_P(TensorBufferTypeMapperTest, type_id_is_capped) { - TensorBufferTypeMapper mapper(300, 2.0, &_ops); - EXPECT_GE(30, mapper.get_max_type_id(1000)); + auto& exp_type_id_caps = GetParam()._type_id_caps; + std::vector<uint32_t> act_type_id_caps; + std::vector<double> grow_factors = { 2.0, default_grow_factor }; + std::vector<size_t> max_buffer_sizes = { default_max_buffer_size, max_max_buffer_size }; + for (auto& grow_factor : grow_factors) { + for (auto max_buffer_size : max_buffer_sizes) { + TensorBufferTypeMapper mapper(1000, grow_factor, max_buffer_size, &_ops); + act_type_id_caps.emplace_back(mapper.get_max_type_id(1000)); + } + } + EXPECT_EQ(exp_type_id_caps, act_type_id_caps); } GTEST_MAIN_RUN_ALL_TESTS() diff --git a/searchlib/src/vespa/searchlib/attribute/raw_buffer_store.cpp b/searchlib/src/vespa/searchlib/attribute/raw_buffer_store.cpp index 0c6dd9c75a8..520309f83b7 100644 --- a/searchlib/src/vespa/searchlib/attribute/raw_buffer_store.cpp +++ b/searchlib/src/vespa/searchlib/attribute/raw_buffer_store.cpp @@ -16,14 +16,14 @@ constexpr float ALLOC_GROW_FACTOR = 0.2; namespace search::attribute { -RawBufferStore::RawBufferStore(std::shared_ptr<vespalib::alloc::MemoryAllocator> allocator, uint32_t max_small_buffer_type_id, double grow_factor) - : _array_store(ArrayStoreType::optimizedConfigForHugePage(max_small_buffer_type_id, - TypeMapper(max_small_buffer_type_id, grow_factor, ArrayStoreConfig::default_max_buffer_size), +RawBufferStore::RawBufferStore(std::shared_ptr<vespalib::alloc::MemoryAllocator> allocator, uint32_t max_type_id, double grow_factor) + : _array_store(ArrayStoreType::optimizedConfigForHugePage(max_type_id, + TypeMapper(max_type_id, grow_factor, max_buffer_size), MemoryAllocator::HUGEPAGE_SIZE, MemoryAllocator::PAGE_SIZE, - ArrayStoreConfig::default_max_buffer_size, + max_buffer_size, 8_Ki, ALLOC_GROW_FACTOR), - std::move(allocator), TypeMapper(max_small_buffer_type_id, grow_factor, ArrayStoreConfig::default_max_buffer_size)) + std::move(allocator), TypeMapper(max_type_id, grow_factor, max_buffer_size)) { } diff --git a/searchlib/src/vespa/searchlib/attribute/raw_buffer_store.h b/searchlib/src/vespa/searchlib/attribute/raw_buffer_store.h index a3f5b564846..8bb500e58da 100644 --- a/searchlib/src/vespa/searchlib/attribute/raw_buffer_store.h +++ b/searchlib/src/vespa/searchlib/attribute/raw_buffer_store.h @@ -22,7 +22,11 @@ class RawBufferStore ArrayStoreType _array_store; public: - RawBufferStore(std::shared_ptr<vespalib::alloc::MemoryAllocator> allocator, uint32_t max_small_buffer_type_id, double grow_factor); + static constexpr double array_store_grow_factor = 1.03; + static constexpr uint32_t array_store_max_type_id = 400; + static constexpr size_t max_buffer_size = vespalib::datastore::ArrayStoreConfig::default_max_buffer_size; + + RawBufferStore(std::shared_ptr<vespalib::alloc::MemoryAllocator> allocator, uint32_t max_type_id, double grow_factor); ~RawBufferStore(); EntryRef set(vespalib::ConstArrayRef<char> raw) { return _array_store.add(raw); }; vespalib::ConstArrayRef<char> get(EntryRef ref) const { return _array_store.get(ref); } diff --git a/searchlib/src/vespa/searchlib/attribute/single_raw_attribute.cpp b/searchlib/src/vespa/searchlib/attribute/single_raw_attribute.cpp index 9a514726985..ae77a6fa9c9 100644 --- a/searchlib/src/vespa/searchlib/attribute/single_raw_attribute.cpp +++ b/searchlib/src/vespa/searchlib/attribute/single_raw_attribute.cpp @@ -11,20 +11,12 @@ using vespalib::alloc::MemoryAllocator; using vespalib::datastore::EntryRef; -namespace { - -constexpr double mapper_grow_factor = 1.03; - -constexpr uint32_t max_small_buffer_type_id = 500u; - -} - namespace search::attribute { SingleRawAttribute::SingleRawAttribute(const vespalib::string& name, const Config& config) : RawAttribute(name, config), _ref_vector(config.getGrowStrategy(), getGenerationHolder()), - _raw_store(get_memory_allocator(), max_small_buffer_type_id, mapper_grow_factor) + _raw_store(get_memory_allocator(), RawBufferStore::array_store_max_type_id, RawBufferStore::array_store_grow_factor) { } diff --git a/searchlib/src/vespa/searchlib/tensor/tensor_buffer_operations.h b/searchlib/src/vespa/searchlib/tensor/tensor_buffer_operations.h index 72940cbd6a0..8f0ddfe5800 100644 --- a/searchlib/src/vespa/searchlib/tensor/tensor_buffer_operations.h +++ b/searchlib/src/vespa/searchlib/tensor/tensor_buffer_operations.h @@ -118,6 +118,7 @@ public: vespalib::ConstArrayRef<vespalib::string_id> labels(reinterpret_cast<const vespalib::string_id*>(buf.data() + get_labels_offset()), num_subspaces * _num_mapped_dimensions); return SerializedTensorRef(VectorBundle(buf.data() + get_cells_offset(num_subspaces, aligner), num_subspaces, _subspace_type), _num_mapped_dimensions, labels); } + bool is_dense() const noexcept { return _num_mapped_dimensions == 0; } }; } diff --git a/searchlib/src/vespa/searchlib/tensor/tensor_buffer_store.cpp b/searchlib/src/vespa/searchlib/tensor/tensor_buffer_store.cpp index 8a7d84010cb..e4f54383821 100644 --- a/searchlib/src/vespa/searchlib/tensor/tensor_buffer_store.cpp +++ b/searchlib/src/vespa/searchlib/tensor/tensor_buffer_store.cpp @@ -13,6 +13,7 @@ using document::DeserializeException; using vespalib::alloc::MemoryAllocator; +using vespalib::datastore::ArrayStoreConfig; using vespalib::datastore::CompactionContext; using vespalib::datastore::CompactionStrategy; using vespalib::datastore::EntryRef; @@ -33,12 +34,12 @@ TensorBufferStore::TensorBufferStore(const ValueType& tensor_type, std::shared_p _tensor_type(tensor_type), _ops(_tensor_type), _array_store(ArrayStoreType::optimizedConfigForHugePage(max_small_subspaces_type_id, - TensorBufferTypeMapper(max_small_subspaces_type_id, array_store_grow_factor, &_ops), + TensorBufferTypeMapper(max_small_subspaces_type_id, array_store_grow_factor, ArrayStoreConfig::default_max_buffer_size, &_ops), MemoryAllocator::HUGEPAGE_SIZE, MemoryAllocator::PAGE_SIZE, - vespalib::datastore::ArrayStoreConfig::default_max_buffer_size, + ArrayStoreConfig::default_max_buffer_size, 8_Ki, ALLOC_GROW_FACTOR), - std::move(allocator), TensorBufferTypeMapper(max_small_subspaces_type_id, array_store_grow_factor, &_ops)) + std::move(allocator), TensorBufferTypeMapper(max_small_subspaces_type_id, array_store_grow_factor, ArrayStoreConfig::default_max_buffer_size, &_ops)) { } diff --git a/searchlib/src/vespa/searchlib/tensor/tensor_buffer_type_mapper.cpp b/searchlib/src/vespa/searchlib/tensor/tensor_buffer_type_mapper.cpp index 3bd9f72c73b..16c2d65d829 100644 --- a/searchlib/src/vespa/searchlib/tensor/tensor_buffer_type_mapper.cpp +++ b/searchlib/src/vespa/searchlib/tensor/tensor_buffer_type_mapper.cpp @@ -14,13 +14,13 @@ TensorBufferTypeMapper::TensorBufferTypeMapper() { } -TensorBufferTypeMapper::TensorBufferTypeMapper(uint32_t max_small_subspaces_type_id, double grow_factor, TensorBufferOperations* ops) +TensorBufferTypeMapper::TensorBufferTypeMapper(uint32_t max_small_subspaces_type_id, double grow_factor, size_t max_buffer_size, TensorBufferOperations* ops) : vespalib::datastore::ArrayStoreTypeMapper(), _ops(ops) { _array_sizes.reserve(max_small_subspaces_type_id + 1); _array_sizes.emplace_back(0); // type id 0 uses LargeSubspacesBufferType - uint32_t num_subspaces = 0; + uint32_t num_subspaces = _ops->is_dense() ? 1 : 0; size_t prev_array_size = 0u; size_t array_size = 0u; for (uint32_t type_id = 1; type_id <= max_small_subspaces_type_id; ++type_id) { @@ -32,10 +32,14 @@ TensorBufferTypeMapper::TensorBufferTypeMapper(uint32_t max_small_subspaces_type ++num_subspaces; array_size = _ops->get_buffer_size(num_subspaces); } - if (array_size > std::numeric_limits<uint32_t>::max()) { + if (array_size > std::numeric_limits<uint32_t>::max() || + array_size >= 2 * max_buffer_size) { break; } _array_sizes.emplace_back(array_size); + if (_ops->is_dense()) { + break; + } prev_array_size = array_size; } } diff --git a/searchlib/src/vespa/searchlib/tensor/tensor_buffer_type_mapper.h b/searchlib/src/vespa/searchlib/tensor/tensor_buffer_type_mapper.h index 3087ef67c4d..74c3d73badb 100644 --- a/searchlib/src/vespa/searchlib/tensor/tensor_buffer_type_mapper.h +++ b/searchlib/src/vespa/searchlib/tensor/tensor_buffer_type_mapper.h @@ -22,7 +22,7 @@ public: using LargeBufferType = LargeSubspacesBufferType; TensorBufferTypeMapper(); - TensorBufferTypeMapper(uint32_t max_small_subspaces_type_id, double grow_factor, TensorBufferOperations* ops); + TensorBufferTypeMapper(uint32_t max_small_subspaces_type_id, double grow_factor, size_t max_buffer_size, TensorBufferOperations* ops); ~TensorBufferTypeMapper(); TensorBufferOperations& get_tensor_buffer_operations() const noexcept { return *_ops; } diff --git a/security-utils/src/main/java/com/yahoo/security/tls/PeerAuthorizer.java b/security-utils/src/main/java/com/yahoo/security/tls/PeerAuthorizer.java index d0e1a33fcac..7c798eddab1 100644 --- a/security-utils/src/main/java/com/yahoo/security/tls/PeerAuthorizer.java +++ b/security-utils/src/main/java/com/yahoo/security/tls/PeerAuthorizer.java @@ -58,16 +58,10 @@ public class PeerAuthorizer { } private static boolean matchesRequiredCredentials(RequiredPeerCredential requiredCredential, String cn, List<String> sans) { - switch (requiredCredential.field()) { - case CN: - return cn != null && requiredCredential.pattern().matches(cn); - case SAN_DNS: - case SAN_URI: - return sans.stream() - .anyMatch(san -> requiredCredential.pattern().matches(san)); - default: - throw new RuntimeException("Unknown field: " + requiredCredential.field()); - } + return switch (requiredCredential.field()) { + case CN -> cn != null && requiredCredential.pattern().matches(cn); + case SAN_DNS, SAN_URI -> sans.stream().anyMatch(san -> requiredCredential.pattern().matches(san)); + }; } private static List<String> getSubjectAlternativeNames(X509Certificate peerCertificate) { diff --git a/vespajlib/abi-spec.json b/vespajlib/abi-spec.json index 3b9f494dc50..76d007dd633 100644 --- a/vespajlib/abi-spec.json +++ b/vespajlib/abi-spec.json @@ -1705,6 +1705,24 @@ ], "fields" : [ ] }, + "com.yahoo.tensor.functions.CosineSimilarity" : { + "superClass" : "com.yahoo.tensor.functions.TensorFunction", + "interfaces" : [ ], + "attributes" : [ + "public" + ], + "methods" : [ + "public void <init>(com.yahoo.tensor.functions.TensorFunction, com.yahoo.tensor.functions.TensorFunction, java.lang.String)", + "public java.util.List arguments()", + "public com.yahoo.tensor.functions.TensorFunction withArguments(java.util.List)", + "public com.yahoo.tensor.TensorType type(com.yahoo.tensor.evaluation.TypeContext)", + "public com.yahoo.tensor.Tensor evaluate(com.yahoo.tensor.evaluation.EvaluationContext)", + "public com.yahoo.tensor.functions.PrimitiveTensorFunction toPrimitive()", + "public java.lang.String toString(com.yahoo.tensor.functions.ToStringContext)", + "public int hashCode()" + ], + "fields" : [ ] + }, "com.yahoo.tensor.functions.Diag" : { "superClass" : "com.yahoo.tensor.functions.CompositeTensorFunction", "interfaces" : [ ], @@ -1741,7 +1759,7 @@ "fields" : [ ] }, "com.yahoo.tensor.functions.EuclideanDistance" : { - "superClass" : "com.yahoo.tensor.functions.CompositeTensorFunction", + "superClass" : "com.yahoo.tensor.functions.TensorFunction", "interfaces" : [ ], "attributes" : [ "public" @@ -1750,6 +1768,8 @@ "public void <init>(com.yahoo.tensor.functions.TensorFunction, com.yahoo.tensor.functions.TensorFunction, java.lang.String)", "public java.util.List arguments()", "public com.yahoo.tensor.functions.TensorFunction withArguments(java.util.List)", + "public com.yahoo.tensor.TensorType type(com.yahoo.tensor.evaluation.TypeContext)", + "public com.yahoo.tensor.Tensor evaluate(com.yahoo.tensor.evaluation.EvaluationContext)", "public com.yahoo.tensor.functions.PrimitiveTensorFunction toPrimitive()", "public java.lang.String toString(com.yahoo.tensor.functions.ToStringContext)", "public int hashCode()" diff --git a/vespajlib/src/main/java/com/yahoo/tensor/functions/CosineSimilarity.java b/vespajlib/src/main/java/com/yahoo/tensor/functions/CosineSimilarity.java new file mode 100644 index 00000000000..ebb8a11fd8a --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/tensor/functions/CosineSimilarity.java @@ -0,0 +1,93 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.tensor.functions; + +import com.yahoo.tensor.evaluation.EvaluationContext; +import com.yahoo.tensor.evaluation.Name; +import com.yahoo.tensor.evaluation.TypeContext; +import com.yahoo.tensor.Tensor; +import com.yahoo.tensor.TensorType; +import com.yahoo.tensor.TensorType.Dimension; + +import java.util.Collections; +import java.util.List; +import java.util.Objects; + +/** + * Convenience for cosine similarity between vectors. + * cosine_similarity(a, b, mydim) == sum(a*b, mydim) / sqrt(sum(a*a, mydim) * sum(b*b, mydim)) + * @author arnej + */ +public class CosineSimilarity<NAMETYPE extends Name> extends TensorFunction<NAMETYPE> { + + private final TensorFunction<NAMETYPE> arg1; + private final TensorFunction<NAMETYPE> arg2; + private final String dimension; + + public CosineSimilarity(TensorFunction<NAMETYPE> argument1, + TensorFunction<NAMETYPE> argument2, + String dimension) + { + this.arg1 = argument1; + this.arg2 = argument2; + this.dimension = dimension; + } + + @Override + public List<TensorFunction<NAMETYPE>> arguments() { return List.of(arg1, arg2); } + + @Override + public TensorFunction<NAMETYPE> withArguments(List<TensorFunction<NAMETYPE>> arguments) { + if ( arguments.size() != 2) + throw new IllegalArgumentException("CosineSimilarity must have 2 arguments, got " + arguments.size()); + return new CosineSimilarity<>(arguments.get(0), arguments.get(1), dimension); + } + + @Override + public TensorType type(TypeContext<NAMETYPE> context) { + TensorType t1 = arg1.toPrimitive().type(context); + TensorType t2 = arg2.toPrimitive().type(context); + var d1 = t1.dimension(dimension); + var d2 = t2.dimension(dimension); + if (d1.isEmpty() || d2.isEmpty() + || d1.get().type() != Dimension.Type.indexedBound + || d2.get().type() != Dimension.Type.indexedBound + || d1.get().size().get() != d2.get().size().get()) + { + throw new IllegalArgumentException("cosine_similarity expects both arguments to have the '" + + dimension + "' dimension with same size, but input types were " + + t1 + " and " + t2); + } + // Finds the type this produces by first converting it to a primitive function + return toPrimitive().type(context); + } + + /** Evaluates this by first converting it to a primitive function */ + @Override + public Tensor evaluate(EvaluationContext<NAMETYPE> context) { + return toPrimitive().evaluate(context); + } + + @Override + public PrimitiveTensorFunction<NAMETYPE> toPrimitive() { + TensorFunction<NAMETYPE> a = arg1.toPrimitive(); + TensorFunction<NAMETYPE> b = arg2.toPrimitive(); + var aa = new Join<>(a, a, ScalarFunctions.multiply()); + var ab = new Join<>(a, b, ScalarFunctions.multiply()); + var bb = new Join<>(b, b, ScalarFunctions.multiply()); + var dot_aa = new Reduce<>(aa, Reduce.Aggregator.sum, dimension); + var dot_ab = new Reduce<>(ab, Reduce.Aggregator.sum, dimension); + var dot_bb = new Reduce<>(bb, Reduce.Aggregator.sum, dimension); + var aabb = new Join<>(dot_aa, dot_bb, ScalarFunctions.multiply()); + var sqrt_aabb = new Map<>(aabb, ScalarFunctions.sqrt()); + return new Join<>(dot_ab, sqrt_aabb, ScalarFunctions.divide()); + } + + @Override + public String toString(ToStringContext<NAMETYPE> context) { + return "cosine_similarity(" + arg1.toString(context) + ", " + arg2.toString(context) + ", " + dimension + ")"; + } + + @Override + public int hashCode() { return Objects.hash("cosine_similarity", arg1, arg2, dimension); } + +} diff --git a/vespajlib/src/main/java/com/yahoo/tensor/functions/EuclideanDistance.java b/vespajlib/src/main/java/com/yahoo/tensor/functions/EuclideanDistance.java index 25399416c29..f9fc8e195d3 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/functions/EuclideanDistance.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/functions/EuclideanDistance.java @@ -1,7 +1,12 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.tensor.functions; +import com.yahoo.tensor.evaluation.EvaluationContext; import com.yahoo.tensor.evaluation.Name; +import com.yahoo.tensor.evaluation.TypeContext; +import com.yahoo.tensor.Tensor; +import com.yahoo.tensor.TensorType; +import com.yahoo.tensor.TensorType.Dimension; import java.util.Collections; import java.util.List; @@ -12,7 +17,7 @@ import java.util.Objects; * euclidean_distance(a, b, mydim) == sqrt(sum(pow(a-b, 2), mydim)) * @author arnej */ -public class EuclideanDistance<NAMETYPE extends Name> extends CompositeTensorFunction<NAMETYPE> { +public class EuclideanDistance<NAMETYPE extends Name> extends TensorFunction<NAMETYPE> { private final TensorFunction<NAMETYPE> arg1; private final TensorFunction<NAMETYPE> arg2; @@ -38,6 +43,31 @@ public class EuclideanDistance<NAMETYPE extends Name> extends CompositeTensorFun } @Override + public TensorType type(TypeContext<NAMETYPE> context) { + TensorType t1 = arg1.toPrimitive().type(context); + TensorType t2 = arg2.toPrimitive().type(context); + var d1 = t1.dimension(dimension); + var d2 = t2.dimension(dimension); + if (d1.isEmpty() || d2.isEmpty() + || d1.get().type() != Dimension.Type.indexedBound + || d2.get().type() != Dimension.Type.indexedBound + || d1.get().size().get() != d2.get().size().get()) + { + throw new IllegalArgumentException("euclidean_distance expects both arguments to have the '" + + dimension + "' dimension with same size, but input types were " + + t1 + " and " + t2); + } + // Finds the type this produces by first converting it to a primitive function + return toPrimitive().type(context); + } + + /** Evaluates this by first converting it to a primitive function */ + @Override + public Tensor evaluate(EvaluationContext<NAMETYPE> context) { + return toPrimitive().evaluate(context); + } + + @Override public PrimitiveTensorFunction<NAMETYPE> toPrimitive() { TensorFunction<NAMETYPE> primitive1 = arg1.toPrimitive(); TensorFunction<NAMETYPE> primitive2 = arg2.toPrimitive(); diff --git a/vespajlib/src/test/java/com/yahoo/tensor/functions/CosineSimilarityTestCase.java b/vespajlib/src/test/java/com/yahoo/tensor/functions/CosineSimilarityTestCase.java new file mode 100644 index 00000000000..b303e2c1739 --- /dev/null +++ b/vespajlib/src/test/java/com/yahoo/tensor/functions/CosineSimilarityTestCase.java @@ -0,0 +1,66 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.tensor.functions; + +import com.yahoo.tensor.Tensor; +import com.yahoo.tensor.TensorType; +import com.yahoo.tensor.evaluation.VariableTensor; +import org.junit.Test; + +import java.util.List; + +import static org.junit.Assert.assertEquals; + +/** + * @author arnej + */ +public class CosineSimilarityTestCase { + + @Test + public void testVectorSimilarity() { + var a = Tensor.from("tensor(x[3]):[ 2.0, 3.0, 6.0]"); + var b = Tensor.from("tensor(x[3]):[-2.0, 0.0, 0.0]"); + var c = Tensor.from("tensor(x[3]):[ 0.0, 4.0, 3.0]"); + var op = new CosineSimilarity<>(new ConstantTensor<>(a), new ConstantTensor<>(b), "x"); + Tensor result = op.evaluate(); + assertEquals((-2.0 / 7.0), result.asDouble(), 0.000001); + op = new CosineSimilarity<>(new ConstantTensor<>(b), new ConstantTensor<>(a), "x"); + result = op.evaluate(); + assertEquals((-2.0 / 7.0), result.asDouble(), 0.000001); + op = new CosineSimilarity<>(new ConstantTensor<>(a), new ConstantTensor<>(c), "x"); + result = op.evaluate(); + assertEquals((30.0 / 35.0), result.asDouble(), 0.000001); + op = new CosineSimilarity<>(new ConstantTensor<>(b), new ConstantTensor<>(c), "x"); + result = op.evaluate(); + assertEquals(0.0, result.asDouble(), 0.000001); + } + + @Test + public void testSimilarityInMixed() { + var a = Tensor.from("tensor(c{},yy[3]):{foo:[3.0, 4.0, 0.0],bar:[0.0, -4.0, 3.0]}"); + var b = Tensor.from("tensor(c{},yy[3]):{foo:[0.0, 4.0, -3.0],bar:[4.0, 0.0, -3.0]}"); + var op = new CosineSimilarity<>(new ConstantTensor<>(a), new ConstantTensor<>(b), "yy"); + Tensor result = op.evaluate(); + var expect = Tensor.from("tensor(c{}):{foo:0.64,bar:-0.36}"); + assertEquals(expect, result); + } + + @Test + public void testExpansion() { + var tType = TensorType.fromSpec("tensor(vecdim[128])"); + var a = new VariableTensor<>("left", tType); + var b = new VariableTensor<>("right", tType); + var op = new CosineSimilarity<>(a, b, "vecdim"); + assertEquals("join(" + + ( "reduce(join(left, right, f(a,b)(a * b)), sum, vecdim), " + + "map(" + + ( "join(" + + ( "reduce(join(left, left, f(a,b)(a * b)), sum, vecdim), " + + "reduce(join(right, right, f(a,b)(a * b)), sum, vecdim), " + + "f(a,b)(a * b)), " ) + + "f(a)(sqrt(a))), " ) + + "f(a,b)(a / b)" ) + + ")", + op.toPrimitive().toString()); + } + +} diff --git a/vespalib/src/tests/rw_spin_lock/rw_spin_lock_test.cpp b/vespalib/src/tests/rw_spin_lock/rw_spin_lock_test.cpp index c6f0581bceb..afd18a13d2e 100644 --- a/vespalib/src/tests/rw_spin_lock/rw_spin_lock_test.cpp +++ b/vespalib/src/tests/rw_spin_lock/rw_spin_lock_test.cpp @@ -11,6 +11,7 @@ #include <ranges> #include <random> #include <array> +#include <algorithm> using namespace vespalib; using namespace vespalib::test; @@ -23,6 +24,71 @@ size_t state_loop = 1; //----------------------------------------------------------------------------- +/** + * Estimates the 80th percentile by throwing away the 2 best samples + * in each set of 10 samples, using the best remaining sample as a + * representative for the set. Representatives are hierarchically + * matched against representatives from other sample sets. Result + * extraction is simplified in that it does not try to estimate the + * actual 80th percentile, but rather tries to drop the best samples + * if possible. + * + * The goal is to have a more robust way of combining repeated + * micro-benchmark samples than simply using minimum time. With simple + * single-threaded CPU-bound tasks, minimum time is a good measure of + * how expensive something is, but when we start benchmarking + * operations that may conflict with themselves, we do not want to + * account for being super lucky. However, we still want to account + * for the benchmark conditions being as good as possible. + **/ +struct Est80P { + struct Level { + int cnt; + std::array<double,3> data; + Level(double value) noexcept + : cnt(1), data{value, 0.0, 0.0} {} + bool empty() const noexcept { return (cnt == 0); } + bool full() const noexcept { return (cnt == 10); } + void add(double value) noexcept { + assert(!full()); + if (cnt < 3 || data[2] > value) { + size_t i = std::min(cnt, 2); + while (i > 0 && data[i - 1] > value) { + data[i] = data[i - 1]; + --i; + } + data[i] = value; + } + ++cnt; + } + double get() const noexcept { + assert(!empty()); + return data[std::min(2, cnt - 1)]; + } + void clear() noexcept { + cnt = 0; + } + }; + std::vector<Level> levels; + void add_sample(double value) { + for (auto &level: levels) { + level.add(value); + if (!level.full()) [[likely]] { + return; + } + value = level.get(); + level.clear(); + } + levels.emplace_back(value); + } + double get_result() { + assert(!levels.empty()); + return levels.back().get(); + } +}; + +//----------------------------------------------------------------------------- + struct DummyLock { constexpr DummyLock() noexcept {} // BasicLockable @@ -158,47 +224,31 @@ double measure_ns(auto &work) { struct BenchmarkResult { double cost_ns; - double range_ns; - size_t threads; - BenchmarkResult(size_t num_threads) - : cost_ns(std::numeric_limits<double>::max()), range_ns(0.0), threads(num_threads) {} + BenchmarkResult(double cost_ns_in) : cost_ns(cost_ns_in) {} void report(vespalib::string desc) { - if (threads == 1) { - fprintf(stderr, "%s: cost_ns: %g\n", - desc.c_str(), cost_ns); - } else { - fprintf(stderr, "%s: cost_ns: %g, range_ns: %g (%zu threads)\n", - desc.c_str(), cost_ns, range_ns, threads); - } + fprintf(stderr, "%s: cost_ns: %g\n", desc.c_str(), cost_ns); } void report(vespalib::string name, vespalib::string desc) { report(name + "(" + desc + ")"); } }; -struct Meets { - vespalib::test::ThreadMeets::Avg avg; - vespalib::test::ThreadMeets::Range<double> range; - Meets(size_t num_threads) : avg(num_threads), range(num_threads) {} -}; - BenchmarkResult benchmark_ns(auto &&work, size_t num_threads = 1) { - Meets meets(num_threads); + Est80P collector; + vespalib::test::ThreadMeets::Avg avg(num_threads); auto entry = [&](Nexus &ctx) { Timer timer; BenchmarkResult result(ctx.num_threads()); for (bool once_more = true; ctx.vote(once_more); once_more = (timer.elapsed() < budget)) { - auto my_ns = measure_ns(work); - auto cost_ns = meets.avg(my_ns); - auto range_ns = meets.range(my_ns); - if (cost_ns < result.cost_ns) { - result.cost_ns = cost_ns; - result.range_ns = range_ns; + auto cost_ns = avg(measure_ns(work)); + if (ctx.is_main()) { + collector.add_sample(cost_ns); } } - return result; }; - return Nexus::run(num_threads, entry); + Nexus::run(num_threads, entry); + auto result = collector.get_result(); + return {result}; } //----------------------------------------------------------------------------- @@ -224,7 +274,7 @@ void estimate_cost() { //----------------------------------------------------------------------------- template <typename T> -void thread_safety_loop(Nexus &ctx, T &lock, MyState &state, Meets &meets, int read_bp) { +void thread_safety_loop(Nexus &ctx, T &lock, MyState &state, auto &max, int read_bp) { Rnd rnd(ctx.thread_id()); size_t write_cnt = 0; size_t bad_reads = 0; @@ -247,16 +297,11 @@ void thread_safety_loop(Nexus &ctx, T &lock, MyState &state, Meets &meets, int r } } } - auto t1 = steady_clock::now(); - ctx.barrier(); - auto t2 = steady_clock::now(); - auto my_ms = count_ns(t1 - t0) / 1'000'000.0; - auto total_ms = count_ns(t2 - t0) / 1'000'000.0; - auto cost_ms = meets.avg(my_ms); - auto range_ms = meets.range(my_ms); - if (ctx.thread_id() == 0) { - fprintf(stderr, "---> %s with %2zu threads (%5d bp r): avg: %10.2f ms, range: %10.2f ms, max: %10.2f ms\n", - getClassName(lock).c_str(), ctx.num_threads(), read_bp, cost_ms, range_ms, total_ms); + auto my_ms = count_ns(steady_clock::now() - t0) / 1'000'000.0; + auto actual_ms = max(my_ms); + if (ctx.is_main()) { + fprintf(stderr, "---> %s with %2zu threads (%5d bp r): time: %10.2f ms\n", + getClassName(lock).c_str(), ctx.num_threads(), read_bp, actual_ms); } state.commit_inconsistent_reads(bad_reads); state.commit_expected_writes(write_cnt); @@ -290,9 +335,9 @@ void benchmark_lock() { for (size_t bp: {10000, 9999, 5000, 0}) { for (size_t num_threads: {8, 4, 2, 1}) { if (bench || (bp == 9999 && num_threads == 8)) { - Meets meets(num_threads); + vespalib::test::ThreadMeets::Max<double> max(num_threads); Nexus::run(num_threads, [&](Nexus &ctx) { - thread_safety_loop(ctx, *lock, *state, meets, bp); + thread_safety_loop(ctx, *lock, *state, max, bp); }); } } diff --git a/vespalib/src/vespa/vespalib/test/nexus.h b/vespalib/src/vespa/vespalib/test/nexus.h index 659b563e43c..7be35d42463 100644 --- a/vespalib/src/vespa/vespalib/test/nexus.h +++ b/vespalib/src/vespa/vespalib/test/nexus.h @@ -36,6 +36,7 @@ public: Nexus &operator=(const Nexus &) = delete; size_t num_threads() const noexcept { return _vote.size(); } size_t thread_id() const noexcept { return _thread_id; } + bool is_main() const noexcept { return _thread_id == 0; } bool vote(bool my_vote) { return _vote(my_vote); } void barrier() { REQUIRE_EQ(_vote(true), true); } struct select_thread_0 {}; diff --git a/vespalib/src/vespa/vespalib/test/thread_meets.h b/vespalib/src/vespa/vespalib/test/thread_meets.h index 7ef4dcb9921..26ca560641d 100644 --- a/vespalib/src/vespa/vespalib/test/thread_meets.h +++ b/vespalib/src/vespa/vespalib/test/thread_meets.h @@ -38,8 +38,8 @@ struct ThreadMeets { explicit Sum(size_t N) : vespalib::Rendezvous<T,T>(N) {} T operator()(T value) { return rendezvous(value); } void mingle() override { - T acc{}; - for (size_t i = 0; i < size(); ++i) { + T acc = in(0); + for (size_t i = 1; i < size(); ++i) { acc += in(i); } for (size_t i = 0; i < size(); ++i) { @@ -47,6 +47,48 @@ struct ThreadMeets { } } }; + // maximum of values across all threads + template <typename T> + struct Max : vespalib::Rendezvous<T,T> { + using vespalib::Rendezvous<T,T>::in; + using vespalib::Rendezvous<T,T>::out; + using vespalib::Rendezvous<T,T>::size; + using vespalib::Rendezvous<T,T>::rendezvous; + explicit Max(size_t N) : vespalib::Rendezvous<T,T>(N) {} + T operator()(T value) { return rendezvous(value); } + void mingle() override { + T max = in(0); + for (size_t i = 1; i < size(); ++i) { + if (in(i) > max) { + max = in(i); + } + } + for (size_t i = 0; i < size(); ++i) { + out(i) = max; + } + } + }; + // minimum of values across all threads + template <typename T> + struct Min : vespalib::Rendezvous<T,T> { + using vespalib::Rendezvous<T,T>::in; + using vespalib::Rendezvous<T,T>::out; + using vespalib::Rendezvous<T,T>::size; + using vespalib::Rendezvous<T,T>::rendezvous; + explicit Min(size_t N) : vespalib::Rendezvous<T,T>(N) {} + T operator()(T value) { return rendezvous(value); } + void mingle() override { + T min = in(0); + for (size_t i = 1; i < size(); ++i) { + if (in(i) < min) { + min = in(i); + } + } + for (size_t i = 0; i < size(); ++i) { + out(i) = min; + } + } + }; // range of values across all threads template <typename T> struct Range : vespalib::Rendezvous<T,T> { |