diff options
26 files changed, 48 insertions, 575 deletions
diff --git a/CMakeLists.txt b/CMakeLists.txt index a4e89cbdeb9..b975f3adf2f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -52,7 +52,6 @@ find_package(JNI REQUIRED) find_package(GTest REQUIRED) find_package(Python 3.6 COMPONENTS Interpreter Development REQUIRED) -find_package(pybind11 CONFIG REQUIRED) find_package(Protobuf REQUIRED) @@ -100,7 +99,6 @@ vespa_install_data(tsan-suppressions.txt etc/vespa) include_directories(BEFORE ${CMAKE_BINARY_DIR}/configdefinitions/src) add_subdirectory(airlift-zstd) -add_subdirectory(ann_benchmark) add_subdirectory(application-model) add_subdirectory(client) add_subdirectory(cloud-tenant-cd) diff --git a/ann_benchmark/CMakeLists.txt b/ann_benchmark/CMakeLists.txt deleted file mode 100644 index 06d742cf072..00000000000 --- a/ann_benchmark/CMakeLists.txt +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_define_module( - DEPENDS - searchlib - - LIBS - src/vespa/ann_benchmark - - APPS - - TESTS - src/tests/ann_benchmark -) diff --git a/ann_benchmark/src/tests/ann_benchmark/.gitignore b/ann_benchmark/src/tests/ann_benchmark/.gitignore deleted file mode 100644 index 225fc6f6650..00000000000 --- a/ann_benchmark/src/tests/ann_benchmark/.gitignore +++ /dev/null @@ -1 +0,0 @@ -/__pycache__ diff --git a/ann_benchmark/src/tests/ann_benchmark/CMakeLists.txt b/ann_benchmark/src/tests/ann_benchmark/CMakeLists.txt deleted file mode 100644 index 03126ce1b47..00000000000 --- a/ann_benchmark/src/tests/ann_benchmark/CMakeLists.txt +++ /dev/null @@ -1,5 +0,0 @@ -# Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -if(NOT DEFINED VESPA_USE_SANITIZER) - vespa_add_test(NAME ann_benchmark_test NO_VALGRIND COMMAND ${Python_EXECUTABLE} -m pytest WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} DEPENDS vespa_ann_benchmark) -endif() diff --git a/ann_benchmark/src/tests/ann_benchmark/test_angular.py b/ann_benchmark/src/tests/ann_benchmark/test_angular.py deleted file mode 100644 index ac7feb29d76..00000000000 --- a/ann_benchmark/src/tests/ann_benchmark/test_angular.py +++ /dev/null @@ -1,41 +0,0 @@ -# Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -import pytest -import sys -import os -import math -sys.path.insert(0, os.path.abspath("../../vespa/ann_benchmark")) -from vespa_ann_benchmark import DistanceMetric, HnswIndexParams, HnswIndex - -class Fixture: - def __init__(self, normalize): - metric = DistanceMetric.InnerProduct if normalize else DistanceMetric.Angular - self.index = HnswIndex(2, HnswIndexParams(16, 200, metric, False), normalize) - self.index.set_vector(0, [1, 0]) - self.index.set_vector(1, [10, 10]) - - def find(self, k, value): - return self.index.find_top_k(k, value, k + 200) - - def run_test(self): - top = self.find(10, [1, 1]) - assert [top[0][0], top[1][0]] == [0, 1] - # Allow some rounding errors - epsilon = 6e-8 - assert abs((1 - top[0][1]) - math.sqrt(0.5)) < epsilon - assert abs((1 - top[1][1]) - 1) < epsilon - top2 = self.find(10, [0, 2]) - # Result is not sorted by distance - assert [top2[0][0], top2[1][0]] == [0, 1] - assert abs((1 - top2[0][1]) - 0) < epsilon - assert abs((1 - top2[1][1]) - math.sqrt(0.5)) < epsilon - assert 1 == self.find(1, [1, 1])[0][0] - assert 0 == self.find(1, [1, -1])[0][0] - -def test_find_angular(): - f = Fixture(False) - f.run_test() - -def test_find_angular_normalized(): - f = Fixture(True) - f.run_test() diff --git a/ann_benchmark/src/tests/ann_benchmark/test_euclidean.py b/ann_benchmark/src/tests/ann_benchmark/test_euclidean.py deleted file mode 100644 index ca4d5ecd6a1..00000000000 --- a/ann_benchmark/src/tests/ann_benchmark/test_euclidean.py +++ /dev/null @@ -1,61 +0,0 @@ -# Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -import pytest -import sys -import os -import math -sys.path.insert(0, os.path.abspath("../../vespa/ann_benchmark")) -from vespa_ann_benchmark import DistanceMetric, HnswIndexParams, HnswIndex - -class Fixture: - def __init__(self): - self.index = HnswIndex(2, HnswIndexParams(16, 200, DistanceMetric.Euclidean, False), False) - - def set(self, lid, value): - self.index.set_vector(lid, value) - - def get(self, lid): - return self.index.get_vector(lid) - - def clear(self, lid): - return self.index.clear_vector(lid) - - def find(self, k, value): - return self.index.find_top_k(k, value, k + 200) - -def test_set_value(): - f = Fixture() - f.set(0, [1, 2]) - f.set(1, [3, 4]) - assert [1, 2] == f.get(0) - assert [3, 4] == f.get(1) - -def test_clear_value(): - f = Fixture() - f.set(0, [1, 2]) - assert [1, 2] == f.get(0) - f.clear(0) - assert [0, 0] == f.get(0) - -def test_find(): - f = Fixture() - f.set(0, [0, 0]) - f.set(1, [10, 10]) - top = f.find(10, [1, 1]) - assert [top[0][0], top[1][0]] == [0, 1] - # Allow some rounding errors - epsilon = 1e-20 - assert abs(top[0][1] - math.sqrt(2)) < epsilon - assert abs(top[1][1] - math.sqrt(162)) < epsilon - top2 = f.find(10, [9, 9]) - # Result is not sorted by distance - assert [top2[0][0], top2[1][0]] == [0, 1] - assert abs(top2[0][1] - math.sqrt(162)) < epsilon - assert abs(top2[1][1] - math.sqrt(2)) < epsilon - assert 0 == f.find(1, [1, 1])[0][0] - assert 1 == f.find(1, [9, 9])[0][0] - f.clear(1) - assert 0 == f.find(1, [9, 9])[0][0] - assert 0 == f.find(1, [0, 0])[0][0] - f.clear(0) - assert 0 == len(f.find(1, [9, 9])) diff --git a/ann_benchmark/src/vespa/ann_benchmark/.gitignore b/ann_benchmark/src/vespa/ann_benchmark/.gitignore deleted file mode 100644 index 3b4605aeee2..00000000000 --- a/ann_benchmark/src/vespa/ann_benchmark/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -/vespa_ann_benchmark.cpython*.so -/setup.py diff --git a/ann_benchmark/src/vespa/ann_benchmark/CMakeLists.txt b/ann_benchmark/src/vespa/ann_benchmark/CMakeLists.txt deleted file mode 100644 index fa3b10b2269..00000000000 --- a/ann_benchmark/src/vespa/ann_benchmark/CMakeLists.txt +++ /dev/null @@ -1,31 +0,0 @@ -# Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -install(DIRECTORY DESTINATION libexec/vespa_ann_benchmark) - -vespa_add_library(vespa_ann_benchmark - ALLOW_UNRESOLVED_SYMBOLS - SOURCES - vespa_ann_benchmark.cpp - - INSTALL libexec/vespa_ann_benchmark - DEPENDS - pybind11::pybind11 -) - -if (TARGET pybind11::lto) - target_link_libraries(vespa_ann_benchmark PRIVATE pybind11::module pybind11::lto) -else() - target_link_libraries(vespa_ann_benchmark PRIVATE pybind11::module) -endif() - -if (COMMAND pybind11_extension) - pybind11_extension(vespa_ann_benchmark) -else() - set_target_properties(vespa_ann_benchmark PROPERTIES PREFIX "${PYTHON_MODULE_PREFIX}") - set_target_properties(vespa_ann_benchmark PROPERTIES SUFFIX "${PYTHON_MODULE_EXTENSION}") -endif() - -set_target_properties(vespa_ann_benchmark PROPERTIES CXX_VISIBILITY_PRESET "hidden") - -configure_file(setup.py.in setup.py @ONLY) - -vespa_install_script(setup.py libexec/vespa_ann_benchmark) diff --git a/ann_benchmark/src/vespa/ann_benchmark/setup.py.in b/ann_benchmark/src/vespa/ann_benchmark/setup.py.in deleted file mode 100644 index 457d6e1b4b5..00000000000 --- a/ann_benchmark/src/vespa/ann_benchmark/setup.py.in +++ /dev/null @@ -1,27 +0,0 @@ -# Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -import subprocess -import sys -import platform -import distutils.sysconfig -from setuptools import setup, Extension -from setuptools.command.build_ext import build_ext - -class PreBuiltExt(build_ext): - def build_extension(self, ext): - print("Using prebuilt extension library") - libdir="lib.%s-%s-%s" % (sys.platform, platform.machine(), distutils.sysconfig.get_python_version()) - subprocess.run(["mkdir", "-p", "build/%s" % libdir]) - subprocess.run(["cp", "-p", "@PYTHON_MODULE_PREFIX@vespa_ann_benchmark@PYTHON_MODULE_EXTENSION@", "build/%s" % libdir]) - -setup( - name="vespa_ann_benchmark", - version="0.1.0", - author="Tor Egge", - author_email="Tor.Egge@yahooinc.com", - description="Python binding for the Vespa implementation of an HNSW index for nearest neighbor search", - long_description="Python binding for the Vespa implementation of an HNSW index for nearest neighbor search used for low-level benchmarking", - ext_modules=[Extension("vespa_ann_benchmark", sources=[])], - cmdclass={"build_ext": PreBuiltExt}, - zip_safe=False, -) diff --git a/ann_benchmark/src/vespa/ann_benchmark/vespa_ann_benchmark.cpp b/ann_benchmark/src/vespa/ann_benchmark/vespa_ann_benchmark.cpp deleted file mode 100644 index ab00f997226..00000000000 --- a/ann_benchmark/src/vespa/ann_benchmark/vespa_ann_benchmark.cpp +++ /dev/null @@ -1,252 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include <pybind11/pybind11.h> -#include <pybind11/stl.h> -#include <vespa/searchcommon/attribute/hnsw_index_params.h> -#include <vespa/searchlib/attribute/attributevector.h> -#include <vespa/searchlib/attribute/attributefactory.h> -#include <vespa/searchlib/tensor/dense_tensor_attribute.h> -#include <vespa/searchlib/tensor/nearest_neighbor_index.h> -#include <vespa/searchcommon/attribute/config.h> -#include <vespa/eval/eval/value.h> -#include <vespa/vespalib/test/insertion_operators.h> -#include <vespa/vespalib/util/fake_doom.h> -#include <iostream> -#include <sstream> -#include <limits> - -namespace py = pybind11; - -using search::AttributeFactory; -using search::AttributeVector; -using search::attribute::BasicType; -using search::attribute::Config; -using search::attribute::CollectionType; -using search::attribute::DistanceMetric; -using search::attribute::HnswIndexParams; -using search::tensor::NearestNeighborIndex; -using search::tensor::TensorAttribute; -using vespalib::eval::CellType; -using vespalib::eval::DenseValueView; -using vespalib::eval::TypedCells; -using vespalib::eval::ValueType; -using vespalib::eval::Value; - -namespace vespa_ann_benchmark { - -using TopKResult = std::vector<std::pair<uint32_t, double>>; - -namespace { - -std::string -make_tensor_spec(uint32_t dim_size) -{ - std::ostringstream os; - os << "tensor<float>(x[" << dim_size << "])"; - return os.str(); -} - -constexpr uint32_t lid_bias = 1; // lid 0 is reserved - -} - -/* - * Class exposing the Vespa implementation of an HNSW index for nearest neighbor search over data points in a high dimensional vector space. - * - * A tensor attribute field (https://docs.vespa.ai/en/reference/schema-reference.html#type:tensor) is used to store the vectors in memory. - * This class only supports single-threaded access (both for indexing and searching), - * and should only be used for low-level benchmarking. - * To use nearest neighbor search in a Vespa application, - * see https://docs.vespa.ai/en/approximate-nn-hnsw.html for more details. - */ -class HnswIndex -{ - ValueType _tensor_type; - HnswIndexParams _hnsw_index_params; - std::shared_ptr<AttributeVector> _attribute; - TensorAttribute* _tensor_attribute; - const NearestNeighborIndex* _nearest_neighbor_index; - size_t _dim_size; - bool _normalize_vectors; - vespalib::FakeDoom _no_doom; - - bool check_lid(uint32_t lid); - bool check_value(const char *op, const std::vector<float>& value); - TypedCells get_typed_cells(const std::vector<float>& value, std::vector<float>& normalized_value); -public: - HnswIndex(uint32_t dim_size, const HnswIndexParams &hnsw_index_params, bool normalize_vectors); - virtual ~HnswIndex(); - void set_vector(uint32_t lid, const std::vector<float>& value); - std::vector<float> get_vector(uint32_t lid); - void clear_vector(uint32_t lid); - TopKResult find_top_k(uint32_t k, const std::vector<float>& value, uint32_t explore_k); -}; - -HnswIndex::HnswIndex(uint32_t dim_size, const HnswIndexParams &hnsw_index_params, bool normalize_vectors) - : _tensor_type(ValueType::error_type()), - _hnsw_index_params(hnsw_index_params), - _attribute(), - _tensor_attribute(nullptr), - _nearest_neighbor_index(nullptr), - _dim_size(0u), - _normalize_vectors(normalize_vectors), - _no_doom() -{ - Config cfg(BasicType::TENSOR, CollectionType::SINGLE); - _tensor_type = ValueType::from_spec(make_tensor_spec(dim_size)); - assert(_tensor_type.is_dense()); - assert(_tensor_type.count_indexed_dimensions() == 1u); - _dim_size = _tensor_type.dimensions()[0].size; - cfg.setTensorType(_tensor_type); - cfg.set_distance_metric(hnsw_index_params.distance_metric()); - cfg.set_hnsw_index_params(hnsw_index_params); - _attribute = AttributeFactory::createAttribute("tensor", cfg); - _tensor_attribute = dynamic_cast<TensorAttribute *>(_attribute.get()); - assert(_tensor_attribute != nullptr); - _nearest_neighbor_index = _tensor_attribute->nearest_neighbor_index(); - assert(_nearest_neighbor_index != nullptr); -} - -HnswIndex::~HnswIndex() = default; - -bool -HnswIndex::check_lid(uint32_t lid) -{ - if (lid >= std::numeric_limits<uint32_t>::max() - lid_bias) { - std::cerr << "lid is too high" << std::endl; - return false; - } - return true; -} - -bool -HnswIndex::check_value(const char *op, const std::vector<float>& value) -{ - if (value.size() != _dim_size) { - std::cerr << op << " failed, expected vector with size " << _dim_size << ", got vector with size " << value.size() << std::endl; - return false; - } - return true; -} - -TypedCells -HnswIndex::get_typed_cells(const std::vector<float>& value, std::vector<float>& normalized_value) -{ - if (!_normalize_vectors) { - return {&value[0], CellType::FLOAT, value.size()}; - } - double sum_of_squared = 0.0; - for (auto elem : value) { - double delem = elem; - sum_of_squared += delem * delem; - } - double factor = 1.0 / (sqrt(sum_of_squared) + 1e-40); - normalized_value.reserve(value.size()); - normalized_value.clear(); - for (auto elem : value) { - normalized_value.emplace_back(elem * factor); - } - return {&normalized_value[0], CellType::FLOAT, normalized_value.size()}; -} - -void -HnswIndex::set_vector(uint32_t lid, const std::vector<float>& value) -{ - if (!check_lid(lid)) { - return; - } - if (!check_value("set_vector", value)) { - return; - } - /* - * Not thread safe against concurrent set_vector(). - */ - std::vector<float> normalized_value; - auto typed_cells = get_typed_cells(value, normalized_value); - DenseValueView tensor_view(_tensor_type, typed_cells); - while (size_t(lid + lid_bias) >= _attribute->getNumDocs()) { - uint32_t new_lid = 0; - _attribute->addDoc(new_lid); - } - _tensor_attribute->setTensor(lid + lid_bias, tensor_view); // lid 0 is special in vespa - _attribute->commit(); -} - -std::vector<float> -HnswIndex::get_vector(uint32_t lid) -{ - if (!check_lid(lid)) { - return {}; - } - TypedCells typed_cells = _tensor_attribute->extract_cells_ref(lid + lid_bias); - assert(typed_cells.size == _dim_size); - const float* data = static_cast<const float* >(typed_cells.data); - return {data, data + _dim_size}; - return {}; -} - -void -HnswIndex::clear_vector(uint32_t lid) -{ - if (!check_lid(lid)) { - return; - } - if (size_t(lid + lid_bias) < _attribute->getNumDocs()) { - _attribute->clearDoc(lid + lid_bias); - _attribute->commit(); - } -} - -TopKResult -HnswIndex::find_top_k(uint32_t k, const std::vector<float>& value, uint32_t explore_k) -{ - if (!check_value("find_top_k", value)) { - return {}; - } - /* - * Not thread safe against concurrent set_vector() since attribute - * read guard is not taken here. - */ - TopKResult result; - std::vector<float> normalized_value; - auto typed_cells = get_typed_cells(value, normalized_value); - auto df = _nearest_neighbor_index->distance_function_factory().for_query_vector(typed_cells); - auto raw_result = _nearest_neighbor_index->find_top_k(k, *df, explore_k, _no_doom.get_doom(), std::numeric_limits<double>::max()); - result.reserve(raw_result.size()); - switch (_hnsw_index_params.distance_metric()) { - case DistanceMetric::Euclidean: - for (auto &raw : raw_result) { - result.emplace_back(raw.docid - lid_bias, sqrt(raw.distance)); - } - break; - default: - for (auto &raw : raw_result) { - result.emplace_back(raw.docid - lid_bias, raw.distance); - } - } - // Results are sorted by lid, not by distance - return result; -} - -} - -using vespa_ann_benchmark::HnswIndex; - -PYBIND11_MODULE(vespa_ann_benchmark, m) { - m.doc() = "vespa_ann_benchmark plugin"; - - py::enum_<DistanceMetric>(m, "DistanceMetric") - .value("Euclidean", DistanceMetric::Euclidean) - .value("Angular", DistanceMetric::Angular) - .value("InnerProduct", DistanceMetric::InnerProduct); - - py::class_<HnswIndexParams>(m, "HnswIndexParams") - .def(py::init<uint32_t, uint32_t, DistanceMetric, bool>()); - - py::class_<HnswIndex>(m, "HnswIndex") - .def(py::init<uint32_t, const HnswIndexParams&, bool>()) - .def("set_vector", &HnswIndex::set_vector) - .def("get_vector", &HnswIndex::get_vector) - .def("clear_vector", &HnswIndex::clear_vector) - .def("find_top_k", &HnswIndex::find_top_k); -} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java index 3de9d5aef4b..e1629a6c2c3 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java @@ -687,7 +687,7 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye private List<String> sortedUnusedFileReferences(File fileReferencesPath, Set<String> fileReferencesInUse, Instant instant) { Set<String> fileReferencesOnDisk = getFileReferencesOnDisk(fileReferencesPath); - log.log(Level.FINE, () -> "File references on disk (in " + fileReferencesPath + "): " + fileReferencesOnDisk); + log.log(Level.FINEST, () -> "File references on disk (in " + fileReferencesPath + "): " + fileReferencesOnDisk); return fileReferencesOnDisk .stream() .filter(fileReference -> ! fileReferencesInUse.contains(fileReference)) diff --git a/container-disc/src/main/java/com/yahoo/container/jdisc/DataplaneProxyService.java b/container-disc/src/main/java/com/yahoo/container/jdisc/DataplaneProxyService.java index 4c5ac951384..d94244b0e47 100644 --- a/container-disc/src/main/java/com/yahoo/container/jdisc/DataplaneProxyService.java +++ b/container-disc/src/main/java/com/yahoo/container/jdisc/DataplaneProxyService.java @@ -109,6 +109,10 @@ public class DataplaneProxyService extends AbstractComponent { config.tokenPort(), config.tokenEndpoints(), root)); + if (configChanged) { + logger.log(Level.INFO, "Configuring data plane proxy service. Token endpoints: [%s]" + .formatted(String.join(", ", config.tokenEndpoints()))); + } if (configChanged && state == NginxState.RUNNING) { changeState(NginxState.RELOAD_REQUIRED); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java index f9798fb2559..f11d67762ad 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java @@ -64,6 +64,8 @@ import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.TreeMap; +import java.util.logging.Level; +import java.util.logging.Logger; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -79,6 +81,8 @@ import static java.util.stream.Collectors.toMap; */ public class RoutingController { + private static final Logger LOG = Logger.getLogger(RoutingController.class.getName()); + private final Controller controller; private final RoutingPolicies routingPolicies; private final RotationRepository rotationRepository; @@ -213,6 +217,8 @@ public class RoutingController { // Register rotation-backed endpoints in DNS registerRotationEndpointsInDns(prepared); + LOG.log(Level.FINE, () -> "Prepared endpoints: " + prepared); + return prepared; } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/AssignedRotation.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/AssignedRotation.java index 7fa2a03d0a9..c2949e395e9 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/AssignedRotation.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/AssignedRotation.java @@ -23,16 +23,6 @@ public record AssignedRotation(ClusterSpec.Id clusterId, EndpointId endpointId, this.regions = Set.copyOf(Objects.requireNonNull(regions)); } - @Override - public String toString() { - return "AssignedRotation{" + - "clusterId=" + clusterId + - ", endpointId='" + endpointId + '\'' + - ", rotationId=" + rotationId + - ", regions=" + regions + - '}'; - } - private static <T> T requireNonEmpty(T object, String value, String field) { Objects.requireNonNull(object); Objects.requireNonNull(value); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandlerV2.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandlerV2.java index 83cd5dab2f3..5b43ede2e22 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandlerV2.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandlerV2.java @@ -4,7 +4,6 @@ package com.yahoo.vespa.hosted.controller.restapi.billing; import com.yahoo.config.provision.TenantName; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.container.jdisc.ThreadedHttpRequestHandler; -import com.yahoo.messagebus.Message; import com.yahoo.restapi.MessageResponse; import com.yahoo.restapi.RestApi; import com.yahoo.restapi.RestApiException; @@ -461,6 +460,7 @@ public class BillingApiHandlerV2 extends RestApiRequestHandler<BillingApiHandler toSlime(slime.setObject("cpu"), item.getCpuHours(), item.getCpuCost()); toSlime(slime.setObject("memory"), item.getMemoryHours(), item.getMemoryCost()); toSlime(slime.setObject("disk"), item.getDiskHours(), item.getDiskCost()); + toSlime(slime.setObject("gpu"), item.getGpuHours(), item.getGpuCost()); } private void toSlime(Cursor slime, Optional<BigDecimal> hours, Optional<BigDecimal> cost) { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandlerV2Test.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandlerV2Test.java index 424b8d84472..ea5729d8fde 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandlerV2Test.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandlerV2Test.java @@ -103,7 +103,7 @@ public class BillingApiHandlerV2Test extends ControllerContainerCloudTest { var singleRequest = request("/billing/v2/tenant/" + tenant + "/bill/id-1").roles(tenantReader); tester.assertResponse(singleRequest, """ - {"id":"id-1","from":"2020-05-23","to":"2020-05-28","total":"123.00","status":"OPEN","statusHistory":[{"at":"2020-05-23T00:00:00Z","status":"OPEN"}],"items":[{"id":"some-id","description":"description","amount":"123.00","plan":{"id":"paid","name":"Paid Plan - for testing purposes"},"majorVersion":0,"cpu":{},"memory":{},"disk":{}}]}"""); + {"id":"id-1","from":"2020-05-23","to":"2020-05-28","total":"123.00","status":"OPEN","statusHistory":[{"at":"2020-05-23T00:00:00Z","status":"OPEN"}],"items":[{"id":"some-id","description":"description","amount":"123.00","plan":{"id":"paid","name":"Paid Plan - for testing purposes"},"majorVersion":0,"cpu":{},"memory":{},"disk":{},"gpu":{}}]}"""); } @Test @@ -175,7 +175,7 @@ public class BillingApiHandlerV2Test extends ControllerContainerCloudTest { var accountantRequest = request("/billing/v2/accountant/tenant/tenant1/items") .roles(Role.hostedAccountant()); tester.assertResponse(accountantRequest, """ - {"items":[{"id":"line-item-id","description":"Additional support costs","amount":"123.45","plan":{"id":"paid","name":"Paid Plan - for testing purposes"},"majorVersion":0,"cpu":{},"memory":{},"disk":{}}]}"""); + {"items":[{"id":"line-item-id","description":"Additional support costs","amount":"123.45","plan":{"id":"paid","name":"Paid Plan - for testing purposes"},"majorVersion":0,"cpu":{},"memory":{},"disk":{},"gpu":{}}]}"""); } { diff --git a/dist/vespa.spec b/dist/vespa.spec index ce302ae0338..cd1381fd48d 100644 --- a/dist/vespa.spec +++ b/dist/vespa.spec @@ -280,29 +280,6 @@ Requires: %{name}-base-libs = %{version}-%{release} Vespa - The open big data serving engine - devel package -%package ann-benchmark - -Summary: Vespa - The open big data serving engine - ann-benchmark - -Requires: %{name}-base-libs = %{version}-%{release} -Requires: %{name}-libs = %{version}-%{release} -%if 0%{?el8} -Requires: python39 -%endif -%if 0%{?el9} -Requires: python3 -%endif -%if 0%{?fedora} -Requires: python3 -%endif - -%description ann-benchmark - -Vespa - The open big data serving engine - ann-benchmark - -Python binding for the Vespa implementation of an HNSW index for -nearest neighbor search used for low-level benchmarking. - %prep %if 0%{?installdir:1} %if 0%{?source_base:1} @@ -383,10 +360,6 @@ export JAVA_HOME=%{?_java_home} export JAVA_HOME=/usr/lib/jvm/java-17-openjdk %endif export PATH="$JAVA_HOME/bin:$PATH" -%if 0%{?el8} -python3.9 -m pip install --user pytest -%endif -export PYTHONPATH="$PYTHONPATH:/usr/local/lib/$(basename $(readlink -f $(which python3)))/site-packages" #%{?_use_mvn_wrapper:./mvnw}%{!?_use_mvn_wrapper:mvn} --batch-mode -nsu -T 1C -Dmaven.javadoc.skip=true test make test ARGS="--output-on-failure %{_smp_mflags}" %endif @@ -537,7 +510,6 @@ fi %{_prefix}/lib/jars/zookeeper-command-line-client-jar-with-dependencies.jar %{_prefix}/lib/perl5 %{_prefix}/libexec -%exclude %{_prefix}/libexec/vespa_ann_benchmark %exclude %{_prefix}/libexec/vespa/common-env.sh %exclude %{_prefix}/libexec/vespa/vespa-wrapper %exclude %{_prefix}/libexec/vespa/find-pid @@ -763,12 +735,4 @@ fi %{_prefix}/include %{_prefix}/share/cmake -%files ann-benchmark -%if %{_defattr_is_vespa_vespa} -%defattr(-,%{_vespa_user},%{_vespa_group},-) -%endif -%dir %{_prefix} -%dir %{_prefix}/libexec -%{_prefix}/libexec/vespa_ann_benchmark - %changelog diff --git a/document/src/vespa/document/bucket/bucketid.cpp b/document/src/vespa/document/bucket/bucketid.cpp index dee818b407e..fcc9a0df4f6 100644 --- a/document/src/vespa/document/bucket/bucketid.cpp +++ b/document/src/vespa/document/bucket/bucketid.cpp @@ -8,7 +8,6 @@ #include <vespa/vespalib/stllike/hash_set.hpp> #include <vespa/vespalib/util/stringfmt.h> #include <limits> -#include <xxhash.h> using vespalib::nbostream; using vespalib::asciistream; @@ -79,29 +78,7 @@ void BucketId::initialize() noexcept { uint64_t BucketId::hash::operator () (const BucketId& bucketId) const noexcept { - const uint64_t raw_id = bucketId.getId(); - /* - * This is a workaround for gcc 12 and on that produces incorrect warning when compiled with -march=haswell or newer - * /home/balder/git/vespa/document/src/vespa/document/bucket/bucketid.cpp: In member function ‘uint64_t document::BucketId::hash::operator()(const document::BucketId&) const’: - * /home/balder/git/vespa/document/src/vespa/document/bucket/bucketid.cpp:83:23: error: ‘raw_id’ may be used uninitialized [-Werror=maybe-uninitialized] - * 83 | return XXH3_64bits(&raw_id, sizeof(uint64_t)); - * | ^ - * In file included from /usr/include/xxh3.h:55, - * from /home/balder/git/vespa/document/src/vespa/document/bucket/bucketid.cpp:11: - * /usr/include/xxhash.h:5719:29: note: by argument 1 of type ‘const void*’ to ‘XXH64_hash_t XXH_INLINE_XXH3_64bits(const void*, size_t)’ declared here - * 5719 | XXH_PUBLIC_API XXH64_hash_t XXH3_64bits(XXH_NOESCAPE const void* input, size_t length) - * | ^~~~~~~~~~~ - * /home/balder/git/vespa/document/src/vespa/document/bucket/bucketid.cpp:82:14: note: ‘raw_id’ declared here - * 82 | uint64_t raw_id = bucketId.getId(); - * | ^~~~~~ - * cc1plus: all warnings being treated as errors - * - * Same issue in storage/src/vespa/storage/persistence/filestorhandlerimpl.cpp:FileStorHandlerImpl::dispersed_bucket_bits - */ - uint8_t raw_as_bytes[sizeof(raw_id)]; - memcpy(raw_as_bytes, &raw_id, sizeof(raw_id)); - - return XXH3_64bits(raw_as_bytes, sizeof(raw_as_bytes)); + return vespalib::xxhash::xxh3_64(bucketId.getId()); } vespalib::string diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index c2f6f782dbc..d02cccb2885 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -309,12 +309,6 @@ public class Flags { "Whether to enable CrowdStrike.", "Takes effect on next host admin tick", HOSTNAME); - public static final UnboundBooleanFlag RANDOMIZED_ENDPOINT_NAMES = defineFeatureFlag( - "randomized-endpoint-names", false, List.of("andreer"), "2023-04-26", "2023-11-14", - "Whether to use randomized endpoint names", - "Takes effect on application deployment", - INSTANCE_ID, APPLICATION_ID, TENANT_ID); - public static final UnboundBooleanFlag ENABLE_THE_ONE_THAT_SHOULD_NOT_BE_NAMED = defineFeatureFlag( "enable-the-one-that-should-not-be-named", false, List.of("hmusum"), "2023-05-08", "2023-11-01", "Whether to enable the one program that should not be named", @@ -387,20 +381,6 @@ public class Flags { "Takes effect immediately", INSTANCE_ID, CLUSTER_ID, CLUSTER_TYPE); - public static final UnboundBooleanFlag ASSIGN_RANDOMIZED_ID = defineFeatureFlag( - "assign-randomized-id", true, - List.of("mortent"), "2023-08-31", "2024-02-01", - "Whether to assign randomized id to the application", - "Takes effect immediately", - INSTANCE_ID); - - public static final UnboundIntFlag ASSIGNED_RANDOMIZED_ID_RATE = defineIntFlag( - "assign-randomized-id-rate", 5, - List.of("mortent"), "2023-09-11", "2024-02-01", - "Rate for requesting assigned ids for existing certificates. Rate is per maintainer cycle.", - "Takes effect immediately", - INSTANCE_ID); - public static final UnboundIntFlag CONTENT_LAYER_METADATA_FEATURE_LEVEL = defineIntFlag( "content-layer-metadata-feature-level", 0, List.of("vekterli"), "2022-09-12", "2024-02-01", @@ -423,12 +403,6 @@ public class Flags { "Takes effect at redeployment", INSTANCE_ID); - public static final UnboundBooleanFlag LEGACY_ENDPOINTS = defineFeatureFlag( - "legacy-endpoints", true, List.of("mpolden", "tokle"), "2023-09-29", "2024-03-01", - "Whether legacy (non-anonymized) endpoints should be created in DNS", - "Takes effect on redeployment through controller", - INSTANCE_ID, APPLICATION_ID, TENANT_ID); - public static final UnboundIntFlag SEARCH_HANDLER_THREADPOOL = defineIntFlag( "search-handler-threadpool", 2, List.of("bjorncs", "baldersheim"), "2023-10-01", "2024-01-01", diff --git a/hosted-tenant-base/pom.xml b/hosted-tenant-base/pom.xml index 729150bcef1..232ecd0e3e5 100644 --- a/hosted-tenant-base/pom.xml +++ b/hosted-tenant-base/pom.xml @@ -284,6 +284,9 @@ <exclude>org.junit.platform:junit-platform-engine:(${junit.platform.vespa.tenant.version},):jar:*</exclude> <exclude>org.junit.platform:junit-platform-commons:(,${junit.platform.vespa.tenant.version}):jar:*</exclude> <exclude>org.junit.platform:junit-platform-commons:(${junit.platform.vespa.tenant.version},):jar:*</exclude> + + <!-- Ban testng as its presence breaks the Java based test framework for Cloud --> + <exclude>org.testng:testng:*:jar:test</exclude> </excludes> </bannedDependencies> </rules> diff --git a/searchcore/src/tests/proton/common/timer/timer_test.cpp b/searchcore/src/tests/proton/common/timer/timer_test.cpp index 1efae97c6e0..4419275230a 100644 --- a/searchcore/src/tests/proton/common/timer/timer_test.cpp +++ b/searchcore/src/tests/proton/common/timer/timer_test.cpp @@ -84,7 +84,7 @@ TYPED_TEST(ScheduledExecutorTest, test_drop_handle) { } TYPED_TEST(ScheduledExecutorTest, test_only_one_instance_running) { - vespalib::TimeBomb time_bomb(60s); + vespalib::TimeBomb time_bomb(120s); vespalib::Gate latch; std::atomic<uint64_t> counter = 0; auto handleA = this->timer->scheduleAtFixedRate(makeLambdaTask([&]() { counter++; latch.await();}), 0ms, 1ms); @@ -96,7 +96,7 @@ TYPED_TEST(ScheduledExecutorTest, test_only_one_instance_running) { } TYPED_TEST(ScheduledExecutorTest, test_sync_delete) { - vespalib::TimeBomb time_bomb(60s); + vespalib::TimeBomb time_bomb(120s); vespalib::Gate latch; std::atomic<uint64_t> counter = 0; std::atomic<uint64_t> reset_counter = 0; diff --git a/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.cpp b/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.cpp index 582b67a4dbc..a96bee2d11a 100644 --- a/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.cpp +++ b/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.cpp @@ -14,7 +14,6 @@ #include <vespa/vespalib/stllike/hash_map.hpp> #include <vespa/vespalib/util/exceptions.h> #include <vespa/vespalib/util/string_escape.h> -#include <xxhash.h> #include <vespa/log/log.h> LOG_SETUP(".persistence.filestor.handler.impl"); @@ -896,14 +895,7 @@ FileStorHandlerImpl::flush() uint64_t FileStorHandlerImpl::dispersed_bucket_bits(const document::Bucket& bucket) noexcept { - const uint64_t raw_id = bucket.getBucketId().getId(); - /* - * This is a workaround for gcc 12 and on that produces incorrect warning when compiled with -march=haswell or newer - * See document/src/vespa/document/bucket/bucketid.cpp: In member function ‘uint64_t document::BucketId::hash::operator()(const document::BucketId&) const - */ - uint8_t raw_as_bytes[sizeof(raw_id)]; - memcpy(raw_as_bytes, &raw_id, sizeof(raw_id)); - return XXH3_64bits(&raw_as_bytes, sizeof(raw_id)); + return vespalib::xxhash::xxh3_64(bucket.getBucketId().getId()); } FileStorHandlerImpl::Stripe::Stripe(const FileStorHandlerImpl & owner, MessageSender & messageSender) diff --git a/vespalib/src/vespa/vespalib/fuzzy/sparse_state.h b/vespalib/src/vespa/vespalib/fuzzy/sparse_state.h index 7e381468fbe..e023f4d3de2 100644 --- a/vespalib/src/vespa/vespalib/fuzzy/sparse_state.h +++ b/vespalib/src/vespa/vespalib/fuzzy/sparse_state.h @@ -2,13 +2,12 @@ #pragma once #include <vespa/config.h> +#include <vespa/vespalib/stllike/hash_fun.h> #include <algorithm> #include <array> #include <cassert> -#include <cstdint> #include <ostream> #include <span> -#include <xxhash.h> // TODO factor out? namespace vespalib::fuzzy { @@ -80,15 +79,8 @@ public: size_t operator()(const FixedSparseState& s) const noexcept { static_assert(std::is_same_v<uint32_t, std::decay_t<decltype(s.indices[0])>>); static_assert(std::is_same_v<uint8_t, std::decay_t<decltype(s.costs[0])>>); - // FIXME GCC 12.2 worse-than-useless(tm) warning false positives :I -#pragma GCC diagnostic push -#ifdef VESPA_USE_SANITIZER -# pragma GCC diagnostic ignored "-Wstringop-overread" // https://gcc.gnu.org/bugzilla/show_bug.cgi?id=98465 etc. -#endif -#pragma GCC diagnostic ignored "-Warray-bounds" - return (XXH3_64bits(s.indices.data(), s.sz * sizeof(uint32_t)) ^ - XXH3_64bits(s.costs.data(), s.sz)); -#pragma GCC diagnostic pop + return (xxhash::xxh3_64(s.indices.data(), s.sz * sizeof(uint32_t)) ^ + xxhash::xxh3_64(s.costs.data(), s.sz)); } }; }; diff --git a/vespalib/src/vespa/vespalib/stllike/hash_fun.cpp b/vespalib/src/vespa/vespalib/stllike/hash_fun.cpp index e4911d2e0c6..6802e5f91a6 100644 --- a/vespalib/src/vespa/vespalib/stllike/hash_fun.cpp +++ b/vespalib/src/vespa/vespalib/stllike/hash_fun.cpp @@ -6,23 +6,21 @@ namespace vespalib { size_t -hashValue(const char *str) noexcept -{ - return hashValue(str, strlen(str)); +hashValue(const char *str) noexcept { + return xxhash::xxh3_64(str, strlen(str)); } -/** - * @brief Calculate hash value. - * - * The hash function XXH3_64bits from xxhash library. - * @param buf input buffer - * @param sz input buffer size - * @return hash value of input - **/ -size_t -hashValue(const void * buf, size_t sz) noexcept -{ +namespace xxhash { + +uint64_t +xxh3_64(uint64_t value) noexcept { + return XXH3_64bits(&value, sizeof(value)); +} + +uint64_t +xxh3_64(const void * buf, size_t sz) noexcept { return XXH3_64bits(buf, sz); } } +} diff --git a/vespalib/src/vespa/vespalib/stllike/hash_fun.h b/vespalib/src/vespa/vespalib/stllike/hash_fun.h index f8a8a06b921..8fecc41b4c1 100644 --- a/vespalib/src/vespa/vespalib/stllike/hash_fun.h +++ b/vespalib/src/vespa/vespalib/stllike/hash_fun.h @@ -2,7 +2,6 @@ #pragma once #include <vespa/vespalib/stllike/string.h> -#include <cstdint> namespace vespalib { @@ -64,9 +63,18 @@ template<typename T> struct hash<const T *> { size_t operator() (const T * arg) const noexcept { return size_t(arg); } }; +namespace xxhash { + +uint64_t xxh3_64(uint64_t value) noexcept; +uint64_t xxh3_64(const void *str, size_t sz) noexcept; + +} + // reuse old string hash function size_t hashValue(const char *str) noexcept; -size_t hashValue(const void *str, size_t sz) noexcept; +inline size_t hashValue(const void *buf, size_t sz) noexcept { + return xxhash::xxh3_64(buf, sz); +} struct hash_strings { size_t operator() (const vespalib::string & arg) const noexcept { return hashValue(arg.data(), arg.size()); } diff --git a/vespalib/src/vespa/vespalib/util/shared_string_repo.cpp b/vespalib/src/vespa/vespalib/util/shared_string_repo.cpp index a5942e1af9b..09f2bbd828d 100644 --- a/vespalib/src/vespa/vespalib/util/shared_string_repo.cpp +++ b/vespalib/src/vespa/vespalib/util/shared_string_repo.cpp @@ -1,7 +1,7 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "shared_string_repo.h" -#include <xxhash.h> +#include <vespa/vespalib/stllike/hash_fun.h> #include <charconv> #include <cassert> @@ -232,7 +232,7 @@ string_id SharedStringRepo::resolve(vespalib::stringref str) { uint32_t direct_id = try_make_direct_id(str); if (direct_id >= ID_BIAS) { - uint64_t full_hash = XXH3_64bits(str.data(), str.size()); + uint64_t full_hash = xxhash::xxh3_64(str.data(), str.size()); uint32_t part = full_hash & PART_MASK; uint32_t local_hash = full_hash >> PART_BITS; uint32_t local_idx = _partitions[part].resolve(AltKey{str, local_hash}); |