diff options
38 files changed, 114 insertions, 599 deletions
diff --git a/CMakeLists.txt b/CMakeLists.txt index a4e89cbdeb9..1bdb3a23730 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -51,9 +51,6 @@ find_package(JNI REQUIRED) find_package(GTest REQUIRED) -find_package(Python 3.6 COMPONENTS Interpreter Development REQUIRED) -find_package(pybind11 CONFIG REQUIRED) - find_package(Protobuf REQUIRED) include(build_settings.cmake) @@ -100,7 +97,6 @@ vespa_install_data(tsan-suppressions.txt etc/vespa) include_directories(BEFORE ${CMAKE_BINARY_DIR}/configdefinitions/src) add_subdirectory(airlift-zstd) -add_subdirectory(ann_benchmark) add_subdirectory(application-model) add_subdirectory(client) add_subdirectory(cloud-tenant-cd) diff --git a/ann_benchmark/CMakeLists.txt b/ann_benchmark/CMakeLists.txt deleted file mode 100644 index 06d742cf072..00000000000 --- a/ann_benchmark/CMakeLists.txt +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_define_module( - DEPENDS - searchlib - - LIBS - src/vespa/ann_benchmark - - APPS - - TESTS - src/tests/ann_benchmark -) diff --git a/ann_benchmark/src/tests/ann_benchmark/.gitignore b/ann_benchmark/src/tests/ann_benchmark/.gitignore deleted file mode 100644 index 225fc6f6650..00000000000 --- a/ann_benchmark/src/tests/ann_benchmark/.gitignore +++ /dev/null @@ -1 +0,0 @@ -/__pycache__ diff --git a/ann_benchmark/src/tests/ann_benchmark/CMakeLists.txt b/ann_benchmark/src/tests/ann_benchmark/CMakeLists.txt deleted file mode 100644 index 03126ce1b47..00000000000 --- a/ann_benchmark/src/tests/ann_benchmark/CMakeLists.txt +++ /dev/null @@ -1,5 +0,0 @@ -# Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -if(NOT DEFINED VESPA_USE_SANITIZER) - vespa_add_test(NAME ann_benchmark_test NO_VALGRIND COMMAND ${Python_EXECUTABLE} -m pytest WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} DEPENDS vespa_ann_benchmark) -endif() diff --git a/ann_benchmark/src/tests/ann_benchmark/test_angular.py b/ann_benchmark/src/tests/ann_benchmark/test_angular.py deleted file mode 100644 index ac7feb29d76..00000000000 --- a/ann_benchmark/src/tests/ann_benchmark/test_angular.py +++ /dev/null @@ -1,41 +0,0 @@ -# Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -import pytest -import sys -import os -import math -sys.path.insert(0, os.path.abspath("../../vespa/ann_benchmark")) -from vespa_ann_benchmark import DistanceMetric, HnswIndexParams, HnswIndex - -class Fixture: - def __init__(self, normalize): - metric = DistanceMetric.InnerProduct if normalize else DistanceMetric.Angular - self.index = HnswIndex(2, HnswIndexParams(16, 200, metric, False), normalize) - self.index.set_vector(0, [1, 0]) - self.index.set_vector(1, [10, 10]) - - def find(self, k, value): - return self.index.find_top_k(k, value, k + 200) - - def run_test(self): - top = self.find(10, [1, 1]) - assert [top[0][0], top[1][0]] == [0, 1] - # Allow some rounding errors - epsilon = 6e-8 - assert abs((1 - top[0][1]) - math.sqrt(0.5)) < epsilon - assert abs((1 - top[1][1]) - 1) < epsilon - top2 = self.find(10, [0, 2]) - # Result is not sorted by distance - assert [top2[0][0], top2[1][0]] == [0, 1] - assert abs((1 - top2[0][1]) - 0) < epsilon - assert abs((1 - top2[1][1]) - math.sqrt(0.5)) < epsilon - assert 1 == self.find(1, [1, 1])[0][0] - assert 0 == self.find(1, [1, -1])[0][0] - -def test_find_angular(): - f = Fixture(False) - f.run_test() - -def test_find_angular_normalized(): - f = Fixture(True) - f.run_test() diff --git a/ann_benchmark/src/tests/ann_benchmark/test_euclidean.py b/ann_benchmark/src/tests/ann_benchmark/test_euclidean.py deleted file mode 100644 index ca4d5ecd6a1..00000000000 --- a/ann_benchmark/src/tests/ann_benchmark/test_euclidean.py +++ /dev/null @@ -1,61 +0,0 @@ -# Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -import pytest -import sys -import os -import math -sys.path.insert(0, os.path.abspath("../../vespa/ann_benchmark")) -from vespa_ann_benchmark import DistanceMetric, HnswIndexParams, HnswIndex - -class Fixture: - def __init__(self): - self.index = HnswIndex(2, HnswIndexParams(16, 200, DistanceMetric.Euclidean, False), False) - - def set(self, lid, value): - self.index.set_vector(lid, value) - - def get(self, lid): - return self.index.get_vector(lid) - - def clear(self, lid): - return self.index.clear_vector(lid) - - def find(self, k, value): - return self.index.find_top_k(k, value, k + 200) - -def test_set_value(): - f = Fixture() - f.set(0, [1, 2]) - f.set(1, [3, 4]) - assert [1, 2] == f.get(0) - assert [3, 4] == f.get(1) - -def test_clear_value(): - f = Fixture() - f.set(0, [1, 2]) - assert [1, 2] == f.get(0) - f.clear(0) - assert [0, 0] == f.get(0) - -def test_find(): - f = Fixture() - f.set(0, [0, 0]) - f.set(1, [10, 10]) - top = f.find(10, [1, 1]) - assert [top[0][0], top[1][0]] == [0, 1] - # Allow some rounding errors - epsilon = 1e-20 - assert abs(top[0][1] - math.sqrt(2)) < epsilon - assert abs(top[1][1] - math.sqrt(162)) < epsilon - top2 = f.find(10, [9, 9]) - # Result is not sorted by distance - assert [top2[0][0], top2[1][0]] == [0, 1] - assert abs(top2[0][1] - math.sqrt(162)) < epsilon - assert abs(top2[1][1] - math.sqrt(2)) < epsilon - assert 0 == f.find(1, [1, 1])[0][0] - assert 1 == f.find(1, [9, 9])[0][0] - f.clear(1) - assert 0 == f.find(1, [9, 9])[0][0] - assert 0 == f.find(1, [0, 0])[0][0] - f.clear(0) - assert 0 == len(f.find(1, [9, 9])) diff --git a/ann_benchmark/src/vespa/ann_benchmark/.gitignore b/ann_benchmark/src/vespa/ann_benchmark/.gitignore deleted file mode 100644 index 3b4605aeee2..00000000000 --- a/ann_benchmark/src/vespa/ann_benchmark/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -/vespa_ann_benchmark.cpython*.so -/setup.py diff --git a/ann_benchmark/src/vespa/ann_benchmark/CMakeLists.txt b/ann_benchmark/src/vespa/ann_benchmark/CMakeLists.txt deleted file mode 100644 index fa3b10b2269..00000000000 --- a/ann_benchmark/src/vespa/ann_benchmark/CMakeLists.txt +++ /dev/null @@ -1,31 +0,0 @@ -# Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -install(DIRECTORY DESTINATION libexec/vespa_ann_benchmark) - -vespa_add_library(vespa_ann_benchmark - ALLOW_UNRESOLVED_SYMBOLS - SOURCES - vespa_ann_benchmark.cpp - - INSTALL libexec/vespa_ann_benchmark - DEPENDS - pybind11::pybind11 -) - -if (TARGET pybind11::lto) - target_link_libraries(vespa_ann_benchmark PRIVATE pybind11::module pybind11::lto) -else() - target_link_libraries(vespa_ann_benchmark PRIVATE pybind11::module) -endif() - -if (COMMAND pybind11_extension) - pybind11_extension(vespa_ann_benchmark) -else() - set_target_properties(vespa_ann_benchmark PROPERTIES PREFIX "${PYTHON_MODULE_PREFIX}") - set_target_properties(vespa_ann_benchmark PROPERTIES SUFFIX "${PYTHON_MODULE_EXTENSION}") -endif() - -set_target_properties(vespa_ann_benchmark PROPERTIES CXX_VISIBILITY_PRESET "hidden") - -configure_file(setup.py.in setup.py @ONLY) - -vespa_install_script(setup.py libexec/vespa_ann_benchmark) diff --git a/ann_benchmark/src/vespa/ann_benchmark/setup.py.in b/ann_benchmark/src/vespa/ann_benchmark/setup.py.in deleted file mode 100644 index 457d6e1b4b5..00000000000 --- a/ann_benchmark/src/vespa/ann_benchmark/setup.py.in +++ /dev/null @@ -1,27 +0,0 @@ -# Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -import subprocess -import sys -import platform -import distutils.sysconfig -from setuptools import setup, Extension -from setuptools.command.build_ext import build_ext - -class PreBuiltExt(build_ext): - def build_extension(self, ext): - print("Using prebuilt extension library") - libdir="lib.%s-%s-%s" % (sys.platform, platform.machine(), distutils.sysconfig.get_python_version()) - subprocess.run(["mkdir", "-p", "build/%s" % libdir]) - subprocess.run(["cp", "-p", "@PYTHON_MODULE_PREFIX@vespa_ann_benchmark@PYTHON_MODULE_EXTENSION@", "build/%s" % libdir]) - -setup( - name="vespa_ann_benchmark", - version="0.1.0", - author="Tor Egge", - author_email="Tor.Egge@yahooinc.com", - description="Python binding for the Vespa implementation of an HNSW index for nearest neighbor search", - long_description="Python binding for the Vespa implementation of an HNSW index for nearest neighbor search used for low-level benchmarking", - ext_modules=[Extension("vespa_ann_benchmark", sources=[])], - cmdclass={"build_ext": PreBuiltExt}, - zip_safe=False, -) diff --git a/ann_benchmark/src/vespa/ann_benchmark/vespa_ann_benchmark.cpp b/ann_benchmark/src/vespa/ann_benchmark/vespa_ann_benchmark.cpp deleted file mode 100644 index ab00f997226..00000000000 --- a/ann_benchmark/src/vespa/ann_benchmark/vespa_ann_benchmark.cpp +++ /dev/null @@ -1,252 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include <pybind11/pybind11.h> -#include <pybind11/stl.h> -#include <vespa/searchcommon/attribute/hnsw_index_params.h> -#include <vespa/searchlib/attribute/attributevector.h> -#include <vespa/searchlib/attribute/attributefactory.h> -#include <vespa/searchlib/tensor/dense_tensor_attribute.h> -#include <vespa/searchlib/tensor/nearest_neighbor_index.h> -#include <vespa/searchcommon/attribute/config.h> -#include <vespa/eval/eval/value.h> -#include <vespa/vespalib/test/insertion_operators.h> -#include <vespa/vespalib/util/fake_doom.h> -#include <iostream> -#include <sstream> -#include <limits> - -namespace py = pybind11; - -using search::AttributeFactory; -using search::AttributeVector; -using search::attribute::BasicType; -using search::attribute::Config; -using search::attribute::CollectionType; -using search::attribute::DistanceMetric; -using search::attribute::HnswIndexParams; -using search::tensor::NearestNeighborIndex; -using search::tensor::TensorAttribute; -using vespalib::eval::CellType; -using vespalib::eval::DenseValueView; -using vespalib::eval::TypedCells; -using vespalib::eval::ValueType; -using vespalib::eval::Value; - -namespace vespa_ann_benchmark { - -using TopKResult = std::vector<std::pair<uint32_t, double>>; - -namespace { - -std::string -make_tensor_spec(uint32_t dim_size) -{ - std::ostringstream os; - os << "tensor<float>(x[" << dim_size << "])"; - return os.str(); -} - -constexpr uint32_t lid_bias = 1; // lid 0 is reserved - -} - -/* - * Class exposing the Vespa implementation of an HNSW index for nearest neighbor search over data points in a high dimensional vector space. - * - * A tensor attribute field (https://docs.vespa.ai/en/reference/schema-reference.html#type:tensor) is used to store the vectors in memory. - * This class only supports single-threaded access (both for indexing and searching), - * and should only be used for low-level benchmarking. - * To use nearest neighbor search in a Vespa application, - * see https://docs.vespa.ai/en/approximate-nn-hnsw.html for more details. - */ -class HnswIndex -{ - ValueType _tensor_type; - HnswIndexParams _hnsw_index_params; - std::shared_ptr<AttributeVector> _attribute; - TensorAttribute* _tensor_attribute; - const NearestNeighborIndex* _nearest_neighbor_index; - size_t _dim_size; - bool _normalize_vectors; - vespalib::FakeDoom _no_doom; - - bool check_lid(uint32_t lid); - bool check_value(const char *op, const std::vector<float>& value); - TypedCells get_typed_cells(const std::vector<float>& value, std::vector<float>& normalized_value); -public: - HnswIndex(uint32_t dim_size, const HnswIndexParams &hnsw_index_params, bool normalize_vectors); - virtual ~HnswIndex(); - void set_vector(uint32_t lid, const std::vector<float>& value); - std::vector<float> get_vector(uint32_t lid); - void clear_vector(uint32_t lid); - TopKResult find_top_k(uint32_t k, const std::vector<float>& value, uint32_t explore_k); -}; - -HnswIndex::HnswIndex(uint32_t dim_size, const HnswIndexParams &hnsw_index_params, bool normalize_vectors) - : _tensor_type(ValueType::error_type()), - _hnsw_index_params(hnsw_index_params), - _attribute(), - _tensor_attribute(nullptr), - _nearest_neighbor_index(nullptr), - _dim_size(0u), - _normalize_vectors(normalize_vectors), - _no_doom() -{ - Config cfg(BasicType::TENSOR, CollectionType::SINGLE); - _tensor_type = ValueType::from_spec(make_tensor_spec(dim_size)); - assert(_tensor_type.is_dense()); - assert(_tensor_type.count_indexed_dimensions() == 1u); - _dim_size = _tensor_type.dimensions()[0].size; - cfg.setTensorType(_tensor_type); - cfg.set_distance_metric(hnsw_index_params.distance_metric()); - cfg.set_hnsw_index_params(hnsw_index_params); - _attribute = AttributeFactory::createAttribute("tensor", cfg); - _tensor_attribute = dynamic_cast<TensorAttribute *>(_attribute.get()); - assert(_tensor_attribute != nullptr); - _nearest_neighbor_index = _tensor_attribute->nearest_neighbor_index(); - assert(_nearest_neighbor_index != nullptr); -} - -HnswIndex::~HnswIndex() = default; - -bool -HnswIndex::check_lid(uint32_t lid) -{ - if (lid >= std::numeric_limits<uint32_t>::max() - lid_bias) { - std::cerr << "lid is too high" << std::endl; - return false; - } - return true; -} - -bool -HnswIndex::check_value(const char *op, const std::vector<float>& value) -{ - if (value.size() != _dim_size) { - std::cerr << op << " failed, expected vector with size " << _dim_size << ", got vector with size " << value.size() << std::endl; - return false; - } - return true; -} - -TypedCells -HnswIndex::get_typed_cells(const std::vector<float>& value, std::vector<float>& normalized_value) -{ - if (!_normalize_vectors) { - return {&value[0], CellType::FLOAT, value.size()}; - } - double sum_of_squared = 0.0; - for (auto elem : value) { - double delem = elem; - sum_of_squared += delem * delem; - } - double factor = 1.0 / (sqrt(sum_of_squared) + 1e-40); - normalized_value.reserve(value.size()); - normalized_value.clear(); - for (auto elem : value) { - normalized_value.emplace_back(elem * factor); - } - return {&normalized_value[0], CellType::FLOAT, normalized_value.size()}; -} - -void -HnswIndex::set_vector(uint32_t lid, const std::vector<float>& value) -{ - if (!check_lid(lid)) { - return; - } - if (!check_value("set_vector", value)) { - return; - } - /* - * Not thread safe against concurrent set_vector(). - */ - std::vector<float> normalized_value; - auto typed_cells = get_typed_cells(value, normalized_value); - DenseValueView tensor_view(_tensor_type, typed_cells); - while (size_t(lid + lid_bias) >= _attribute->getNumDocs()) { - uint32_t new_lid = 0; - _attribute->addDoc(new_lid); - } - _tensor_attribute->setTensor(lid + lid_bias, tensor_view); // lid 0 is special in vespa - _attribute->commit(); -} - -std::vector<float> -HnswIndex::get_vector(uint32_t lid) -{ - if (!check_lid(lid)) { - return {}; - } - TypedCells typed_cells = _tensor_attribute->extract_cells_ref(lid + lid_bias); - assert(typed_cells.size == _dim_size); - const float* data = static_cast<const float* >(typed_cells.data); - return {data, data + _dim_size}; - return {}; -} - -void -HnswIndex::clear_vector(uint32_t lid) -{ - if (!check_lid(lid)) { - return; - } - if (size_t(lid + lid_bias) < _attribute->getNumDocs()) { - _attribute->clearDoc(lid + lid_bias); - _attribute->commit(); - } -} - -TopKResult -HnswIndex::find_top_k(uint32_t k, const std::vector<float>& value, uint32_t explore_k) -{ - if (!check_value("find_top_k", value)) { - return {}; - } - /* - * Not thread safe against concurrent set_vector() since attribute - * read guard is not taken here. - */ - TopKResult result; - std::vector<float> normalized_value; - auto typed_cells = get_typed_cells(value, normalized_value); - auto df = _nearest_neighbor_index->distance_function_factory().for_query_vector(typed_cells); - auto raw_result = _nearest_neighbor_index->find_top_k(k, *df, explore_k, _no_doom.get_doom(), std::numeric_limits<double>::max()); - result.reserve(raw_result.size()); - switch (_hnsw_index_params.distance_metric()) { - case DistanceMetric::Euclidean: - for (auto &raw : raw_result) { - result.emplace_back(raw.docid - lid_bias, sqrt(raw.distance)); - } - break; - default: - for (auto &raw : raw_result) { - result.emplace_back(raw.docid - lid_bias, raw.distance); - } - } - // Results are sorted by lid, not by distance - return result; -} - -} - -using vespa_ann_benchmark::HnswIndex; - -PYBIND11_MODULE(vespa_ann_benchmark, m) { - m.doc() = "vespa_ann_benchmark plugin"; - - py::enum_<DistanceMetric>(m, "DistanceMetric") - .value("Euclidean", DistanceMetric::Euclidean) - .value("Angular", DistanceMetric::Angular) - .value("InnerProduct", DistanceMetric::InnerProduct); - - py::class_<HnswIndexParams>(m, "HnswIndexParams") - .def(py::init<uint32_t, uint32_t, DistanceMetric, bool>()); - - py::class_<HnswIndex>(m, "HnswIndex") - .def(py::init<uint32_t, const HnswIndexParams&, bool>()) - .def("set_vector", &HnswIndex::set_vector) - .def("get_vector", &HnswIndex::get_vector) - .def("clear_vector", &HnswIndex::clear_vector) - .def("find_top_k", &HnswIndex::find_top_k); -} diff --git a/client/go/internal/vespa/target.go b/client/go/internal/vespa/target.go index 065537d8610..662e7383f97 100644 --- a/client/go/internal/vespa/target.go +++ b/client/go/internal/vespa/target.go @@ -37,6 +37,7 @@ const ( ) var errWaitTimeout = errors.New("wait timed out") +var errAuth = errors.New("auth failed") // Authenticator authenticates the given HTTP request. type Authenticator interface { @@ -113,7 +114,7 @@ func (s *Service) Do(request *http.Request, timeout time.Duration) (*http.Respon }) if s.auth != nil { if err := s.auth.Authenticate(request); err != nil { - return nil, err + return nil, fmt.Errorf("%w: %s", errAuth, err) } } return s.httpClient.Do(request, timeout) @@ -220,7 +221,9 @@ func wait(service *Service, okFn responseFunc, reqFn requestFunc, timeout, retry loopOnce := timeout == 0 for time.Now().Before(deadline) || loopOnce { response, err = service.Do(reqFn(), 10*time.Second) - if err == nil { + if errors.Is(err, errAuth) { + return status, fmt.Errorf("aborting wait: %w", err) + } else if err == nil { status = response.StatusCode body, err := io.ReadAll(response.Body) if err != nil { diff --git a/client/go/internal/vespa/target_test.go b/client/go/internal/vespa/target_test.go index abcbefe5529..25e38a30151 100644 --- a/client/go/internal/vespa/target_test.go +++ b/client/go/internal/vespa/target_test.go @@ -120,6 +120,8 @@ func TestCustomTargetAwaitDeployment(t *testing.T) { func TestCloudTargetWait(t *testing.T) { var logWriter bytes.Buffer target, client := createCloudTarget(t, &logWriter) + client.NextResponseError(errAuth) + assertService(t, true, target, "deploy", time.Second) // No retrying on auth error client.NextStatus(401) assertService(t, true, target, "deploy", time.Second) // No retrying on 4xx client.NextStatus(500) diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/JvmHeapSizeValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/JvmHeapSizeValidator.java index 510fff66c10..425a662bb2d 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/JvmHeapSizeValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/JvmHeapSizeValidator.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.model.application.validation; import com.yahoo.config.model.deploy.DeployState; +import com.yahoo.text.Text; import com.yahoo.vespa.model.VespaModel; import java.util.logging.Level; @@ -32,21 +33,20 @@ public class JvmHeapSizeValidator extends Validator { double gbLimit = 0.6; double availableMemoryGb = mp.availableMemoryGb().getAsDouble(); double modelCostGb = jvmModelCost / (1024D * 1024 * 1024); - ds.getDeployLogger().log(Level.FINE, () -> "JVM: %d%% (limit: %d%%), %.2fGB (limit: %.2fGB), ONNX: %.2fGB" - .formatted(mp.percentage(), percentLimit, availableMemoryGb, gbLimit, modelCostGb)); + ds.getDeployLogger().log(Level.FINE, () -> Text.format("JVM: %d%% (limit: %d%%), %.2fGB (limit: %.2fGB), ONNX: %.2fGB", + mp.percentage(), percentLimit, availableMemoryGb, gbLimit, modelCostGb)); if (mp.percentage() < percentLimit) { - throw new IllegalArgumentException( - ("Allocated percentage of memory of JVM in cluster '%s' is too low (%d%% < %d%%). " + + throw new IllegalArgumentException(Text.format("Allocated percentage of memory of JVM in cluster '%s' is too low (%d%% < %d%%). " + "Estimated cost of ONNX models is %.2fGB. Either use a node flavor with more memory or use less expensive models. " + - "You may override this validation by specifying 'allocated-memory' (https://docs.vespa.ai/en/performance/container-tuning.html#jvm-heap-size).") - .formatted(clusterId, mp.percentage(), percentLimit, modelCostGb)); + "You may override this validation by specifying 'allocated-memory' (https://docs.vespa.ai/en/performance/container-tuning.html#jvm-heap-size).", + clusterId, mp.percentage(), percentLimit, modelCostGb)); } if (availableMemoryGb < gbLimit) { throw new IllegalArgumentException( - ("Allocated memory to JVM in cluster '%s' is too low (%.2fGB < %.2fGB). " + + Text.format("Allocated memory to JVM in cluster '%s' is too low (%.2fGB < %.2fGB). " + "Estimated cost of ONNX models is %.2fGB. Either use a node flavor with more memory or use less expensive models. " + - "You may override this validation by specifying 'allocated-memory' (https://docs.vespa.ai/en/performance/container-tuning.html#jvm-heap-size).") - .formatted(clusterId, availableMemoryGb, gbLimit, modelCostGb)); + "You may override this validation by specifying 'allocated-memory' (https://docs.vespa.ai/en/performance/container-tuning.html#jvm-heap-size).", + clusterId, availableMemoryGb, gbLimit, modelCostGb)); } } }); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java index 830440aaf8e..2093d0cfbe3 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java @@ -574,7 +574,12 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { Reader reader = file.createReader(); String certPem = IOUtils.readAll(reader); reader.close(); - List<X509Certificate> x509Certificates = X509CertificateUtils.certificateListFromPem(certPem); + List<X509Certificate> x509Certificates; + try { + x509Certificates = X509CertificateUtils.certificateListFromPem(certPem); + } catch (IllegalArgumentException e) { + throw new IllegalArgumentException("File %s contains an invalid certificate".formatted(file.getPath().getRelative()), e); + } if (x509Certificates.isEmpty()) { throw new IllegalArgumentException("File %s does not contain any certificates.".formatted(file.getPath().getRelative())); } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/JvmHeapSizeValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/JvmHeapSizeValidatorTest.java index d6da03f5b94..ebd8b5836da 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/JvmHeapSizeValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/JvmHeapSizeValidatorTest.java @@ -15,6 +15,7 @@ import com.yahoo.config.model.deploy.TestProperties; import com.yahoo.config.model.provision.InMemoryProvisioner; import com.yahoo.config.model.test.MockApplicationPackage; import com.yahoo.config.provision.NodeResources; +import com.yahoo.text.Text; import com.yahoo.vespa.model.VespaModel; import org.junit.jupiter.api.Test; import org.xml.sax.SAXException; @@ -97,7 +98,7 @@ class JvmHeapSizeValidatorTest { private static DeployState createDeployState(double nodeGb, long modelCostBytes) { String servicesXml = - """ + Text.format(""" <?xml version="1.0" encoding="utf-8" ?> <services version='1.0'> <container version='1.0'> @@ -109,7 +110,7 @@ class JvmHeapSizeValidatorTest { <tokenizer-model path="app/tokenizer.json"/> </component> </container> - </services>""".formatted(nodeGb); + </services>""", nodeGb); return createDeployState(servicesXml, nodeGb, modelCostBytes); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java index 3de9d5aef4b..e1629a6c2c3 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java @@ -687,7 +687,7 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye private List<String> sortedUnusedFileReferences(File fileReferencesPath, Set<String> fileReferencesInUse, Instant instant) { Set<String> fileReferencesOnDisk = getFileReferencesOnDisk(fileReferencesPath); - log.log(Level.FINE, () -> "File references on disk (in " + fileReferencesPath + "): " + fileReferencesOnDisk); + log.log(Level.FINEST, () -> "File references on disk (in " + fileReferencesPath + "): " + fileReferencesOnDisk); return fileReferencesOnDisk .stream() .filter(fileReference -> ! fileReferencesInUse.contains(fileReference)) diff --git a/container-core/src/main/java/com/yahoo/container/handler/Coverage.java b/container-core/src/main/java/com/yahoo/container/handler/Coverage.java index 69c6c9681bc..bbb47d14571 100644 --- a/container-core/src/main/java/com/yahoo/container/handler/Coverage.java +++ b/container-core/src/main/java/com/yahoo/container/handler/Coverage.java @@ -110,7 +110,7 @@ public class Coverage { return switch (fullReason) { case EXPLICITLY_FULL: yield true; case EXPLICITLY_INCOMPLETE: yield false; - case DOCUMENT_COUNT: yield docs == active; + case DOCUMENT_COUNT: yield (docs == active) && !((active == 0) && isDegradedByTimeout()) ; }; } @@ -163,6 +163,9 @@ public class Coverage { if (docs < total) { return (int) Math.round(docs * 100.0d / total); } + if ((total == 0) && isDegradedByTimeout()) { + return 0; + } return getFullResultSets() * 100 / getResultSets(); } diff --git a/container-core/src/main/java/com/yahoo/container/logging/Coverage.java b/container-core/src/main/java/com/yahoo/container/logging/Coverage.java index ee85711852d..397ff35b54c 100644 --- a/container-core/src/main/java/com/yahoo/container/logging/Coverage.java +++ b/container-core/src/main/java/com/yahoo/container/logging/Coverage.java @@ -59,6 +59,9 @@ public class Coverage { if (docs < total) { return (int) Math.round(docs * 100.0d / total); } + if ((total == 0) && isDegradedByTimeout()) { + return 0; + } return 100; } diff --git a/container-core/src/test/java/com/yahoo/container/logging/JSONLogTestCase.java b/container-core/src/test/java/com/yahoo/container/logging/JSONLogTestCase.java index 8887bc720ed..7132a0a5beb 100644 --- a/container-core/src/test/java/com/yahoo/container/logging/JSONLogTestCase.java +++ b/container-core/src/test/java/com/yahoo/container/logging/JSONLogTestCase.java @@ -291,6 +291,8 @@ public class JSONLogTestCase { newRequestLogEntry("test", new Coverage(100, 200, 200, 2)).build()); verifyCoverage("\"coverage\":{\"coverage\":50,\"documents\":100,\"degraded\":{\"adaptive-timeout\":true}}", newRequestLogEntry("test", new Coverage(100, 200, 200, 4)).build()); + verifyCoverage("\"coverage\":{\"coverage\":0,\"documents\":0,\"degraded\":{\"timeout\":true}}", + newRequestLogEntry("test", new Coverage(0, 0, 0, 2)).build()); } private String formatEntry(RequestLogEntry entry) { diff --git a/container-disc/src/main/java/com/yahoo/container/jdisc/DataplaneProxyService.java b/container-disc/src/main/java/com/yahoo/container/jdisc/DataplaneProxyService.java index 4c5ac951384..d94244b0e47 100644 --- a/container-disc/src/main/java/com/yahoo/container/jdisc/DataplaneProxyService.java +++ b/container-disc/src/main/java/com/yahoo/container/jdisc/DataplaneProxyService.java @@ -109,6 +109,10 @@ public class DataplaneProxyService extends AbstractComponent { config.tokenPort(), config.tokenEndpoints(), root)); + if (configChanged) { + logger.log(Level.INFO, "Configuring data plane proxy service. Token endpoints: [%s]" + .formatted(String.join(", ", config.tokenEndpoints()))); + } if (configChanged && state == NginxState.RUNNING) { changeState(NginxState.RELOAD_REQUIRED); } diff --git a/container-search/src/test/java/com/yahoo/search/result/CoverageTestCase.java b/container-search/src/test/java/com/yahoo/search/result/CoverageTestCase.java index 2d1dac64302..cadbc6adf5c 100644 --- a/container-search/src/test/java/com/yahoo/search/result/CoverageTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/result/CoverageTestCase.java @@ -6,7 +6,9 @@ import com.yahoo.search.Result; import org.junit.jupiter.api.Test; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; /** * @author Steinar Knutsen @@ -103,4 +105,22 @@ public class CoverageTestCase { } } + private void verifyNoCoverage(Coverage zero) { + assertFalse(zero.isDegraded()); + assertEquals(100, zero.getResultPercentage()); + assertTrue(zero.getFull()); + zero.setDegradedReason(com.yahoo.container.handler.Coverage.DEGRADED_BY_TIMEOUT); + assertTrue(zero.isDegraded()); + assertEquals(0, zero.getResultPercentage()); + assertFalse(zero.getFull()); + } + @Test + void testCoverageWithNoResponseFromSearchNodesAndTimeout() { + verifyNoCoverage(new Coverage(0, 0, 0)); + } + @Test + void testCoverageWithResponseFromSearchNodesAndTimeout() { + verifyNoCoverage(new Coverage(0, 0, 1)); + } + } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java index f9798fb2559..f11d67762ad 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java @@ -64,6 +64,8 @@ import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.TreeMap; +import java.util.logging.Level; +import java.util.logging.Logger; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -79,6 +81,8 @@ import static java.util.stream.Collectors.toMap; */ public class RoutingController { + private static final Logger LOG = Logger.getLogger(RoutingController.class.getName()); + private final Controller controller; private final RoutingPolicies routingPolicies; private final RotationRepository rotationRepository; @@ -213,6 +217,8 @@ public class RoutingController { // Register rotation-backed endpoints in DNS registerRotationEndpointsInDns(prepared); + LOG.log(Level.FINE, () -> "Prepared endpoints: " + prepared); + return prepared; } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/AssignedRotation.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/AssignedRotation.java index 7fa2a03d0a9..c2949e395e9 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/AssignedRotation.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/AssignedRotation.java @@ -23,16 +23,6 @@ public record AssignedRotation(ClusterSpec.Id clusterId, EndpointId endpointId, this.regions = Set.copyOf(Objects.requireNonNull(regions)); } - @Override - public String toString() { - return "AssignedRotation{" + - "clusterId=" + clusterId + - ", endpointId='" + endpointId + '\'' + - ", rotationId=" + rotationId + - ", regions=" + regions + - '}'; - } - private static <T> T requireNonEmpty(T object, String value, String field) { Objects.requireNonNull(object); Objects.requireNonNull(value); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/BcpGroupUpdater.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/BcpGroupUpdater.java index 1327bfb09b2..1ad4feb1897 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/BcpGroupUpdater.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/BcpGroupUpdater.java @@ -84,7 +84,7 @@ public class BcpGroupUpdater extends ControllerMaintainer { double successFactorDeviation = asSuccessFactorDeviation(attempts, failures); if ( successFactorDeviation == -successFactorBaseline ) log.log(Level.WARNING, "Could not update traffic share on any applications", lastException); - else if ( successFactorDeviation < -0.1 ) + else if ( successFactorDeviation < 0 ) log.log(Level.FINE, "Could not update traffic share on all applications", lastException); return successFactorDeviation; } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandlerV2.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandlerV2.java index 83cd5dab2f3..5b43ede2e22 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandlerV2.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandlerV2.java @@ -4,7 +4,6 @@ package com.yahoo.vespa.hosted.controller.restapi.billing; import com.yahoo.config.provision.TenantName; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.container.jdisc.ThreadedHttpRequestHandler; -import com.yahoo.messagebus.Message; import com.yahoo.restapi.MessageResponse; import com.yahoo.restapi.RestApi; import com.yahoo.restapi.RestApiException; @@ -461,6 +460,7 @@ public class BillingApiHandlerV2 extends RestApiRequestHandler<BillingApiHandler toSlime(slime.setObject("cpu"), item.getCpuHours(), item.getCpuCost()); toSlime(slime.setObject("memory"), item.getMemoryHours(), item.getMemoryCost()); toSlime(slime.setObject("disk"), item.getDiskHours(), item.getDiskCost()); + toSlime(slime.setObject("gpu"), item.getGpuHours(), item.getGpuCost()); } private void toSlime(Cursor slime, Optional<BigDecimal> hours, Optional<BigDecimal> cost) { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandlerV2Test.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandlerV2Test.java index 424b8d84472..ea5729d8fde 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandlerV2Test.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandlerV2Test.java @@ -103,7 +103,7 @@ public class BillingApiHandlerV2Test extends ControllerContainerCloudTest { var singleRequest = request("/billing/v2/tenant/" + tenant + "/bill/id-1").roles(tenantReader); tester.assertResponse(singleRequest, """ - {"id":"id-1","from":"2020-05-23","to":"2020-05-28","total":"123.00","status":"OPEN","statusHistory":[{"at":"2020-05-23T00:00:00Z","status":"OPEN"}],"items":[{"id":"some-id","description":"description","amount":"123.00","plan":{"id":"paid","name":"Paid Plan - for testing purposes"},"majorVersion":0,"cpu":{},"memory":{},"disk":{}}]}"""); + {"id":"id-1","from":"2020-05-23","to":"2020-05-28","total":"123.00","status":"OPEN","statusHistory":[{"at":"2020-05-23T00:00:00Z","status":"OPEN"}],"items":[{"id":"some-id","description":"description","amount":"123.00","plan":{"id":"paid","name":"Paid Plan - for testing purposes"},"majorVersion":0,"cpu":{},"memory":{},"disk":{},"gpu":{}}]}"""); } @Test @@ -175,7 +175,7 @@ public class BillingApiHandlerV2Test extends ControllerContainerCloudTest { var accountantRequest = request("/billing/v2/accountant/tenant/tenant1/items") .roles(Role.hostedAccountant()); tester.assertResponse(accountantRequest, """ - {"items":[{"id":"line-item-id","description":"Additional support costs","amount":"123.45","plan":{"id":"paid","name":"Paid Plan - for testing purposes"},"majorVersion":0,"cpu":{},"memory":{},"disk":{}}]}"""); + {"items":[{"id":"line-item-id","description":"Additional support costs","amount":"123.45","plan":{"id":"paid","name":"Paid Plan - for testing purposes"},"majorVersion":0,"cpu":{},"memory":{},"disk":{},"gpu":{}}]}"""); } { diff --git a/dist/vespa.spec b/dist/vespa.spec index ce302ae0338..cd1381fd48d 100644 --- a/dist/vespa.spec +++ b/dist/vespa.spec @@ -280,29 +280,6 @@ Requires: %{name}-base-libs = %{version}-%{release} Vespa - The open big data serving engine - devel package -%package ann-benchmark - -Summary: Vespa - The open big data serving engine - ann-benchmark - -Requires: %{name}-base-libs = %{version}-%{release} -Requires: %{name}-libs = %{version}-%{release} -%if 0%{?el8} -Requires: python39 -%endif -%if 0%{?el9} -Requires: python3 -%endif -%if 0%{?fedora} -Requires: python3 -%endif - -%description ann-benchmark - -Vespa - The open big data serving engine - ann-benchmark - -Python binding for the Vespa implementation of an HNSW index for -nearest neighbor search used for low-level benchmarking. - %prep %if 0%{?installdir:1} %if 0%{?source_base:1} @@ -383,10 +360,6 @@ export JAVA_HOME=%{?_java_home} export JAVA_HOME=/usr/lib/jvm/java-17-openjdk %endif export PATH="$JAVA_HOME/bin:$PATH" -%if 0%{?el8} -python3.9 -m pip install --user pytest -%endif -export PYTHONPATH="$PYTHONPATH:/usr/local/lib/$(basename $(readlink -f $(which python3)))/site-packages" #%{?_use_mvn_wrapper:./mvnw}%{!?_use_mvn_wrapper:mvn} --batch-mode -nsu -T 1C -Dmaven.javadoc.skip=true test make test ARGS="--output-on-failure %{_smp_mflags}" %endif @@ -537,7 +510,6 @@ fi %{_prefix}/lib/jars/zookeeper-command-line-client-jar-with-dependencies.jar %{_prefix}/lib/perl5 %{_prefix}/libexec -%exclude %{_prefix}/libexec/vespa_ann_benchmark %exclude %{_prefix}/libexec/vespa/common-env.sh %exclude %{_prefix}/libexec/vespa/vespa-wrapper %exclude %{_prefix}/libexec/vespa/find-pid @@ -763,12 +735,4 @@ fi %{_prefix}/include %{_prefix}/share/cmake -%files ann-benchmark -%if %{_defattr_is_vespa_vespa} -%defattr(-,%{_vespa_user},%{_vespa_group},-) -%endif -%dir %{_prefix} -%dir %{_prefix}/libexec -%{_prefix}/libexec/vespa_ann_benchmark - %changelog diff --git a/document/src/vespa/document/bucket/bucketid.cpp b/document/src/vespa/document/bucket/bucketid.cpp index dee818b407e..fcc9a0df4f6 100644 --- a/document/src/vespa/document/bucket/bucketid.cpp +++ b/document/src/vespa/document/bucket/bucketid.cpp @@ -8,7 +8,6 @@ #include <vespa/vespalib/stllike/hash_set.hpp> #include <vespa/vespalib/util/stringfmt.h> #include <limits> -#include <xxhash.h> using vespalib::nbostream; using vespalib::asciistream; @@ -79,29 +78,7 @@ void BucketId::initialize() noexcept { uint64_t BucketId::hash::operator () (const BucketId& bucketId) const noexcept { - const uint64_t raw_id = bucketId.getId(); - /* - * This is a workaround for gcc 12 and on that produces incorrect warning when compiled with -march=haswell or newer - * /home/balder/git/vespa/document/src/vespa/document/bucket/bucketid.cpp: In member function ‘uint64_t document::BucketId::hash::operator()(const document::BucketId&) const’: - * /home/balder/git/vespa/document/src/vespa/document/bucket/bucketid.cpp:83:23: error: ‘raw_id’ may be used uninitialized [-Werror=maybe-uninitialized] - * 83 | return XXH3_64bits(&raw_id, sizeof(uint64_t)); - * | ^ - * In file included from /usr/include/xxh3.h:55, - * from /home/balder/git/vespa/document/src/vespa/document/bucket/bucketid.cpp:11: - * /usr/include/xxhash.h:5719:29: note: by argument 1 of type ‘const void*’ to ‘XXH64_hash_t XXH_INLINE_XXH3_64bits(const void*, size_t)’ declared here - * 5719 | XXH_PUBLIC_API XXH64_hash_t XXH3_64bits(XXH_NOESCAPE const void* input, size_t length) - * | ^~~~~~~~~~~ - * /home/balder/git/vespa/document/src/vespa/document/bucket/bucketid.cpp:82:14: note: ‘raw_id’ declared here - * 82 | uint64_t raw_id = bucketId.getId(); - * | ^~~~~~ - * cc1plus: all warnings being treated as errors - * - * Same issue in storage/src/vespa/storage/persistence/filestorhandlerimpl.cpp:FileStorHandlerImpl::dispersed_bucket_bits - */ - uint8_t raw_as_bytes[sizeof(raw_id)]; - memcpy(raw_as_bytes, &raw_id, sizeof(raw_id)); - - return XXH3_64bits(raw_as_bytes, sizeof(raw_as_bytes)); + return vespalib::xxhash::xxh3_64(bucketId.getId()); } vespalib::string diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index c2f6f782dbc..d02cccb2885 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -309,12 +309,6 @@ public class Flags { "Whether to enable CrowdStrike.", "Takes effect on next host admin tick", HOSTNAME); - public static final UnboundBooleanFlag RANDOMIZED_ENDPOINT_NAMES = defineFeatureFlag( - "randomized-endpoint-names", false, List.of("andreer"), "2023-04-26", "2023-11-14", - "Whether to use randomized endpoint names", - "Takes effect on application deployment", - INSTANCE_ID, APPLICATION_ID, TENANT_ID); - public static final UnboundBooleanFlag ENABLE_THE_ONE_THAT_SHOULD_NOT_BE_NAMED = defineFeatureFlag( "enable-the-one-that-should-not-be-named", false, List.of("hmusum"), "2023-05-08", "2023-11-01", "Whether to enable the one program that should not be named", @@ -387,20 +381,6 @@ public class Flags { "Takes effect immediately", INSTANCE_ID, CLUSTER_ID, CLUSTER_TYPE); - public static final UnboundBooleanFlag ASSIGN_RANDOMIZED_ID = defineFeatureFlag( - "assign-randomized-id", true, - List.of("mortent"), "2023-08-31", "2024-02-01", - "Whether to assign randomized id to the application", - "Takes effect immediately", - INSTANCE_ID); - - public static final UnboundIntFlag ASSIGNED_RANDOMIZED_ID_RATE = defineIntFlag( - "assign-randomized-id-rate", 5, - List.of("mortent"), "2023-09-11", "2024-02-01", - "Rate for requesting assigned ids for existing certificates. Rate is per maintainer cycle.", - "Takes effect immediately", - INSTANCE_ID); - public static final UnboundIntFlag CONTENT_LAYER_METADATA_FEATURE_LEVEL = defineIntFlag( "content-layer-metadata-feature-level", 0, List.of("vekterli"), "2022-09-12", "2024-02-01", @@ -423,12 +403,6 @@ public class Flags { "Takes effect at redeployment", INSTANCE_ID); - public static final UnboundBooleanFlag LEGACY_ENDPOINTS = defineFeatureFlag( - "legacy-endpoints", true, List.of("mpolden", "tokle"), "2023-09-29", "2024-03-01", - "Whether legacy (non-anonymized) endpoints should be created in DNS", - "Takes effect on redeployment through controller", - INSTANCE_ID, APPLICATION_ID, TENANT_ID); - public static final UnboundIntFlag SEARCH_HANDLER_THREADPOOL = defineIntFlag( "search-handler-threadpool", 2, List.of("bjorncs", "baldersheim"), "2023-10-01", "2024-01-01", diff --git a/hosted-tenant-base/pom.xml b/hosted-tenant-base/pom.xml index 729150bcef1..232ecd0e3e5 100644 --- a/hosted-tenant-base/pom.xml +++ b/hosted-tenant-base/pom.xml @@ -284,6 +284,9 @@ <exclude>org.junit.platform:junit-platform-engine:(${junit.platform.vespa.tenant.version},):jar:*</exclude> <exclude>org.junit.platform:junit-platform-commons:(,${junit.platform.vespa.tenant.version}):jar:*</exclude> <exclude>org.junit.platform:junit-platform-commons:(${junit.platform.vespa.tenant.version},):jar:*</exclude> + + <!-- Ban testng as its presence breaks the Java based test framework for Cloud --> + <exclude>org.testng:testng:*:jar:test</exclude> </excludes> </bannedDependencies> </rules> diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java index c3ec9e91343..43dc3d72c46 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java @@ -518,6 +518,8 @@ public class NodeAgentImpl implements NodeAgent { container = Optional.of(updateContainerIfNeeded(context, container.get())); } + serviceDumper.processServiceDumpRequest(context); + startServicesIfNeeded(context); resumeNodeIfNeeded(context); if (healthChecker.isPresent()) { @@ -531,7 +533,6 @@ public class NodeAgentImpl implements NodeAgent { throw ConvergenceException.ofTransient("Refusing to resume until warm up period ends (" + (timeLeft.isNegative() ? "next tick" : "in " + timeLeft) + ")"); } - serviceDumper.processServiceDumpRequest(context); // Because it's more important to stop a bad release from rolling out in prod, // we put the resume call last. So if we fail after updating the node repo attributes diff --git a/searchcore/src/tests/proton/common/timer/timer_test.cpp b/searchcore/src/tests/proton/common/timer/timer_test.cpp index 1efae97c6e0..4419275230a 100644 --- a/searchcore/src/tests/proton/common/timer/timer_test.cpp +++ b/searchcore/src/tests/proton/common/timer/timer_test.cpp @@ -84,7 +84,7 @@ TYPED_TEST(ScheduledExecutorTest, test_drop_handle) { } TYPED_TEST(ScheduledExecutorTest, test_only_one_instance_running) { - vespalib::TimeBomb time_bomb(60s); + vespalib::TimeBomb time_bomb(120s); vespalib::Gate latch; std::atomic<uint64_t> counter = 0; auto handleA = this->timer->scheduleAtFixedRate(makeLambdaTask([&]() { counter++; latch.await();}), 0ms, 1ms); @@ -96,7 +96,7 @@ TYPED_TEST(ScheduledExecutorTest, test_only_one_instance_running) { } TYPED_TEST(ScheduledExecutorTest, test_sync_delete) { - vespalib::TimeBomb time_bomb(60s); + vespalib::TimeBomb time_bomb(120s); vespalib::Gate latch; std::atomic<uint64_t> counter = 0; std::atomic<uint64_t> reset_counter = 0; diff --git a/security-utils/src/main/java/com/yahoo/security/X509CertificateUtils.java b/security-utils/src/main/java/com/yahoo/security/X509CertificateUtils.java index 9bcc6e7b8c6..171a8e890d0 100644 --- a/security-utils/src/main/java/com/yahoo/security/X509CertificateUtils.java +++ b/security-utils/src/main/java/com/yahoo/security/X509CertificateUtils.java @@ -4,6 +4,7 @@ package com.yahoo.security; import org.bouncycastle.asn1.ASN1Encodable; import org.bouncycastle.asn1.ASN1OctetString; import org.bouncycastle.asn1.ASN1Primitive; +import org.bouncycastle.asn1.pkcs.PrivateKeyInfo; import org.bouncycastle.asn1.x509.GeneralNames; import org.bouncycastle.cert.X509CertificateHolder; import org.bouncycastle.cert.jcajce.JcaX509CertificateConverter; @@ -73,15 +74,18 @@ public class X509CertificateUtils { } private static X509Certificate toX509Certificate(Object pemObject) throws CertificateException { - if (pemObject instanceof X509Certificate) { - return (X509Certificate) pemObject; + if (pemObject instanceof X509Certificate certificate) { + return certificate; } - if (pemObject instanceof X509CertificateHolder) { + if (pemObject instanceof X509CertificateHolder certificateHolder) { return new JcaX509CertificateConverter() .setProvider(BouncyCastleProviderHolder.getInstance()) - .getCertificate((X509CertificateHolder) pemObject); + .getCertificate(certificateHolder); } - throw new IllegalArgumentException("Invalid type of PEM object: " + pemObject); + if (pemObject instanceof PrivateKeyInfo) { + throw new IllegalArgumentException("Expected X509 certificate, but got private key"); + } + throw new IllegalArgumentException("Invalid type of PEM object, got " + pemObject.getClass().getName()); } public static String toPem(X509Certificate certificate) { diff --git a/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.cpp b/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.cpp index 582b67a4dbc..a96bee2d11a 100644 --- a/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.cpp +++ b/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.cpp @@ -14,7 +14,6 @@ #include <vespa/vespalib/stllike/hash_map.hpp> #include <vespa/vespalib/util/exceptions.h> #include <vespa/vespalib/util/string_escape.h> -#include <xxhash.h> #include <vespa/log/log.h> LOG_SETUP(".persistence.filestor.handler.impl"); @@ -896,14 +895,7 @@ FileStorHandlerImpl::flush() uint64_t FileStorHandlerImpl::dispersed_bucket_bits(const document::Bucket& bucket) noexcept { - const uint64_t raw_id = bucket.getBucketId().getId(); - /* - * This is a workaround for gcc 12 and on that produces incorrect warning when compiled with -march=haswell or newer - * See document/src/vespa/document/bucket/bucketid.cpp: In member function ‘uint64_t document::BucketId::hash::operator()(const document::BucketId&) const - */ - uint8_t raw_as_bytes[sizeof(raw_id)]; - memcpy(raw_as_bytes, &raw_id, sizeof(raw_id)); - return XXH3_64bits(&raw_as_bytes, sizeof(raw_id)); + return vespalib::xxhash::xxh3_64(bucket.getBucketId().getId()); } FileStorHandlerImpl::Stripe::Stripe(const FileStorHandlerImpl & owner, MessageSender & messageSender) diff --git a/vespalib/src/vespa/vespalib/fuzzy/sparse_state.h b/vespalib/src/vespa/vespalib/fuzzy/sparse_state.h index 7e381468fbe..e023f4d3de2 100644 --- a/vespalib/src/vespa/vespalib/fuzzy/sparse_state.h +++ b/vespalib/src/vespa/vespalib/fuzzy/sparse_state.h @@ -2,13 +2,12 @@ #pragma once #include <vespa/config.h> +#include <vespa/vespalib/stllike/hash_fun.h> #include <algorithm> #include <array> #include <cassert> -#include <cstdint> #include <ostream> #include <span> -#include <xxhash.h> // TODO factor out? namespace vespalib::fuzzy { @@ -80,15 +79,8 @@ public: size_t operator()(const FixedSparseState& s) const noexcept { static_assert(std::is_same_v<uint32_t, std::decay_t<decltype(s.indices[0])>>); static_assert(std::is_same_v<uint8_t, std::decay_t<decltype(s.costs[0])>>); - // FIXME GCC 12.2 worse-than-useless(tm) warning false positives :I -#pragma GCC diagnostic push -#ifdef VESPA_USE_SANITIZER -# pragma GCC diagnostic ignored "-Wstringop-overread" // https://gcc.gnu.org/bugzilla/show_bug.cgi?id=98465 etc. -#endif -#pragma GCC diagnostic ignored "-Warray-bounds" - return (XXH3_64bits(s.indices.data(), s.sz * sizeof(uint32_t)) ^ - XXH3_64bits(s.costs.data(), s.sz)); -#pragma GCC diagnostic pop + return (xxhash::xxh3_64(s.indices.data(), s.sz * sizeof(uint32_t)) ^ + xxhash::xxh3_64(s.costs.data(), s.sz)); } }; }; diff --git a/vespalib/src/vespa/vespalib/stllike/hash_fun.cpp b/vespalib/src/vespa/vespalib/stllike/hash_fun.cpp index e4911d2e0c6..6802e5f91a6 100644 --- a/vespalib/src/vespa/vespalib/stllike/hash_fun.cpp +++ b/vespalib/src/vespa/vespalib/stllike/hash_fun.cpp @@ -6,23 +6,21 @@ namespace vespalib { size_t -hashValue(const char *str) noexcept -{ - return hashValue(str, strlen(str)); +hashValue(const char *str) noexcept { + return xxhash::xxh3_64(str, strlen(str)); } -/** - * @brief Calculate hash value. - * - * The hash function XXH3_64bits from xxhash library. - * @param buf input buffer - * @param sz input buffer size - * @return hash value of input - **/ -size_t -hashValue(const void * buf, size_t sz) noexcept -{ +namespace xxhash { + +uint64_t +xxh3_64(uint64_t value) noexcept { + return XXH3_64bits(&value, sizeof(value)); +} + +uint64_t +xxh3_64(const void * buf, size_t sz) noexcept { return XXH3_64bits(buf, sz); } } +} diff --git a/vespalib/src/vespa/vespalib/stllike/hash_fun.h b/vespalib/src/vespa/vespalib/stllike/hash_fun.h index f8a8a06b921..8fecc41b4c1 100644 --- a/vespalib/src/vespa/vespalib/stllike/hash_fun.h +++ b/vespalib/src/vespa/vespalib/stllike/hash_fun.h @@ -2,7 +2,6 @@ #pragma once #include <vespa/vespalib/stllike/string.h> -#include <cstdint> namespace vespalib { @@ -64,9 +63,18 @@ template<typename T> struct hash<const T *> { size_t operator() (const T * arg) const noexcept { return size_t(arg); } }; +namespace xxhash { + +uint64_t xxh3_64(uint64_t value) noexcept; +uint64_t xxh3_64(const void *str, size_t sz) noexcept; + +} + // reuse old string hash function size_t hashValue(const char *str) noexcept; -size_t hashValue(const void *str, size_t sz) noexcept; +inline size_t hashValue(const void *buf, size_t sz) noexcept { + return xxhash::xxh3_64(buf, sz); +} struct hash_strings { size_t operator() (const vespalib::string & arg) const noexcept { return hashValue(arg.data(), arg.size()); } diff --git a/vespalib/src/vespa/vespalib/util/shared_string_repo.cpp b/vespalib/src/vespa/vespalib/util/shared_string_repo.cpp index a5942e1af9b..09f2bbd828d 100644 --- a/vespalib/src/vespa/vespalib/util/shared_string_repo.cpp +++ b/vespalib/src/vespa/vespalib/util/shared_string_repo.cpp @@ -1,7 +1,7 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "shared_string_repo.h" -#include <xxhash.h> +#include <vespa/vespalib/stllike/hash_fun.h> #include <charconv> #include <cassert> @@ -232,7 +232,7 @@ string_id SharedStringRepo::resolve(vespalib::stringref str) { uint32_t direct_id = try_make_direct_id(str); if (direct_id >= ID_BIAS) { - uint64_t full_hash = XXH3_64bits(str.data(), str.size()); + uint64_t full_hash = xxhash::xxh3_64(str.data(), str.size()); uint32_t part = full_hash & PART_MASK; uint32_t local_hash = full_hash >> PART_BITS; uint32_t local_idx = _partitions[part].resolve(AltKey{str, local_hash}); |