diff options
507 files changed, 12928 insertions, 5210 deletions
diff --git a/CMakeLists.txt b/CMakeLists.txt index a4e89cbdeb9..1bdb3a23730 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -51,9 +51,6 @@ find_package(JNI REQUIRED) find_package(GTest REQUIRED) -find_package(Python 3.6 COMPONENTS Interpreter Development REQUIRED) -find_package(pybind11 CONFIG REQUIRED) - find_package(Protobuf REQUIRED) include(build_settings.cmake) @@ -100,7 +97,6 @@ vespa_install_data(tsan-suppressions.txt etc/vespa) include_directories(BEFORE ${CMAKE_BINARY_DIR}/configdefinitions/src) add_subdirectory(airlift-zstd) -add_subdirectory(ann_benchmark) add_subdirectory(application-model) add_subdirectory(client) add_subdirectory(cloud-tenant-cd) diff --git a/ann_benchmark/CMakeLists.txt b/ann_benchmark/CMakeLists.txt deleted file mode 100644 index 06d742cf072..00000000000 --- a/ann_benchmark/CMakeLists.txt +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_define_module( - DEPENDS - searchlib - - LIBS - src/vespa/ann_benchmark - - APPS - - TESTS - src/tests/ann_benchmark -) diff --git a/ann_benchmark/src/tests/ann_benchmark/.gitignore b/ann_benchmark/src/tests/ann_benchmark/.gitignore deleted file mode 100644 index 225fc6f6650..00000000000 --- a/ann_benchmark/src/tests/ann_benchmark/.gitignore +++ /dev/null @@ -1 +0,0 @@ -/__pycache__ diff --git a/ann_benchmark/src/tests/ann_benchmark/CMakeLists.txt b/ann_benchmark/src/tests/ann_benchmark/CMakeLists.txt deleted file mode 100644 index 03126ce1b47..00000000000 --- a/ann_benchmark/src/tests/ann_benchmark/CMakeLists.txt +++ /dev/null @@ -1,5 +0,0 @@ -# Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -if(NOT DEFINED VESPA_USE_SANITIZER) - vespa_add_test(NAME ann_benchmark_test NO_VALGRIND COMMAND ${Python_EXECUTABLE} -m pytest WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} DEPENDS vespa_ann_benchmark) -endif() diff --git a/ann_benchmark/src/tests/ann_benchmark/test_angular.py b/ann_benchmark/src/tests/ann_benchmark/test_angular.py deleted file mode 100644 index ac7feb29d76..00000000000 --- a/ann_benchmark/src/tests/ann_benchmark/test_angular.py +++ /dev/null @@ -1,41 +0,0 @@ -# Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -import pytest -import sys -import os -import math -sys.path.insert(0, os.path.abspath("../../vespa/ann_benchmark")) -from vespa_ann_benchmark import DistanceMetric, HnswIndexParams, HnswIndex - -class Fixture: - def __init__(self, normalize): - metric = DistanceMetric.InnerProduct if normalize else DistanceMetric.Angular - self.index = HnswIndex(2, HnswIndexParams(16, 200, metric, False), normalize) - self.index.set_vector(0, [1, 0]) - self.index.set_vector(1, [10, 10]) - - def find(self, k, value): - return self.index.find_top_k(k, value, k + 200) - - def run_test(self): - top = self.find(10, [1, 1]) - assert [top[0][0], top[1][0]] == [0, 1] - # Allow some rounding errors - epsilon = 6e-8 - assert abs((1 - top[0][1]) - math.sqrt(0.5)) < epsilon - assert abs((1 - top[1][1]) - 1) < epsilon - top2 = self.find(10, [0, 2]) - # Result is not sorted by distance - assert [top2[0][0], top2[1][0]] == [0, 1] - assert abs((1 - top2[0][1]) - 0) < epsilon - assert abs((1 - top2[1][1]) - math.sqrt(0.5)) < epsilon - assert 1 == self.find(1, [1, 1])[0][0] - assert 0 == self.find(1, [1, -1])[0][0] - -def test_find_angular(): - f = Fixture(False) - f.run_test() - -def test_find_angular_normalized(): - f = Fixture(True) - f.run_test() diff --git a/ann_benchmark/src/tests/ann_benchmark/test_euclidean.py b/ann_benchmark/src/tests/ann_benchmark/test_euclidean.py deleted file mode 100644 index ca4d5ecd6a1..00000000000 --- a/ann_benchmark/src/tests/ann_benchmark/test_euclidean.py +++ /dev/null @@ -1,61 +0,0 @@ -# Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -import pytest -import sys -import os -import math -sys.path.insert(0, os.path.abspath("../../vespa/ann_benchmark")) -from vespa_ann_benchmark import DistanceMetric, HnswIndexParams, HnswIndex - -class Fixture: - def __init__(self): - self.index = HnswIndex(2, HnswIndexParams(16, 200, DistanceMetric.Euclidean, False), False) - - def set(self, lid, value): - self.index.set_vector(lid, value) - - def get(self, lid): - return self.index.get_vector(lid) - - def clear(self, lid): - return self.index.clear_vector(lid) - - def find(self, k, value): - return self.index.find_top_k(k, value, k + 200) - -def test_set_value(): - f = Fixture() - f.set(0, [1, 2]) - f.set(1, [3, 4]) - assert [1, 2] == f.get(0) - assert [3, 4] == f.get(1) - -def test_clear_value(): - f = Fixture() - f.set(0, [1, 2]) - assert [1, 2] == f.get(0) - f.clear(0) - assert [0, 0] == f.get(0) - -def test_find(): - f = Fixture() - f.set(0, [0, 0]) - f.set(1, [10, 10]) - top = f.find(10, [1, 1]) - assert [top[0][0], top[1][0]] == [0, 1] - # Allow some rounding errors - epsilon = 1e-20 - assert abs(top[0][1] - math.sqrt(2)) < epsilon - assert abs(top[1][1] - math.sqrt(162)) < epsilon - top2 = f.find(10, [9, 9]) - # Result is not sorted by distance - assert [top2[0][0], top2[1][0]] == [0, 1] - assert abs(top2[0][1] - math.sqrt(162)) < epsilon - assert abs(top2[1][1] - math.sqrt(2)) < epsilon - assert 0 == f.find(1, [1, 1])[0][0] - assert 1 == f.find(1, [9, 9])[0][0] - f.clear(1) - assert 0 == f.find(1, [9, 9])[0][0] - assert 0 == f.find(1, [0, 0])[0][0] - f.clear(0) - assert 0 == len(f.find(1, [9, 9])) diff --git a/ann_benchmark/src/vespa/ann_benchmark/.gitignore b/ann_benchmark/src/vespa/ann_benchmark/.gitignore deleted file mode 100644 index 3b4605aeee2..00000000000 --- a/ann_benchmark/src/vespa/ann_benchmark/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -/vespa_ann_benchmark.cpython*.so -/setup.py diff --git a/ann_benchmark/src/vespa/ann_benchmark/CMakeLists.txt b/ann_benchmark/src/vespa/ann_benchmark/CMakeLists.txt deleted file mode 100644 index fa3b10b2269..00000000000 --- a/ann_benchmark/src/vespa/ann_benchmark/CMakeLists.txt +++ /dev/null @@ -1,31 +0,0 @@ -# Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -install(DIRECTORY DESTINATION libexec/vespa_ann_benchmark) - -vespa_add_library(vespa_ann_benchmark - ALLOW_UNRESOLVED_SYMBOLS - SOURCES - vespa_ann_benchmark.cpp - - INSTALL libexec/vespa_ann_benchmark - DEPENDS - pybind11::pybind11 -) - -if (TARGET pybind11::lto) - target_link_libraries(vespa_ann_benchmark PRIVATE pybind11::module pybind11::lto) -else() - target_link_libraries(vespa_ann_benchmark PRIVATE pybind11::module) -endif() - -if (COMMAND pybind11_extension) - pybind11_extension(vespa_ann_benchmark) -else() - set_target_properties(vespa_ann_benchmark PROPERTIES PREFIX "${PYTHON_MODULE_PREFIX}") - set_target_properties(vespa_ann_benchmark PROPERTIES SUFFIX "${PYTHON_MODULE_EXTENSION}") -endif() - -set_target_properties(vespa_ann_benchmark PROPERTIES CXX_VISIBILITY_PRESET "hidden") - -configure_file(setup.py.in setup.py @ONLY) - -vespa_install_script(setup.py libexec/vespa_ann_benchmark) diff --git a/ann_benchmark/src/vespa/ann_benchmark/setup.py.in b/ann_benchmark/src/vespa/ann_benchmark/setup.py.in deleted file mode 100644 index 457d6e1b4b5..00000000000 --- a/ann_benchmark/src/vespa/ann_benchmark/setup.py.in +++ /dev/null @@ -1,27 +0,0 @@ -# Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -import subprocess -import sys -import platform -import distutils.sysconfig -from setuptools import setup, Extension -from setuptools.command.build_ext import build_ext - -class PreBuiltExt(build_ext): - def build_extension(self, ext): - print("Using prebuilt extension library") - libdir="lib.%s-%s-%s" % (sys.platform, platform.machine(), distutils.sysconfig.get_python_version()) - subprocess.run(["mkdir", "-p", "build/%s" % libdir]) - subprocess.run(["cp", "-p", "@PYTHON_MODULE_PREFIX@vespa_ann_benchmark@PYTHON_MODULE_EXTENSION@", "build/%s" % libdir]) - -setup( - name="vespa_ann_benchmark", - version="0.1.0", - author="Tor Egge", - author_email="Tor.Egge@yahooinc.com", - description="Python binding for the Vespa implementation of an HNSW index for nearest neighbor search", - long_description="Python binding for the Vespa implementation of an HNSW index for nearest neighbor search used for low-level benchmarking", - ext_modules=[Extension("vespa_ann_benchmark", sources=[])], - cmdclass={"build_ext": PreBuiltExt}, - zip_safe=False, -) diff --git a/ann_benchmark/src/vespa/ann_benchmark/vespa_ann_benchmark.cpp b/ann_benchmark/src/vespa/ann_benchmark/vespa_ann_benchmark.cpp deleted file mode 100644 index ab00f997226..00000000000 --- a/ann_benchmark/src/vespa/ann_benchmark/vespa_ann_benchmark.cpp +++ /dev/null @@ -1,252 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include <pybind11/pybind11.h> -#include <pybind11/stl.h> -#include <vespa/searchcommon/attribute/hnsw_index_params.h> -#include <vespa/searchlib/attribute/attributevector.h> -#include <vespa/searchlib/attribute/attributefactory.h> -#include <vespa/searchlib/tensor/dense_tensor_attribute.h> -#include <vespa/searchlib/tensor/nearest_neighbor_index.h> -#include <vespa/searchcommon/attribute/config.h> -#include <vespa/eval/eval/value.h> -#include <vespa/vespalib/test/insertion_operators.h> -#include <vespa/vespalib/util/fake_doom.h> -#include <iostream> -#include <sstream> -#include <limits> - -namespace py = pybind11; - -using search::AttributeFactory; -using search::AttributeVector; -using search::attribute::BasicType; -using search::attribute::Config; -using search::attribute::CollectionType; -using search::attribute::DistanceMetric; -using search::attribute::HnswIndexParams; -using search::tensor::NearestNeighborIndex; -using search::tensor::TensorAttribute; -using vespalib::eval::CellType; -using vespalib::eval::DenseValueView; -using vespalib::eval::TypedCells; -using vespalib::eval::ValueType; -using vespalib::eval::Value; - -namespace vespa_ann_benchmark { - -using TopKResult = std::vector<std::pair<uint32_t, double>>; - -namespace { - -std::string -make_tensor_spec(uint32_t dim_size) -{ - std::ostringstream os; - os << "tensor<float>(x[" << dim_size << "])"; - return os.str(); -} - -constexpr uint32_t lid_bias = 1; // lid 0 is reserved - -} - -/* - * Class exposing the Vespa implementation of an HNSW index for nearest neighbor search over data points in a high dimensional vector space. - * - * A tensor attribute field (https://docs.vespa.ai/en/reference/schema-reference.html#type:tensor) is used to store the vectors in memory. - * This class only supports single-threaded access (both for indexing and searching), - * and should only be used for low-level benchmarking. - * To use nearest neighbor search in a Vespa application, - * see https://docs.vespa.ai/en/approximate-nn-hnsw.html for more details. - */ -class HnswIndex -{ - ValueType _tensor_type; - HnswIndexParams _hnsw_index_params; - std::shared_ptr<AttributeVector> _attribute; - TensorAttribute* _tensor_attribute; - const NearestNeighborIndex* _nearest_neighbor_index; - size_t _dim_size; - bool _normalize_vectors; - vespalib::FakeDoom _no_doom; - - bool check_lid(uint32_t lid); - bool check_value(const char *op, const std::vector<float>& value); - TypedCells get_typed_cells(const std::vector<float>& value, std::vector<float>& normalized_value); -public: - HnswIndex(uint32_t dim_size, const HnswIndexParams &hnsw_index_params, bool normalize_vectors); - virtual ~HnswIndex(); - void set_vector(uint32_t lid, const std::vector<float>& value); - std::vector<float> get_vector(uint32_t lid); - void clear_vector(uint32_t lid); - TopKResult find_top_k(uint32_t k, const std::vector<float>& value, uint32_t explore_k); -}; - -HnswIndex::HnswIndex(uint32_t dim_size, const HnswIndexParams &hnsw_index_params, bool normalize_vectors) - : _tensor_type(ValueType::error_type()), - _hnsw_index_params(hnsw_index_params), - _attribute(), - _tensor_attribute(nullptr), - _nearest_neighbor_index(nullptr), - _dim_size(0u), - _normalize_vectors(normalize_vectors), - _no_doom() -{ - Config cfg(BasicType::TENSOR, CollectionType::SINGLE); - _tensor_type = ValueType::from_spec(make_tensor_spec(dim_size)); - assert(_tensor_type.is_dense()); - assert(_tensor_type.count_indexed_dimensions() == 1u); - _dim_size = _tensor_type.dimensions()[0].size; - cfg.setTensorType(_tensor_type); - cfg.set_distance_metric(hnsw_index_params.distance_metric()); - cfg.set_hnsw_index_params(hnsw_index_params); - _attribute = AttributeFactory::createAttribute("tensor", cfg); - _tensor_attribute = dynamic_cast<TensorAttribute *>(_attribute.get()); - assert(_tensor_attribute != nullptr); - _nearest_neighbor_index = _tensor_attribute->nearest_neighbor_index(); - assert(_nearest_neighbor_index != nullptr); -} - -HnswIndex::~HnswIndex() = default; - -bool -HnswIndex::check_lid(uint32_t lid) -{ - if (lid >= std::numeric_limits<uint32_t>::max() - lid_bias) { - std::cerr << "lid is too high" << std::endl; - return false; - } - return true; -} - -bool -HnswIndex::check_value(const char *op, const std::vector<float>& value) -{ - if (value.size() != _dim_size) { - std::cerr << op << " failed, expected vector with size " << _dim_size << ", got vector with size " << value.size() << std::endl; - return false; - } - return true; -} - -TypedCells -HnswIndex::get_typed_cells(const std::vector<float>& value, std::vector<float>& normalized_value) -{ - if (!_normalize_vectors) { - return {&value[0], CellType::FLOAT, value.size()}; - } - double sum_of_squared = 0.0; - for (auto elem : value) { - double delem = elem; - sum_of_squared += delem * delem; - } - double factor = 1.0 / (sqrt(sum_of_squared) + 1e-40); - normalized_value.reserve(value.size()); - normalized_value.clear(); - for (auto elem : value) { - normalized_value.emplace_back(elem * factor); - } - return {&normalized_value[0], CellType::FLOAT, normalized_value.size()}; -} - -void -HnswIndex::set_vector(uint32_t lid, const std::vector<float>& value) -{ - if (!check_lid(lid)) { - return; - } - if (!check_value("set_vector", value)) { - return; - } - /* - * Not thread safe against concurrent set_vector(). - */ - std::vector<float> normalized_value; - auto typed_cells = get_typed_cells(value, normalized_value); - DenseValueView tensor_view(_tensor_type, typed_cells); - while (size_t(lid + lid_bias) >= _attribute->getNumDocs()) { - uint32_t new_lid = 0; - _attribute->addDoc(new_lid); - } - _tensor_attribute->setTensor(lid + lid_bias, tensor_view); // lid 0 is special in vespa - _attribute->commit(); -} - -std::vector<float> -HnswIndex::get_vector(uint32_t lid) -{ - if (!check_lid(lid)) { - return {}; - } - TypedCells typed_cells = _tensor_attribute->extract_cells_ref(lid + lid_bias); - assert(typed_cells.size == _dim_size); - const float* data = static_cast<const float* >(typed_cells.data); - return {data, data + _dim_size}; - return {}; -} - -void -HnswIndex::clear_vector(uint32_t lid) -{ - if (!check_lid(lid)) { - return; - } - if (size_t(lid + lid_bias) < _attribute->getNumDocs()) { - _attribute->clearDoc(lid + lid_bias); - _attribute->commit(); - } -} - -TopKResult -HnswIndex::find_top_k(uint32_t k, const std::vector<float>& value, uint32_t explore_k) -{ - if (!check_value("find_top_k", value)) { - return {}; - } - /* - * Not thread safe against concurrent set_vector() since attribute - * read guard is not taken here. - */ - TopKResult result; - std::vector<float> normalized_value; - auto typed_cells = get_typed_cells(value, normalized_value); - auto df = _nearest_neighbor_index->distance_function_factory().for_query_vector(typed_cells); - auto raw_result = _nearest_neighbor_index->find_top_k(k, *df, explore_k, _no_doom.get_doom(), std::numeric_limits<double>::max()); - result.reserve(raw_result.size()); - switch (_hnsw_index_params.distance_metric()) { - case DistanceMetric::Euclidean: - for (auto &raw : raw_result) { - result.emplace_back(raw.docid - lid_bias, sqrt(raw.distance)); - } - break; - default: - for (auto &raw : raw_result) { - result.emplace_back(raw.docid - lid_bias, raw.distance); - } - } - // Results are sorted by lid, not by distance - return result; -} - -} - -using vespa_ann_benchmark::HnswIndex; - -PYBIND11_MODULE(vespa_ann_benchmark, m) { - m.doc() = "vespa_ann_benchmark plugin"; - - py::enum_<DistanceMetric>(m, "DistanceMetric") - .value("Euclidean", DistanceMetric::Euclidean) - .value("Angular", DistanceMetric::Angular) - .value("InnerProduct", DistanceMetric::InnerProduct); - - py::class_<HnswIndexParams>(m, "HnswIndexParams") - .def(py::init<uint32_t, uint32_t, DistanceMetric, bool>()); - - py::class_<HnswIndex>(m, "HnswIndex") - .def(py::init<uint32_t, const HnswIndexParams&, bool>()) - .def("set_vector", &HnswIndex::set_vector) - .def("get_vector", &HnswIndex::get_vector) - .def("clear_vector", &HnswIndex::clear_vector) - .def("find_top_k", &HnswIndex::find_top_k); -} diff --git a/client/go/go.mod b/client/go/go.mod index bf0e53a0f03..75996032a47 100644 --- a/client/go/go.mod +++ b/client/go/go.mod @@ -8,9 +8,9 @@ require ( github.com/fatih/color v1.15.0 // This is the most recent version compatible with Go 1.19. Upgrade when we upgrade our Go version github.com/go-json-experiment/json v0.0.0-20230216065249-540f01442424 - github.com/klauspost/compress v1.17.0 + github.com/klauspost/compress v1.17.2 github.com/mattn/go-colorable v0.1.13 - github.com/mattn/go-isatty v0.0.19 + github.com/mattn/go-isatty v0.0.20 github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 github.com/spf13/cobra v1.7.0 github.com/spf13/pflag v1.0.5 diff --git a/client/go/go.sum b/client/go/go.sum index 87282411b18..cab782708f5 100644 --- a/client/go/go.sum +++ b/client/go/go.sum @@ -27,6 +27,10 @@ github.com/klauspost/compress v1.16.7 h1:2mk3MPGNzKyxErAw8YaohYh69+pa4sIQSC0fPGC github.com/klauspost/compress v1.16.7/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= github.com/klauspost/compress v1.17.0 h1:Rnbp4K9EjcDuVuHtd0dgA4qNuv9yKDYKK1ulpJwgrqM= github.com/klauspost/compress v1.17.0/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= +github.com/klauspost/compress v1.17.1 h1:NE3C767s2ak2bweCZo3+rdP4U/HoyVXLv/X9f2gPS5g= +github.com/klauspost/compress v1.17.1/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= +github.com/klauspost/compress v1.17.2 h1:RlWWUY/Dr4fL8qk9YG7DTZ7PDgME2V4csBXA8L/ixi4= +github.com/klauspost/compress v1.17.2/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= @@ -39,6 +43,8 @@ github.com/mattn/go-isatty v0.0.18 h1:DOKFKCQ7FNG2L1rbrmstDN4QVRdS89Nkh85u68Uwp9 github.com/mattn/go-isatty v0.0.18/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA= github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 h1:KoWmjvw+nsYOo29YJK9vDA65RGE3NrOnUtO7a+RF9HU= github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8/go.mod h1:HKlIX3XHQyzLZPlr7++PzdhaXEj94dEiJgZDTsxEqUI= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= diff --git a/client/go/internal/cli/cmd/api_key.go b/client/go/internal/cli/cmd/api_key.go index e6e4307bb44..ef04532314c 100644 --- a/client/go/internal/cli/cmd/api_key.go +++ b/client/go/internal/cli/cmd/api_key.go @@ -18,14 +18,17 @@ func newAPIKeyCmd(cli *CLI) *cobra.Command { var overwriteKey bool cmd := &cobra.Command{ Use: "api-key", - Short: "Create a new user API key for control-plane authentication with Vespa Cloud", - Long: `Create a new user API key for control-plane authentication with Vespa Cloud. + Short: "Create a new developer key for headless authentication with Vespa Cloud control plane", + Long: `Create a new developer key for headless authentication with Vespa Cloud control plane -The API key will be stored in the Vespa CLI home directory -(see 'vespa help config'). Other commands will then automatically load the API +A developer key is intended for headless communication with the Vespa Cloud +control plane. For example when deploying from a continuous integration system. + +The developer key will be stored in the Vespa CLI home directory +(see 'vespa help config'). Other commands will then automatically load the developer key as necessary. -It's possible to override the API key used through environment variables. This +It's possible to override the developer key used through environment variables. This can be useful in continuous integration systems. Example of setting the key in-line: @@ -36,8 +39,9 @@ Example of loading the key from a custom path: export VESPA_CLI_API_KEY_FILE=/path/to/api-key -Note that when overriding API key through environment variables, that key will -always be used. It's not possible to specify a tenant-specific key. +Note that when overriding the developer key through environment variables, +that key will always be used. It's not possible to specify a tenant-specific +key through the environment. Read more in https://cloud.vespa.ai/en/security/guide`, Example: "$ vespa auth api-key -a my-tenant.my-app.my-instance", @@ -48,7 +52,7 @@ Read more in https://cloud.vespa.ai/en/security/guide`, return doApiKey(cli, overwriteKey, args) }, } - cmd.Flags().BoolVarP(&overwriteKey, "force", "f", false, "Force overwrite of existing API key") + cmd.Flags().BoolVarP(&overwriteKey, "force", "f", false, "Force overwrite of existing developer key") cmd.MarkPersistentFlagRequired(applicationFlag) return cmd } @@ -78,7 +82,7 @@ func doApiKey(cli *CLI, overwriteKey bool, args []string) error { return fmt.Errorf("could not create api key: %w", err) } if err := os.WriteFile(apiKeyFile, apiKey, 0600); err == nil { - cli.printSuccess("API private key written to ", apiKeyFile) + cli.printSuccess("Developer private key written to ", apiKeyFile) return printPublicKey(system, apiKeyFile, app.Tenant) } else { return fmt.Errorf("failed to write: %s: %w", apiKeyFile, err) diff --git a/client/go/internal/cli/cmd/api_key_test.go b/client/go/internal/cli/cmd/api_key_test.go index 9e6ee06c7fd..18baec91e0c 100644 --- a/client/go/internal/cli/cmd/api_key_test.go +++ b/client/go/internal/cli/cmd/api_key_test.go @@ -25,7 +25,7 @@ func testAPIKey(t *testing.T, subcommand []string) { err = cli.Run(args...) assert.Nil(t, err) assert.Equal(t, "", stderr.String()) - assert.Contains(t, stdout.String(), "Success: API private key written to") + assert.Contains(t, stdout.String(), "Success: Developer private key written to") err = cli.Run(subcommand...) assert.NotNil(t, err) diff --git a/client/go/internal/cli/cmd/cert.go b/client/go/internal/cli/cmd/cert.go index abcee5a4408..1cc50b1faea 100644 --- a/client/go/internal/cli/cmd/cert.go +++ b/client/go/internal/cli/cmd/cert.go @@ -21,8 +21,8 @@ func newCertCmd(cli *CLI) *cobra.Command { ) cmd := &cobra.Command{ Use: "cert", - Short: "Create a new private key and self-signed certificate for data-plane access with Vespa Cloud", - Long: `Create a new private key and self-signed certificate for data-plane access with Vespa Cloud. + Short: "Create a new self-signed certificate for authentication with Vespa Cloud data plane", + Long: `Create a new self-signed certificate for authentication with Vespa Cloud data plane. The private key and certificate will be stored in the Vespa CLI home directory (see 'vespa help config'). Other commands will then automatically load the @@ -32,8 +32,10 @@ package specified as an argument to this command (default '.'). It's possible to override the private key and certificate used through environment variables. This can be useful in continuous integration systems. -It's also possible override the CA certificate which can be useful when using self-signed certificates with a -self-hosted Vespa service. See https://docs.vespa.ai/en/operations-selfhosted/mtls.html for more information. +It's also possible override the CA certificate which can be useful when using +self-signed certificates with a self-hosted Vespa service. +See https://docs.vespa.ai/en/operations-selfhosted/mtls.html for more +information. Example of setting the CA certificate, certificate and key in-line: @@ -47,12 +49,18 @@ Example of loading CA certificate, certificate and key from custom paths: export VESPA_CLI_DATA_PLANE_CERT_FILE=/path/to/cert export VESPA_CLI_DATA_PLANE_KEY_FILE=/path/to/key +Example of disabling verification of the server's certificate chain and +hostname: + + export VESPA_CLI_DATA_PLANE_TRUST_ALL=true + Note that when overriding key pair through environment variables, that key pair will always be used for all applications. It's not possible to specify an application-specific key. -Read more in https://cloud.vespa.ai/en/security/guide`, - Example: `$ vespa auth cert -a my-tenant.my-app.my-instance +See https://cloud.vespa.ai/en/security/guide for more details.`, + Example: `$ vespa auth cert +$ vespa auth cert -a my-tenant.my-app.my-instance $ vespa auth cert -a my-tenant.my-app.my-instance path/to/application/package`, DisableAutoGenTag: true, SilenceUsage: true, diff --git a/client/go/internal/cli/cmd/clone.go b/client/go/internal/cli/cmd/clone.go index 007a99c1f96..8fd12eb9e6e 100644 --- a/client/go/internal/cli/cmd/clone.go +++ b/client/go/internal/cli/cmd/clone.go @@ -31,8 +31,8 @@ func newCloneCmd(cli *CLI) *cobra.Command { ) cmd := &cobra.Command{ Use: "clone sample-application-path target-directory", - Short: "Create files and directory structure for a new Vespa application from a sample application", - Long: `Create files and directory structure for a new Vespa application + Short: "Create files and directory structure from a Vespa sample application", + Long: `Create files and directory structure from a Vespa sample application from a sample application. Sample applications are downloaded from diff --git a/client/go/internal/cli/cmd/config.go b/client/go/internal/cli/cmd/config.go index 40104563bef..cfadc6d32c5 100644 --- a/client/go/internal/cli/cmd/config.go +++ b/client/go/internal/cli/cmd/config.go @@ -30,8 +30,8 @@ const ( func newConfigCmd() *cobra.Command { return &cobra.Command{ Use: "config", - Short: "Configure persistent values for global flags", - Long: `Configure persistent values for global flags. + Short: "Manage persistent values for global flags", + Long: `Manage persistent values for global flags. This command allows setting persistent values for global flags. On future invocations the flag can then be omitted as it is read from the config file @@ -42,7 +42,8 @@ overridden by setting the VESPA_CLI_HOME environment variable. When setting an option locally, the configuration is written to .vespa in the working directory, where that directory is assumed to be a Vespa application -directory. This allows you have separate configuration options per application. +directory. This allows you to have separate configuration options per +application. Vespa CLI chooses the value for a given option in the following order, from most to least preferred: @@ -57,10 +58,10 @@ The following global flags/options can be configured: application Specifies the application ID to manage. It has three parts, separated by -dots, with the third part being optional. This is only relevant for the "cloud" -and "hosted" targets. See https://cloud.vespa.ai/en/tenant-apps-instances for -more details. This has no default value. Examples: tenant1.app1, -tenant1.app1.instance1 +dots, with the third part being optional. If the part is omitted it defaults to +"default". This is only relevant for the "cloud" and "hosted" targets. See +https://cloud.vespa.ai/en/tenant-apps-instances for more details. This has no +default value. Examples: tenant1.app1, tenant1.app1.instance1 cluster @@ -80,28 +81,40 @@ instance Specifies the instance of the application to manage. When specified, this takes precedence over the instance specified as part of application. This has no -default value. Example: instance2 +default value and is only relevant for the "cloud" and "hosted" targets. +Example: instance2 quiet -Print only errors. +Suppress informational output. Errors are still printed. target Specifies the target to use for commands that interact with a Vespa platform, e.g. vespa deploy or vespa query. Possible values are: -- local: (default) Connect to a Vespa platform running at localhost -- cloud: Connect to Vespa Cloud -- hosted: Connect to hosted Vespa (internal platform) -- *url*: Connect to a platform running at given URL. +- local: (default) Connect to a Vespa platform running at localhost. When using + this target, container clusters are automatically discovered and are + chosen with the cluster option. This assumes that the configserver is + available on port 19071 (the default when using the Vespa container + image). +- cloud: Connect to Vespa Cloud. When using this target, container clusters are + automatically discovered and can be selected with the cluster option. +- hosted: Connect to hosted Vespa (reserved for internal use) +- *url*: Connect to a platform running at given URL. This instructs the command + you're running to target a concrete URL. The cluster option cannot be + used with this target. + +Authentication is configured automatically for the cloud and hosted targets. To +set a custom private key and certificate, e.g. for use with a self-hosted Vespa +installation using mTLS, see the documentation of 'vespa cert'. zone -Specifies a custom dev or perf zone to use when connecting to a Vespa platform. -This is only relevant for cloud and hosted targets. By default, a zone is -chosen automatically. See https://cloud.vespa.ai/en/reference/zones for -available zones. Examples: dev.aws-us-east-1c, perf.aws-us-east-1c +Specifies a custom zone to use when connecting to a Vespa Cloud application. +This is only relevant for cloud and hosted targets and defaults to a dev zone. +See https://cloud.vespa.ai/en/reference/zones for available zones. Examples: +dev.aws-us-east-1c, dev.gcp-us-central1-f, perf.aws-us-east-1c `, DisableAutoGenTag: true, SilenceUsage: false, diff --git a/client/go/internal/cli/cmd/curl.go b/client/go/internal/cli/cmd/curl.go index d605ac6948c..4a919d18d49 100644 --- a/client/go/internal/cli/cmd/curl.go +++ b/client/go/internal/cli/cmd/curl.go @@ -41,7 +41,7 @@ $ vespa curl -- -v --data-urlencode "yql=select * from music where album contain } var service *vespa.Service useDeploy := curlService == "deploy" - waiter := cli.waiter(false, time.Duration(waitSecs)*time.Second) + waiter := cli.waiter(time.Duration(waitSecs) * time.Second) if useDeploy { if cli.config.cluster() != "" { return fmt.Errorf("cannot specify cluster for service %s", curlService) @@ -59,6 +59,7 @@ $ vespa curl -- -v --data-urlencode "yql=select * from music where album contain if err != nil { return err } + // TODO(mpolden): Support issuing request to deploy service if useDeploy { if err := addAccessToken(c, target); err != nil { return err diff --git a/client/go/internal/cli/cmd/deploy.go b/client/go/internal/cli/cmd/deploy.go index 0a4d72a8a48..aee26975901 100644 --- a/client/go/internal/cli/cmd/deploy.go +++ b/client/go/internal/cli/cmd/deploy.go @@ -73,7 +73,7 @@ $ vespa deploy -t cloud -z perf.aws-us-east-1c`, return err } } - waiter := cli.waiter(false, timeout) + waiter := cli.waiter(timeout) if _, err := waiter.DeployService(target); err != nil { return err } @@ -158,7 +158,7 @@ func newActivateCmd(cli *CLI) *cobra.Command { return err } timeout := time.Duration(waitSecs) * time.Second - waiter := cli.waiter(false, timeout) + waiter := cli.waiter(timeout) if _, err := waiter.DeployService(target); err != nil { return err } @@ -179,7 +179,7 @@ func waitForDeploymentReady(cli *CLI, target vespa.Target, sessionOrRunID int64, if timeout == 0 { return nil } - waiter := cli.waiter(false, timeout) + waiter := cli.waiter(timeout) if _, err := waiter.Deployment(target, sessionOrRunID); err != nil { return err } diff --git a/client/go/internal/cli/cmd/destroy.go b/client/go/internal/cli/cmd/destroy.go index 2c32b7406b7..f4822330e05 100644 --- a/client/go/internal/cli/cmd/destroy.go +++ b/client/go/internal/cli/cmd/destroy.go @@ -13,8 +13,8 @@ func newDestroyCmd(cli *CLI) *cobra.Command { force := false cmd := &cobra.Command{ Use: "destroy", - Short: "Remove a deployed application and its data", - Long: `Remove a deployed application and its data. + Short: "Remove a deployed Vespa application and its data", + Long: `Remove a deployed Vespa application and its data. This command removes the currently deployed application and permanently deletes its data. diff --git a/client/go/internal/cli/cmd/document.go b/client/go/internal/cli/cmd/document.go index 090060d578c..a98e9867d34 100644 --- a/client/go/internal/cli/cmd/document.go +++ b/client/go/internal/cli/cmd/document.go @@ -298,7 +298,7 @@ func documentService(cli *CLI, waitSecs int) (*vespa.Service, error) { if err != nil { return nil, err } - waiter := cli.waiter(false, time.Duration(waitSecs)*time.Second) + waiter := cli.waiter(time.Duration(waitSecs) * time.Second) return waiter.Service(target, cli.config.cluster()) } diff --git a/client/go/internal/cli/cmd/feed.go b/client/go/internal/cli/cmd/feed.go index 79949a02106..1a32ac7110d 100644 --- a/client/go/internal/cli/cmd/feed.go +++ b/client/go/internal/cli/cmd/feed.go @@ -58,8 +58,8 @@ func newFeedCmd(cli *CLI) *cobra.Command { var options feedOptions cmd := &cobra.Command{ Use: "feed json-file [json-file]...", - Short: "Feed multiple document operations to a Vespa cluster", - Long: `Feed multiple document operations to a Vespa cluster. + Short: "Feed multiple document operations to Vespa", + Long: `Feed multiple document operations to Vespa. This command can be used to feed large amounts of documents to a Vespa cluster efficiently. @@ -108,7 +108,7 @@ func createServices(n int, timeout time.Duration, waitSecs int, cli *CLI) ([]uti } services := make([]util.HTTPClient, 0, n) baseURL := "" - waiter := cli.waiter(false, time.Duration(waitSecs)*time.Second) + waiter := cli.waiter(time.Duration(waitSecs) * time.Second) for i := 0; i < n; i++ { service, err := waiter.Service(target, cli.config.cluster()) if err != nil { diff --git a/client/go/internal/cli/cmd/login.go b/client/go/internal/cli/cmd/login.go index 0072c0033c8..b380e627203 100644 --- a/client/go/internal/cli/cmd/login.go +++ b/client/go/internal/cli/cmd/login.go @@ -19,9 +19,13 @@ import ( // this will only affect the messages. func newLoginCmd(cli *CLI) *cobra.Command { return &cobra.Command{ - Use: "login", - Args: cobra.NoArgs, - Short: "Authenticate Vespa CLI with Vespa Cloud", + Use: "login", + Args: cobra.NoArgs, + Short: "Authenticate Vespa CLI with Vespa Cloud control plane. This is preferred over api-key for interactive use", + Long: `Authenticate Vespa CLI with Vespa Cloud control plane. This is preferred over api-key for interactive use. + +This command runs a browser-based authentication flow for the Vespa Cloud control plane. +`, Example: "$ vespa auth login", DisableAutoGenTag: true, SilenceUsage: true, diff --git a/client/go/internal/cli/cmd/query.go b/client/go/internal/cli/cmd/query.go index 6fb571dc36b..bf2272ca981 100644 --- a/client/go/internal/cli/cmd/query.go +++ b/client/go/internal/cli/cmd/query.go @@ -64,7 +64,7 @@ func query(cli *CLI, arguments []string, timeoutSecs, waitSecs int, curl bool) e if err != nil { return err } - waiter := cli.waiter(false, time.Duration(waitSecs)*time.Second) + waiter := cli.waiter(time.Duration(waitSecs) * time.Second) service, err := waiter.Service(target, cli.config.cluster()) if err != nil { return err diff --git a/client/go/internal/cli/cmd/root.go b/client/go/internal/cli/cmd/root.go index 8271386d07a..068a7ed90b6 100644 --- a/client/go/internal/cli/cmd/root.go +++ b/client/go/internal/cli/cmd/root.go @@ -213,10 +213,10 @@ func (c *CLI) configureFlags() map[string]*pflag.Flag { quiet bool ) c.cmd.PersistentFlags().StringVarP(&target, targetFlag, "t", "local", `The target platform to use. Must be "local", "cloud", "hosted" or an URL`) - c.cmd.PersistentFlags().StringVarP(&application, applicationFlag, "a", "", "The application to use") - c.cmd.PersistentFlags().StringVarP(&instance, instanceFlag, "i", "", "The instance of the application to use") + c.cmd.PersistentFlags().StringVarP(&application, applicationFlag, "a", "", "The application to use (cloud only)") + c.cmd.PersistentFlags().StringVarP(&instance, instanceFlag, "i", "", "The instance of the application to use (cloud only)") c.cmd.PersistentFlags().StringVarP(&cluster, clusterFlag, "C", "", "The container cluster to use. This is only required for applications with multiple clusters") - c.cmd.PersistentFlags().StringVarP(&zone, zoneFlag, "z", "", "The zone to use. This defaults to a dev zone") + c.cmd.PersistentFlags().StringVarP(&zone, zoneFlag, "z", "", "The zone to use. This defaults to a dev zone (cloud only)") c.cmd.PersistentFlags().StringVarP(&color, colorFlag, "c", "auto", `Whether to use colors in output. Must be "auto", "never", or "always"`) c.cmd.PersistentFlags().BoolVarP(&quiet, quietFlag, "q", false, "Print only errors") flags := make(map[string]*pflag.Flag) @@ -345,9 +345,7 @@ func (c *CLI) confirm(question string, confirmByDefault bool) (bool, error) { } } -func (c *CLI) waiter(once bool, timeout time.Duration) *Waiter { - return &Waiter{Once: once, Timeout: timeout, cli: c} -} +func (c *CLI) waiter(timeout time.Duration) *Waiter { return &Waiter{Timeout: timeout, cli: c} } // target creates a target according the configuration of this CLI and given opts. func (c *CLI) target(opts targetOptions) (vespa.Target, error) { diff --git a/client/go/internal/cli/cmd/status.go b/client/go/internal/cli/cmd/status.go index b10801ae546..a0602494bff 100644 --- a/client/go/internal/cli/cmd/status.go +++ b/client/go/internal/cli/cmd/status.go @@ -25,9 +25,14 @@ func newStatusCmd(cli *CLI) *cobra.Command { "status document", // TODO: Remove on Vespa 9 "status query", // TODO: Remove on Vespa 9 }, - Short: "Verify that container service(s) are ready to use", + Short: "Show Vespa endpoints and status", + Long: `Show Vespa endpoints and status. + +This command shows the current endpoints, and their status, of a deployed Vespa +application.`, Example: `$ vespa status -$ vespa status --cluster mycluster`, +$ vespa status --cluster mycluster +$ vespa status --cluster mycluster --wait 600`, DisableAutoGenTag: true, SilenceUsage: true, Args: cobra.MaximumNArgs(1), @@ -37,7 +42,8 @@ $ vespa status --cluster mycluster`, if err != nil { return err } - waiter := cli.waiter(true, time.Duration(waitSecs)*time.Second) + waiter := cli.waiter(time.Duration(waitSecs) * time.Second) + var failingContainers []*vespa.Service if cluster == "" { services, err := waiter.Services(t) if err != nil { @@ -47,28 +53,46 @@ $ vespa status --cluster mycluster`, return errHint(fmt.Errorf("no services exist"), "Deployment may not be ready yet", "Try 'vespa status deployment'") } for _, s := range services { - printReadyService(s, cli) + if !printServiceStatus(s, waiter, cli) { + failingContainers = append(failingContainers, s) + } } - return nil } else { s, err := waiter.Service(t, cluster) if err != nil { return err } - printReadyService(s, cli) - return nil + if !printServiceStatus(s, waiter, cli) { + failingContainers = append(failingContainers, s) + } } + return failingServicesErr(failingContainers...) }, } cli.bindWaitFlag(cmd, 0, &waitSecs) return cmd } +func failingServicesErr(services ...*vespa.Service) error { + if len(services) == 0 { + return nil + } + var nameOrURL []string + for _, s := range services { + if s.Name != "" { + nameOrURL = append(nameOrURL, s.Name) + } else { + nameOrURL = append(nameOrURL, s.BaseURL) + } + } + return fmt.Errorf("services not ready: %s", strings.Join(nameOrURL, ", ")) +} + func newStatusDeployCmd(cli *CLI) *cobra.Command { var waitSecs int cmd := &cobra.Command{ Use: "deploy", - Short: "Verify that the deploy service is ready to use", + Short: "Show status of the Vespa deploy service", Example: `$ vespa status deploy`, DisableAutoGenTag: true, SilenceUsage: true, @@ -78,12 +102,14 @@ func newStatusDeployCmd(cli *CLI) *cobra.Command { if err != nil { return err } - waiter := cli.waiter(true, time.Duration(waitSecs)*time.Second) + waiter := cli.waiter(time.Duration(waitSecs) * time.Second) s, err := waiter.DeployService(t) if err != nil { return err } - printReadyService(s, cli) + if !printServiceStatus(s, waiter, cli) { + return failingServicesErr(s) + } return nil }, } @@ -95,10 +121,17 @@ func newStatusDeploymentCmd(cli *CLI) *cobra.Command { var waitSecs int cmd := &cobra.Command{ Use: "deployment", - Short: "Verify that deployment has converged on latest, or given, ID", + Short: "Show status of a Vespa deployment", + Long: `Show status of a Vespa deployment. + +This commands shows whether a Vespa deployment has converged on the latest run + (Vespa Cloud) or config generation (self-hosted). If an argument is given, +show the convergence status of that particular run or generation. +`, Example: `$ vespa status deployment $ vespa status deployment -t cloud [run-id] $ vespa status deployment -t local [session-id] +$ vespa status deployment -t local [session-id] --wait 600 `, DisableAutoGenTag: true, SilenceUsage: true, @@ -116,7 +149,7 @@ $ vespa status deployment -t local [session-id] if err != nil { return err } - waiter := cli.waiter(true, time.Duration(waitSecs)*time.Second) + waiter := cli.waiter(time.Duration(waitSecs) * time.Second) id, err := waiter.Deployment(t, wantedID) if err != nil { return err @@ -134,8 +167,19 @@ $ vespa status deployment -t local [session-id] return cmd } -func printReadyService(s *vespa.Service, cli *CLI) { +func printServiceStatus(s *vespa.Service, waiter *Waiter, cli *CLI) bool { desc := s.Description() desc = strings.ToUpper(string(desc[0])) + string(desc[1:]) - log.Print(desc, " at ", color.CyanString(s.BaseURL), " is ", color.GreenString("ready")) + err := s.Wait(waiter.Timeout) + var sb strings.Builder + sb.WriteString(fmt.Sprintf("%s at %s is ", desc, color.CyanString(s.BaseURL))) + if err == nil { + sb.WriteString(color.GreenString("ready")) + } else { + sb.WriteString(color.RedString("not ready")) + sb.WriteString(": ") + sb.WriteString(err.Error()) + } + fmt.Fprintln(cli.Stdout, sb.String()) + return err == nil } diff --git a/client/go/internal/cli/cmd/status_test.go b/client/go/internal/cli/cmd/status_test.go index 1e6c3230db3..1473e39ff54 100644 --- a/client/go/internal/cli/cmd/status_test.go +++ b/client/go/internal/cli/cmd/status_test.go @@ -38,12 +38,18 @@ func TestStatusCommandMultiCluster(t *testing.T) { mockServiceStatus(client) assert.NotNil(t, cli.Run("status")) assert.Equal(t, "Error: no services exist\nHint: Deployment may not be ready yet\nHint: Try 'vespa status deployment'\n", stderr.String()) + stderr.Reset() mockServiceStatus(client, "foo", "bar") - assert.Nil(t, cli.Run("status")) + client.NextStatus(200) + client.NextStatus(400) // One cluster is unavilable + assert.NotNil(t, cli.Run("status")) assert.Equal(t, `Container bar at http://127.0.0.1:8080 is ready -Container foo at http://127.0.0.1:8080 is ready +Container foo at http://127.0.0.1:8080 is not ready: unhealthy container foo: status 400 at http://127.0.0.1:8080/status.html: aborting wait: got status 400 `, stdout.String()) + assert.Equal(t, + "Error: services not ready: foo\n", + stderr.String()) stdout.Reset() mockServiceStatus(client, "foo", "bar") @@ -60,7 +66,7 @@ func TestStatusCommandMultiClusterWait(t *testing.T) { client.NextStatus(400) assert.NotNil(t, cli.Run("status", "--cluster", "foo", "--wait", "10")) assert.Equal(t, "Waiting up to 10s for cluster discovery...\nWaiting up to 10s for container foo...\n"+ - "Error: unhealthy container foo after waiting up to 10s: status 400 at http://127.0.0.1:8080/ApplicationStatus: aborting wait: got status 400\n", stderr.String()) + "Error: unhealthy container foo after waiting up to 10s: status 400 at http://127.0.0.1:8080/status.html: aborting wait: got status 400\n", stderr.String()) } func TestStatusCommandWithUrlTarget(t *testing.T) { @@ -75,18 +81,25 @@ func TestStatusError(t *testing.T) { client := &mock.HTTPClient{} mockServiceStatus(client, "default") client.NextStatus(500) - cli, _, stderr := newTestCLI(t) + cli, stdout, stderr := newTestCLI(t) cli.httpClient = client assert.NotNil(t, cli.Run("status", "container")) assert.Equal(t, - "Error: unhealthy container default: status 500 at http://127.0.0.1:8080/ApplicationStatus: wait timed out\n", + "Container default at http://127.0.0.1:8080 is not ready: unhealthy container default: status 500 at http://127.0.0.1:8080/status.html: wait timed out\n", + stdout.String()) + assert.Equal(t, + "Error: services not ready: default\n", stderr.String()) + stdout.Reset() stderr.Reset() client.NextResponseError(io.EOF) assert.NotNil(t, cli.Run("status", "container", "-t", "http://example.com")) assert.Equal(t, - "Error: unhealthy container at http://example.com/ApplicationStatus: EOF\n", + "Container at http://example.com is not ready: unhealthy container at http://example.com/status.html: EOF\n", + stdout.String()) + assert.Equal(t, + "Error: services not ready: http://example.com\n", stderr.String()) } @@ -189,7 +202,7 @@ func assertStatus(expectedTarget string, args []string, t *testing.T) { clusterName = "foo" mockServiceStatus(client, clusterName) } - client.NextResponse(mock.HTTPResponse{URI: "/ApplicationStatus", Status: 200}) + client.NextResponse(mock.HTTPResponse{URI: "/status.html", Status: 200}) } cli, stdout, _ := newTestCLI(t) cli.httpClient = client @@ -200,14 +213,14 @@ func assertStatus(expectedTarget string, args []string, t *testing.T) { prefix += " " + clusterName } assert.Equal(t, prefix+" at "+expectedTarget+" is ready\n", stdout.String()) - assert.Equal(t, expectedTarget+"/ApplicationStatus", client.LastRequest.URL.String()) + assert.Equal(t, expectedTarget+"/status.html", client.LastRequest.URL.String()) // Test legacy command statusArgs = []string{"status query"} stdout.Reset() assert.Nil(t, cli.Run(append(statusArgs, args...)...)) assert.Equal(t, prefix+" at "+expectedTarget+" is ready\n", stdout.String()) - assert.Equal(t, expectedTarget+"/ApplicationStatus", client.LastRequest.URL.String()) + assert.Equal(t, expectedTarget+"/status.html", client.LastRequest.URL.String()) } func assertDocumentStatus(target string, args []string, t *testing.T) { @@ -223,5 +236,5 @@ func assertDocumentStatus(target string, args []string, t *testing.T) { "Container (document API) at "+target+" is ready\n", stdout.String(), "vespa status container") - assert.Equal(t, target+"/ApplicationStatus", client.LastRequest.URL.String()) + assert.Equal(t, target+"/status.html", client.LastRequest.URL.String()) } diff --git a/client/go/internal/cli/cmd/test.go b/client/go/internal/cli/cmd/test.go index e842cab606e..f0432dd4f70 100644 --- a/client/go/internal/cli/cmd/test.go +++ b/client/go/internal/cli/cmd/test.go @@ -220,7 +220,7 @@ func verify(step step, defaultCluster string, defaultParameters map[string]strin service, ok = context.clusters[cluster] if !ok { // Cache service so we don't have to discover it for every step - waiter := context.cli.waiter(false, time.Duration(waitSecs)*time.Second) + waiter := context.cli.waiter(time.Duration(waitSecs) * time.Second) service, err = waiter.Service(target, cluster) if err != nil { return "", "", err diff --git a/client/go/internal/cli/cmd/version.go b/client/go/internal/cli/cmd/version.go index eb2618f7274..f110541c192 100644 --- a/client/go/internal/cli/cmd/version.go +++ b/client/go/internal/cli/cmd/version.go @@ -22,7 +22,7 @@ func newVersionCmd(cli *CLI) *cobra.Command { var skipVersionCheck bool cmd := &cobra.Command{ Use: "version", - Short: "Show current version and check for updates", + Short: "Show current CLI version and check for updates", DisableAutoGenTag: true, SilenceUsage: true, Args: cobra.ExactArgs(0), diff --git a/client/go/internal/cli/cmd/visit.go b/client/go/internal/cli/cmd/visit.go index 2a63a5fb57e..bb226701e0a 100644 --- a/client/go/internal/cli/cmd/visit.go +++ b/client/go/internal/cli/cmd/visit.go @@ -90,8 +90,8 @@ func newVisitCmd(cli *CLI) *cobra.Command { ) cmd := &cobra.Command{ Use: "visit", - Short: "Visit and print all documents in a Vespa cluster", - Long: `Visit and print all documents in a Vespa cluster. + Short: "Fetch and print all documents from Vespa", + Long: `Fetch and print all documents from Vespa. By default prints each document received on its own line (JSONL format). `, diff --git a/client/go/internal/cli/cmd/waiter.go b/client/go/internal/cli/cmd/waiter.go index 302bc679885..0cfb3aa76d5 100644 --- a/client/go/internal/cli/cmd/waiter.go +++ b/client/go/internal/cli/cmd/waiter.go @@ -11,17 +11,12 @@ import ( // Waiter waits for Vespa services to become ready, within a timeout. type Waiter struct { - // Once species whether we should wait at least one time, irregardless of timeout. - Once bool - // Timeout specifies how long we should wait for an operation to complete. Timeout time.Duration // TODO(mpolden): Consider making this a budget cli *CLI } -func (w *Waiter) wait() bool { return w.Once || w.Timeout > 0 } - // DeployService returns the service providing the deploy API on given target, func (w *Waiter) DeployService(target vespa.Target) (*vespa.Service, error) { s, err := target.DeployService() @@ -74,8 +69,6 @@ func (w *Waiter) Services(target vespa.Target) ([]*vespa.Service, error) { func (w *Waiter) maybeWaitFor(service *vespa.Service) error { if w.Timeout > 0 { w.cli.printInfo("Waiting up to ", color.CyanString(w.Timeout.String()), " for ", service.Description(), "...") - } - if w.wait() { return service.Wait(w.Timeout) } return nil diff --git a/client/go/internal/vespa/target.go b/client/go/internal/vespa/target.go index 065537d8610..8c3f5c9b7c3 100644 --- a/client/go/internal/vespa/target.go +++ b/client/go/internal/vespa/target.go @@ -37,6 +37,7 @@ const ( ) var errWaitTimeout = errors.New("wait timed out") +var errAuth = errors.New("auth failed") // Authenticator authenticates the given HTTP request. type Authenticator interface { @@ -108,12 +109,10 @@ type LogOptions struct { // Do sends request to this service. Authentication of the request happens automatically. func (s *Service) Do(request *http.Request, timeout time.Duration) (*http.Response, error) { - s.once.Do(func() { - util.ConfigureTLS(s.httpClient, s.TLSOptions.KeyPair, s.TLSOptions.CACertificate, s.TLSOptions.TrustAll) - }) + util.ConfigureTLS(s.httpClient, s.TLSOptions.KeyPair, s.TLSOptions.CACertificate, s.TLSOptions.TrustAll) if s.auth != nil { if err := s.auth.Authenticate(request); err != nil { - return nil, err + return nil, fmt.Errorf("%w: %s", errAuth, err) } } return s.httpClient.Do(request, timeout) @@ -124,12 +123,8 @@ func (s *Service) SetClient(client util.HTTPClient) { s.httpClient = client } // Wait polls the health check of this service until it succeeds or timeout passes. func (s *Service) Wait(timeout time.Duration) error { - url := s.BaseURL - if s.deployAPI { - url += "/status.html" // because /ApplicationStatus is not publicly reachable in Vespa Cloud - } else { - url += "/ApplicationStatus" - } + // A path that does not need authentication, on any target + url := strings.TrimRight(s.BaseURL, "/") + "/status.html" req, err := http.NewRequest("GET", url, nil) if err != nil { return err @@ -220,7 +215,9 @@ func wait(service *Service, okFn responseFunc, reqFn requestFunc, timeout, retry loopOnce := timeout == 0 for time.Now().Before(deadline) || loopOnce { response, err = service.Do(reqFn(), 10*time.Second) - if err == nil { + if errors.Is(err, errAuth) { + return status, fmt.Errorf("aborting wait: %w", err) + } else if err == nil { status = response.StatusCode body, err := io.ReadAll(response.Body) if err != nil { diff --git a/client/go/internal/vespa/target_test.go b/client/go/internal/vespa/target_test.go index abcbefe5529..4c2fda8368e 100644 --- a/client/go/internal/vespa/target_test.go +++ b/client/go/internal/vespa/target_test.go @@ -88,7 +88,7 @@ func TestCustomTargetWait(t *testing.T) { client.NextResponseError(io.EOF) } // Then succeeds - client.NextResponse(mock.HTTPResponse{URI: "/ApplicationStatus", Status: 200}) + client.NextResponse(mock.HTTPResponse{URI: "/status.html", Status: 200}) assertService(t, false, target, "", time.Second) } @@ -120,6 +120,8 @@ func TestCustomTargetAwaitDeployment(t *testing.T) { func TestCloudTargetWait(t *testing.T) { var logWriter bytes.Buffer target, client := createCloudTarget(t, &logWriter) + client.NextResponseError(errAuth) + assertService(t, true, target, "deploy", time.Second) // No retrying on auth error client.NextStatus(401) assertService(t, true, target, "deploy", time.Second) // No retrying on 4xx client.NextStatus(500) @@ -151,10 +153,10 @@ func TestCloudTargetWait(t *testing.T) { assert.Equal(t, 2, len(services)) client.NextResponse(response) - client.NextResponse(mock.HTTPResponse{URI: "/ApplicationStatus", Status: 500}) + client.NextResponse(mock.HTTPResponse{URI: "/status.html", Status: 500}) assertService(t, true, target, "default", 0) client.NextResponse(response) - client.NextResponse(mock.HTTPResponse{URI: "/ApplicationStatus", Status: 200}) + client.NextResponse(mock.HTTPResponse{URI: "/status.html", Status: 200}) assertService(t, false, target, "feed", 0) } diff --git a/client/go/internal/vespa/xml/config.go b/client/go/internal/vespa/xml/config.go index e77e04c3a6f..05d73474ffc 100644 --- a/client/go/internal/vespa/xml/config.go +++ b/client/go/internal/vespa/xml/config.go @@ -223,15 +223,40 @@ func ParseNodeCount(s string) (int, int, error) { // IsProdRegion returns whether string s is a valid production region. func IsProdRegion(s string, system vespa.System) bool { - // TODO: Add support for cd and main systems - if system.Name == vespa.PublicCDSystem.Name { - return s == "aws-us-east-1c" - } - switch s { - case "aws-us-east-1c", "aws-us-west-2a", - "aws-eu-west-1a", "aws-ap-northeast-1a", - "gcp-us-central1-f": - return true + switch system.Name { + case vespa.CDSystem.Name: + switch s { + case "aws-us-east-1a", "cd-us-east-1", + "cd-us-west-1": + return true + } + case vespa.MainSystem.Name: + switch s { + case "prod.ap-northeast-1", "prod.ap-northeast-2", + "prod.ap-southeast-1", "prod.aws-ap-northeast-2a", + "prod.aws-apse1-az1", "prod.aws-apse1-az3", + "prod.aws-ap-southeast-1a", "prod.aws-euw1-az1", + "prod.aws-euw1-az3", "prod.aws-eu-west-1a", + "prod.aws-use1-az2", "prod.aws-us-east-1a", + "prod.aws-us-east-1b", "prod.aws-us-east-2a", + "prod.aws-usw2-az2", "prod.aws-usw2-az3", + "prod.aws-us-west-2a", "prod.eu-west-1", + "prod.us-central-1", "prod.us-east-3", + "prod.us-west-1": + return true + } + case vespa.PublicCDSystem.Name: + switch s { + case "aws-us-east-1c", "gcp-us-central1-f": + return true + } + case vespa.PublicSystem.Name: + switch s { + case "aws-us-east-1c", "aws-us-west-2a", + "aws-eu-west-1a", "aws-ap-northeast-1a", + "gcp-europe-west3-b", "gcp-us-central1-f": + return true + } } return false } diff --git a/client/js/app/yarn.lock b/client/js/app/yarn.lock index a00131e7ba0..4c3b5946c46 100644 --- a/client/js/app/yarn.lock +++ b/client/js/app/yarn.lock @@ -15,7 +15,7 @@ "@jridgewell/gen-mapping" "^0.3.0" "@jridgewell/trace-mapping" "^0.3.9" -"@babel/code-frame@^7.0.0", "@babel/code-frame@^7.12.13", "@babel/code-frame@^7.22.10", "@babel/code-frame@^7.22.13", "@babel/code-frame@^7.22.5": +"@babel/code-frame@^7.0.0", "@babel/code-frame@^7.12.13", "@babel/code-frame@^7.22.13", "@babel/code-frame@^7.22.5": version "7.22.13" resolved "https://registry.yarnpkg.com/@babel/code-frame/-/code-frame-7.22.13.tgz#e3c1c099402598483b7a8c46a721d1038803755e" integrity sha512-XktuhWlJ5g+3TJXc5upd9Ks1HutSArik6jf2eAjYFyIOf4ej3RN+184cZbzDvbPnuTJIUhPKKJE3cIsYTiAT3w== @@ -91,7 +91,7 @@ json5 "^2.2.3" semver "^6.3.1" -"@babel/generator@^7.22.10", "@babel/generator@^7.22.15", "@babel/generator@^7.7.2": +"@babel/generator@^7.22.15", "@babel/generator@^7.7.2": version "7.22.15" resolved "https://registry.yarnpkg.com/@babel/generator/-/generator-7.22.15.tgz#1564189c7ec94cb8f77b5e8a90c4d200d21b2339" integrity sha512-Zu9oWARBqeVOW0dZOjXc3JObrzuqothQ3y/n1kUtrjCoCPLkXUwMvOo/F/TCfoHMbWIFlWwpZtkZVb9ga4U2pA== @@ -111,6 +111,16 @@ "@jridgewell/trace-mapping" "^0.3.17" jsesc "^2.5.1" +"@babel/generator@^7.23.0": + version "7.23.0" + resolved "https://registry.yarnpkg.com/@babel/generator/-/generator-7.23.0.tgz#df5c386e2218be505b34837acbcb874d7a983420" + integrity sha512-lN85QRR+5IbYrMWM6Y4pE/noaQtg4pNiqeNGX60eqOfo6gtEj6uw/JagelB8vVztSd7R6M5n1+PQkDbHbBRU4g== + dependencies: + "@babel/types" "^7.23.0" + "@jridgewell/gen-mapping" "^0.3.2" + "@jridgewell/trace-mapping" "^0.3.17" + jsesc "^2.5.1" + "@babel/helper-compilation-targets@^7.22.15": version "7.22.15" resolved "https://registry.yarnpkg.com/@babel/helper-compilation-targets/-/helper-compilation-targets-7.22.15.tgz#0698fc44551a26cf29f18d4662d5bf545a6cfc52" @@ -138,13 +148,13 @@ resolved "https://registry.yarnpkg.com/@babel/helper-environment-visitor/-/helper-environment-visitor-7.22.20.tgz#96159db61d34a29dba454c959f5ae4a649ba9167" integrity sha512-zfedSIzFhat/gFhWfHtgWvlec0nqB9YEIVrpuwjruLlXfUSnA8cJB0miHKwqDnQ7d32aKo2xt88/xZptwxbfhA== -"@babel/helper-function-name@^7.22.5": - version "7.22.5" - resolved "https://registry.yarnpkg.com/@babel/helper-function-name/-/helper-function-name-7.22.5.tgz#ede300828905bb15e582c037162f99d5183af1be" - integrity sha512-wtHSq6jMRE3uF2otvfuD3DIvVhOsSNshQl0Qrd7qC9oQJzHvOL4qQXlQn2916+CXGywIjpGuIkoyZRRxHPiNQQ== +"@babel/helper-function-name@^7.23.0": + version "7.23.0" + resolved "https://registry.yarnpkg.com/@babel/helper-function-name/-/helper-function-name-7.23.0.tgz#1f9a3cdbd5b2698a670c30d2735f9af95ed52759" + integrity sha512-OErEqsrxjZTJciZ4Oo+eoZqeW9UIiOcuYKRJA4ZAgV9myA+pOXhhmpfNCKjEH/auVfEYVFJ6y1Tc4r0eIApqiw== dependencies: - "@babel/template" "^7.22.5" - "@babel/types" "^7.22.5" + "@babel/template" "^7.22.15" + "@babel/types" "^7.23.0" "@babel/helper-hoist-variables@^7.22.5": version "7.22.5" @@ -261,7 +271,7 @@ chalk "^2.4.2" js-tokens "^4.0.0" -"@babel/parser@^7.1.0", "@babel/parser@^7.14.7", "@babel/parser@^7.20.7", "@babel/parser@^7.22.11", "@babel/parser@^7.22.15", "@babel/parser@^7.22.16": +"@babel/parser@^7.1.0", "@babel/parser@^7.14.7", "@babel/parser@^7.20.7", "@babel/parser@^7.22.15", "@babel/parser@^7.22.16": version "7.22.16" resolved "https://registry.yarnpkg.com/@babel/parser/-/parser-7.22.16.tgz#180aead7f247305cce6551bea2720934e2fa2c95" integrity sha512-+gPfKv8UWeKKeJTUxe59+OobVcrYHETCsORl61EmSkmgymguYk/X5bp7GuUIXaFsc6y++v8ZxPsLSSuujqDphA== @@ -271,6 +281,11 @@ resolved "https://registry.yarnpkg.com/@babel/parser/-/parser-7.22.13.tgz#23fb17892b2be7afef94f573031c2f4b42839a2b" integrity sha512-3l6+4YOvc9wx7VlCSw4yQfcBo01ECA8TicQfbnCPuCEpRQrf+gTUyGdxNw+pyTUyywp6JRD1w0YQs9TpBXYlkw== +"@babel/parser@^7.23.0": + version "7.23.0" + resolved "https://registry.yarnpkg.com/@babel/parser/-/parser-7.23.0.tgz#da950e622420bf96ca0d0f2909cdddac3acd8719" + integrity sha512-vvPKKdMemU85V9WE/l5wZEmImpCtLqbnTvqDS2U1fJ96KrxoW7KrXhNsNCblQlg8Ck4b85yxdTyelsMUgFUXiw== + "@babel/plugin-syntax-async-generators@^7.8.4": version "7.8.4" resolved "https://registry.yarnpkg.com/@babel/plugin-syntax-async-generators/-/plugin-syntax-async-generators-7.8.4.tgz#a983fb1aeb2ec3f6ed042a210f640e90e786fe0d" @@ -415,51 +430,19 @@ "@babel/parser" "^7.22.15" "@babel/types" "^7.22.15" -"@babel/traverse@^7.22.11", "@babel/traverse@^7.22.17": - version "7.22.17" - resolved "https://registry.yarnpkg.com/@babel/traverse/-/traverse-7.22.17.tgz#b23c203ab3707e3be816043081b4a994fcacec44" - integrity sha512-xK4Uwm0JnAMvxYZxOVecss85WxTEIbTa7bnGyf/+EgCL5Zt3U7htUpEOWv9detPlamGKuRzCqw74xVglDWpPdg== - dependencies: - "@babel/code-frame" "^7.22.13" - "@babel/generator" "^7.22.15" - "@babel/helper-environment-visitor" "^7.22.5" - "@babel/helper-function-name" "^7.22.5" - "@babel/helper-hoist-variables" "^7.22.5" - "@babel/helper-split-export-declaration" "^7.22.6" - "@babel/parser" "^7.22.16" - "@babel/types" "^7.22.17" - debug "^4.1.0" - globals "^11.1.0" - -"@babel/traverse@^7.22.15", "@babel/traverse@^7.22.20": - version "7.22.20" - resolved "https://registry.yarnpkg.com/@babel/traverse/-/traverse-7.22.20.tgz#db572d9cb5c79e02d83e5618b82f6991c07584c9" - integrity sha512-eU260mPZbU7mZ0N+X10pxXhQFMGTeLb9eFS0mxehS8HZp9o1uSnFeWQuG1UPrlxgA7QoUzFhOnilHDp0AXCyHw== +"@babel/traverse@^7.22.11", "@babel/traverse@^7.22.15", "@babel/traverse@^7.22.17", "@babel/traverse@^7.22.20", "@babel/traverse@^7.22.8": + version "7.23.2" + resolved "https://registry.yarnpkg.com/@babel/traverse/-/traverse-7.23.2.tgz#329c7a06735e144a506bdb2cad0268b7f46f4ad8" + integrity sha512-azpe59SQ48qG6nu2CzcMLbxUudtN+dOM9kDbUqGq3HXUJRlo7i8fvPoxQUzYgLZ4cMVmuZgm8vvBpNeRhd6XSw== dependencies: "@babel/code-frame" "^7.22.13" - "@babel/generator" "^7.22.15" + "@babel/generator" "^7.23.0" "@babel/helper-environment-visitor" "^7.22.20" - "@babel/helper-function-name" "^7.22.5" + "@babel/helper-function-name" "^7.23.0" "@babel/helper-hoist-variables" "^7.22.5" "@babel/helper-split-export-declaration" "^7.22.6" - "@babel/parser" "^7.22.16" - "@babel/types" "^7.22.19" - debug "^4.1.0" - globals "^11.1.0" - -"@babel/traverse@^7.22.8": - version "7.22.11" - resolved "https://registry.yarnpkg.com/@babel/traverse/-/traverse-7.22.11.tgz#71ebb3af7a05ff97280b83f05f8865ac94b2027c" - integrity sha512-mzAenteTfomcB7mfPtyi+4oe5BZ6MXxWcn4CX+h4IRJ+OOGXBrWU6jDQavkQI9Vuc5P+donFabBfFCcmWka9lQ== - dependencies: - "@babel/code-frame" "^7.22.10" - "@babel/generator" "^7.22.10" - "@babel/helper-environment-visitor" "^7.22.5" - "@babel/helper-function-name" "^7.22.5" - "@babel/helper-hoist-variables" "^7.22.5" - "@babel/helper-split-export-declaration" "^7.22.6" - "@babel/parser" "^7.22.11" - "@babel/types" "^7.22.11" + "@babel/parser" "^7.23.0" + "@babel/types" "^7.23.0" debug "^4.1.0" globals "^11.1.0" @@ -481,6 +464,15 @@ "@babel/helper-validator-identifier" "^7.22.15" to-fast-properties "^2.0.0" +"@babel/types@^7.23.0": + version "7.23.0" + resolved "https://registry.yarnpkg.com/@babel/types/-/types-7.23.0.tgz#8c1f020c9df0e737e4e247c0619f58c68458aaeb" + integrity sha512-0oIyUfKoI3mSqMvsxBdclDwxXKXAUA8v/apZbc+iSyARYou1o8ZGDxbUYyLFoW2arqS2jDGqJuZvv1d/io1axg== + dependencies: + "@babel/helper-string-parser" "^7.22.5" + "@babel/helper-validator-identifier" "^7.22.20" + to-fast-properties "^2.0.0" + "@bcoe/v8-coverage@^0.2.3": version "0.2.3" resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39" @@ -719,10 +711,10 @@ minimatch "^3.1.2" strip-json-comments "^3.1.1" -"@eslint/js@8.51.0": - version "8.51.0" - resolved "https://registry.yarnpkg.com/@eslint/js/-/js-8.51.0.tgz#6d419c240cfb2b66da37df230f7e7eef801c32fa" - integrity sha512-HxjQ8Qn+4SI3/AFv6sOrDB+g6PpUTDwSJiQqOrnneEk8L71161srI9gjzzZvYVbzHiVg/BvcH95+cK/zfIt4pg== +"@eslint/js@8.52.0": + version "8.52.0" + resolved "https://registry.yarnpkg.com/@eslint/js/-/js-8.52.0.tgz#78fe5f117840f69dc4a353adf9b9cd926353378c" + integrity sha512-mjZVbpaeMZludF2fsWLD0Z9gCref1Tk4i9+wddjRvpUNqqcndPkBD09N/Mapey0b3jaXbLm2kICwFv2E64QinA== "@floating-ui/core@^1.4.2": version "1.5.0" @@ -793,12 +785,12 @@ dependencies: prop-types "^15.8.1" -"@humanwhocodes/config-array@^0.11.11": - version "0.11.11" - resolved "https://registry.yarnpkg.com/@humanwhocodes/config-array/-/config-array-0.11.11.tgz#88a04c570dbbc7dd943e4712429c3df09bc32844" - integrity sha512-N2brEuAadi0CcdeMXUkhbZB84eskAc8MEX1By6qEchoVywSgXPIjou4rYsl0V3Hj0ZnuGycGCjdNgockbzeWNA== +"@humanwhocodes/config-array@^0.11.13": + version "0.11.13" + resolved "https://registry.yarnpkg.com/@humanwhocodes/config-array/-/config-array-0.11.13.tgz#075dc9684f40a531d9b26b0822153c1e832ee297" + integrity sha512-JSBDMiDKSzQVngfRjOdFXgFfklaXI4K9nLF49Auh21lmBWRLIK3+xTErTWD4KU54pb6coM6ESE7Awz/FNU3zgQ== dependencies: - "@humanwhocodes/object-schema" "^1.2.1" + "@humanwhocodes/object-schema" "^2.0.1" debug "^4.1.1" minimatch "^3.0.5" @@ -807,10 +799,10 @@ resolved "https://registry.yarnpkg.com/@humanwhocodes/module-importer/-/module-importer-1.0.1.tgz#af5b2691a22b44be847b0ca81641c5fb6ad0172c" integrity sha512-bxveV4V8v5Yb4ncFTT3rPSgZBOpCkjfK0y4oVVVJwIuDVBRMDXrPyXRL988i5ap9m9bnyEEjWfm5WkBmtffLfA== -"@humanwhocodes/object-schema@^1.2.1": - version "1.2.1" - resolved "https://registry.yarnpkg.com/@humanwhocodes/object-schema/-/object-schema-1.2.1.tgz#b520529ec21d8e5945a1851dfd1c32e94e39ff45" - integrity sha512-ZnQMnLV4e7hDlUvw8H+U8ASL02SS2Gn6+9Ac3wGGLIe7+je2AeAOxPY+izIPJDfFDb7eDjev0Us8MO1iFRN8hA== +"@humanwhocodes/object-schema@^2.0.1": + version "2.0.1" + resolved "https://registry.yarnpkg.com/@humanwhocodes/object-schema/-/object-schema-2.0.1.tgz#e5211452df060fa8522b55c7b3c0c4d1981cb044" + integrity sha512-dvuCeX5fC9dXgJn9t+X5atfmgQAzUOWqS1254Gh0m6i8wKd10ebXkfNKiRK+1GWi/yTvvLDHpoxLr0xxxeslWw== "@istanbuljs/load-nyc-config@^1.0.0": version "1.1.0" @@ -1245,10 +1237,10 @@ dependencies: "@babel/runtime" "^7.13.10" -"@remix-run/router@1.9.0": - version "1.9.0" - resolved "https://registry.yarnpkg.com/@remix-run/router/-/router-1.9.0.tgz#9033238b41c4cbe1e961eccb3f79e2c588328cf6" - integrity sha512-bV63itrKBC0zdT27qYm6SDZHlkXwFL1xMBuhkn+X7l0+IIhNaH5wuuvZKp6eKhCD4KFhujhfhCT1YxXW6esUIA== +"@remix-run/router@1.10.0": + version "1.10.0" + resolved "https://registry.yarnpkg.com/@remix-run/router/-/router-1.10.0.tgz#e2170dc2049b06e65bbe883adad0e8ddf8291278" + integrity sha512-Lm+fYpMfZoEucJ7cMxgt4dYt8jLfbpwRCzAjm9UgSLOkmlqo9gupxt6YX3DY0Fk155NT9l17d/ydi+964uS9Lw== "@sinclair/typebox@^0.27.8": version "0.27.8" @@ -1390,6 +1382,11 @@ dependencies: "@types/yargs-parser" "*" +"@ungap/structured-clone@^1.2.0": + version "1.2.0" + resolved "https://registry.yarnpkg.com/@ungap/structured-clone/-/structured-clone-1.2.0.tgz#756641adb587851b5ccb3e095daf27ae581c8406" + integrity sha512-zuVdFrMJiuCDQUMCzQaD6KL28MjnqqN8XnAqiEq9PNm/hCPTSGfrXCOfwj1ow4LFb/tNymJPwsNbVePc1xFqrQ== + "@vitejs/plugin-react@^4": version "4.1.0" resolved "https://registry.yarnpkg.com/@vitejs/plugin-react/-/plugin-react-4.1.0.tgz#e4f56f46fd737c5d386bb1f1ade86ba275fe09bd" @@ -1526,6 +1523,17 @@ array-includes@^3.1.6: get-intrinsic "^1.1.3" is-string "^1.0.7" +array-includes@^3.1.7: + version "3.1.7" + resolved "https://registry.yarnpkg.com/array-includes/-/array-includes-3.1.7.tgz#8cd2e01b26f7a3086cbc87271593fe921c62abda" + integrity sha512-dlcsNBIiWhPkHdOEEKnehA+RNUWDc4UqFtnIXU4uuYDPtA4LDkr7qip2p0VvFAEXNDr0yWZ9PJyIRiGjRLQzwQ== + dependencies: + call-bind "^1.0.2" + define-properties "^1.2.0" + es-abstract "^1.22.1" + get-intrinsic "^1.2.1" + is-string "^1.0.7" + array-union@^2.1.0: version "2.1.0" resolved "https://registry.yarnpkg.com/array-union/-/array-union-2.1.0.tgz#b798420adbeb1de828d84acd8a2e23d3efe85e8d" @@ -1536,16 +1544,16 @@ array-unique@^0.3.2: resolved "https://registry.yarnpkg.com/array-unique/-/array-unique-0.3.2.tgz#a894b75d4bc4f6cd679ef3244a9fd8f46ae2d428" integrity sha512-SleRWjh9JUud2wH1hPs9rZBZ33H6T9HOiL0uwGnGx9FpE6wKGyfWugmbkEOIs6qWrZhg0LWeLziLrEwQJhs5mQ== -array.prototype.findlastindex@^1.2.2: - version "1.2.2" - resolved "https://registry.yarnpkg.com/array.prototype.findlastindex/-/array.prototype.findlastindex-1.2.2.tgz#bc229aef98f6bd0533a2bc61ff95209875526c9b" - integrity sha512-tb5thFFlUcp7NdNF6/MpDk/1r/4awWG1FIz3YqDf+/zJSTezBb+/5WViH41obXULHVpDzoiCLpJ/ZO9YbJMsdw== +array.prototype.findlastindex@^1.2.3: + version "1.2.3" + resolved "https://registry.yarnpkg.com/array.prototype.findlastindex/-/array.prototype.findlastindex-1.2.3.tgz#b37598438f97b579166940814e2c0493a4f50207" + integrity sha512-LzLoiOMAxvy+Gd3BAq3B7VeIgPdo+Q8hthvKtXybMvRV0jrXfJM/t8mw7nNlpEcVlVUnCnM2KSX4XU5HmpodOA== dependencies: call-bind "^1.0.2" - define-properties "^1.1.4" - es-abstract "^1.20.4" + define-properties "^1.2.0" + es-abstract "^1.22.1" es-shim-unscopables "^1.0.0" - get-intrinsic "^1.1.3" + get-intrinsic "^1.2.1" array.prototype.flat@^1.3.1: version "1.3.1" @@ -1557,6 +1565,16 @@ array.prototype.flat@^1.3.1: es-abstract "^1.20.4" es-shim-unscopables "^1.0.0" +array.prototype.flat@^1.3.2: + version "1.3.2" + resolved "https://registry.yarnpkg.com/array.prototype.flat/-/array.prototype.flat-1.3.2.tgz#1476217df8cff17d72ee8f3ba06738db5b387d18" + integrity sha512-djYB+Zx2vLewY8RWlNCUdHjDXs2XOgm602S9E7P/UpHgfeHL00cRiIF+IN/G/aUJ7kGPb6yO/ErDI5V2s8iycA== + dependencies: + call-bind "^1.0.2" + define-properties "^1.2.0" + es-abstract "^1.22.1" + es-shim-unscopables "^1.0.0" + array.prototype.flatmap@^1.3.1: version "1.3.1" resolved "https://registry.yarnpkg.com/array.prototype.flatmap/-/array.prototype.flatmap-1.3.1.tgz#1aae7903c2100433cb8261cd4ed310aab5c4a183" @@ -1567,6 +1585,16 @@ array.prototype.flatmap@^1.3.1: es-abstract "^1.20.4" es-shim-unscopables "^1.0.0" +array.prototype.flatmap@^1.3.2: + version "1.3.2" + resolved "https://registry.yarnpkg.com/array.prototype.flatmap/-/array.prototype.flatmap-1.3.2.tgz#c9a7c6831db8e719d6ce639190146c24bbd3e527" + integrity sha512-Ewyx0c9PmpcsByhSW4r+9zDU7sGjFc86qf/kKtuSCRdhfbk0SNLLkaT5qvcHnRGgc5NP/ly/y+qkXkqONX54CQ== + dependencies: + call-bind "^1.0.2" + define-properties "^1.2.0" + es-abstract "^1.22.1" + es-shim-unscopables "^1.0.0" + array.prototype.tosorted@^1.1.1: version "1.1.1" resolved "https://registry.yarnpkg.com/array.prototype.tosorted/-/array.prototype.tosorted-1.1.1.tgz#ccf44738aa2b5ac56578ffda97c03fd3e23dd532" @@ -1578,14 +1606,15 @@ array.prototype.tosorted@^1.1.1: es-shim-unscopables "^1.0.0" get-intrinsic "^1.1.3" -arraybuffer.prototype.slice@^1.0.1: - version "1.0.1" - resolved "https://registry.yarnpkg.com/arraybuffer.prototype.slice/-/arraybuffer.prototype.slice-1.0.1.tgz#9b5ea3868a6eebc30273da577eb888381c0044bb" - integrity sha512-09x0ZWFEjj4WD8PDbykUwo3t9arLn8NIzmmYEJFpYekOAQjpkGSyrQhNoRTcwwcFRu+ycWF78QZ63oWTqSjBcw== +arraybuffer.prototype.slice@^1.0.2: + version "1.0.2" + resolved "https://registry.yarnpkg.com/arraybuffer.prototype.slice/-/arraybuffer.prototype.slice-1.0.2.tgz#98bd561953e3e74bb34938e77647179dfe6e9f12" + integrity sha512-yMBKppFur/fbHu9/6USUe03bZ4knMYiwFBcyiaXB8Go0qNehwX6inYPzK9U0NeQvGxKthcmHcaR8P5MStSRBAw== dependencies: array-buffer-byte-length "^1.0.0" call-bind "^1.0.2" define-properties "^1.2.0" + es-abstract "^1.22.1" get-intrinsic "^1.2.1" is-array-buffer "^3.0.2" is-shared-array-buffer "^1.0.2" @@ -1823,13 +1852,14 @@ cache-base@^1.0.1: union-value "^1.0.0" unset-value "^1.0.0" -call-bind@^1.0.0, call-bind@^1.0.2: - version "1.0.2" - resolved "https://registry.yarnpkg.com/call-bind/-/call-bind-1.0.2.tgz#b1d4e89e688119c3c9a903ad30abb2f6a919be3c" - integrity sha512-7O+FbCihrB5WGbFYesctwmTKae6rOiIzmz1icreWJ+0aA7LJfuqhEso2T9ncpcFtzMQtzXf2QGGueWJGTYsqrA== +call-bind@^1.0.0, call-bind@^1.0.2, call-bind@^1.0.4, call-bind@^1.0.5: + version "1.0.5" + resolved "https://registry.yarnpkg.com/call-bind/-/call-bind-1.0.5.tgz#6fa2b7845ce0ea49bf4d8b9ef64727a2c2e2e513" + integrity sha512-C3nQxfFZxFRVoJoGKKI8y3MOEo129NQ+FgQ08iye+Mk4zNZZGdjfs06bVTr+DBSlA66Q2VEcMki/cUCP4SercQ== dependencies: - function-bind "^1.1.1" - get-intrinsic "^1.0.2" + function-bind "^1.1.2" + get-intrinsic "^1.2.1" + set-function-length "^1.1.1" callsites@^3.0.0: version "3.1.0" @@ -2107,16 +2137,26 @@ default-browser@^4.0.0: execa "^7.1.1" titleize "^3.0.0" +define-data-property@^1.0.1, define-data-property@^1.1.1: + version "1.1.1" + resolved "https://registry.yarnpkg.com/define-data-property/-/define-data-property-1.1.1.tgz#c35f7cd0ab09883480d12ac5cb213715587800b3" + integrity sha512-E7uGkTzkk1d0ByLeSc6ZsFS79Axg+m1P/VsgYsxHgiuc3tFSj+MjMIwe90FC4lOAZzNBdY7kkO2P2wKdsQ1vgQ== + dependencies: + get-intrinsic "^1.2.1" + gopd "^1.0.1" + has-property-descriptors "^1.0.0" + define-lazy-prop@^3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/define-lazy-prop/-/define-lazy-prop-3.0.0.tgz#dbb19adfb746d7fc6d734a06b72f4a00d021255f" integrity sha512-N+MeXYoqr3pOgn8xfyRPREN7gHakLYjhsHhWGT3fWAiL4IkAt0iDw14QiiEm2bE30c5XX5q0FtAA3CK5f9/BUg== define-properties@^1.1.3, define-properties@^1.1.4, define-properties@^1.2.0: - version "1.2.0" - resolved "https://registry.yarnpkg.com/define-properties/-/define-properties-1.2.0.tgz#52988570670c9eacedd8064f4a990f2405849bd5" - integrity sha512-xvqAVKGfT1+UAvPwKTVw/njhdQ8ZhXK4lI0bCIuCMrp2up9nPnaDftrLtmpTazqd1o+UY4zgzU+avtMbDP+ldA== + version "1.2.1" + resolved "https://registry.yarnpkg.com/define-properties/-/define-properties-1.2.1.tgz#10781cc616eb951a80a034bafcaa7377f6af2b6c" + integrity sha512-8QmQKqEASLd5nx0U1B1okLElbUuuttJ/AnYmRXbbbGDWh6uS208EjD4Xqq/I9wK7u0v6O08XhTWnt5XtEbR6Dg== dependencies: + define-data-property "^1.0.1" has-property-descriptors "^1.0.0" object-keys "^1.1.1" @@ -2209,25 +2249,25 @@ error-ex@^1.3.1: is-arrayish "^0.2.1" es-abstract@^1.20.4, es-abstract@^1.22.1: - version "1.22.1" - resolved "https://registry.yarnpkg.com/es-abstract/-/es-abstract-1.22.1.tgz#8b4e5fc5cefd7f1660f0f8e1a52900dfbc9d9ccc" - integrity sha512-ioRRcXMO6OFyRpyzV3kE1IIBd4WG5/kltnzdxSCqoP8CMGs/Li+M1uF5o7lOkZVFjDs+NLesthnF66Pg/0q0Lw== + version "1.22.3" + resolved "https://registry.yarnpkg.com/es-abstract/-/es-abstract-1.22.3.tgz#48e79f5573198de6dee3589195727f4f74bc4f32" + integrity sha512-eiiY8HQeYfYH2Con2berK+To6GrK2RxbPawDkGq4UiCQQfZHb6wX9qQqkbpPqaxQFcl8d9QzZqo0tGE0VcrdwA== dependencies: array-buffer-byte-length "^1.0.0" - arraybuffer.prototype.slice "^1.0.1" + arraybuffer.prototype.slice "^1.0.2" available-typed-arrays "^1.0.5" - call-bind "^1.0.2" + call-bind "^1.0.5" es-set-tostringtag "^2.0.1" es-to-primitive "^1.2.1" - function.prototype.name "^1.1.5" - get-intrinsic "^1.2.1" + function.prototype.name "^1.1.6" + get-intrinsic "^1.2.2" get-symbol-description "^1.0.0" globalthis "^1.0.3" gopd "^1.0.1" - has "^1.0.3" has-property-descriptors "^1.0.0" has-proto "^1.0.1" has-symbols "^1.0.3" + hasown "^2.0.0" internal-slot "^1.0.5" is-array-buffer "^3.0.2" is-callable "^1.2.7" @@ -2235,23 +2275,23 @@ es-abstract@^1.20.4, es-abstract@^1.22.1: is-regex "^1.1.4" is-shared-array-buffer "^1.0.2" is-string "^1.0.7" - is-typed-array "^1.1.10" + is-typed-array "^1.1.12" is-weakref "^1.0.2" - object-inspect "^1.12.3" + object-inspect "^1.13.1" object-keys "^1.1.1" object.assign "^4.1.4" - regexp.prototype.flags "^1.5.0" - safe-array-concat "^1.0.0" + regexp.prototype.flags "^1.5.1" + safe-array-concat "^1.0.1" safe-regex-test "^1.0.0" - string.prototype.trim "^1.2.7" - string.prototype.trimend "^1.0.6" - string.prototype.trimstart "^1.0.6" + string.prototype.trim "^1.2.8" + string.prototype.trimend "^1.0.7" + string.prototype.trimstart "^1.0.7" typed-array-buffer "^1.0.0" typed-array-byte-length "^1.0.0" typed-array-byte-offset "^1.0.0" typed-array-length "^1.0.4" unbox-primitive "^1.0.2" - which-typed-array "^1.1.10" + which-typed-array "^1.1.13" es-iterator-helpers@^1.0.12: version "1.0.14" @@ -2274,20 +2314,20 @@ es-iterator-helpers@^1.0.12: safe-array-concat "^1.0.0" es-set-tostringtag@^2.0.1: - version "2.0.1" - resolved "https://registry.yarnpkg.com/es-set-tostringtag/-/es-set-tostringtag-2.0.1.tgz#338d502f6f674301d710b80c8592de8a15f09cd8" - integrity sha512-g3OMbtlwY3QewlqAiMLI47KywjWZoEytKr8pf6iTC8uJq5bIAH52Z9pnQ8pVL6whrCto53JZDuUIsifGeLorTg== + version "2.0.2" + resolved "https://registry.yarnpkg.com/es-set-tostringtag/-/es-set-tostringtag-2.0.2.tgz#11f7cc9f63376930a5f20be4915834f4bc74f9c9" + integrity sha512-BuDyupZt65P9D2D2vA/zqcI3G5xRsklm5N3xCwuiy+/vKy8i0ifdsQP1sLgO4tZDSCaQUSnmC48khknGMV3D2Q== dependencies: - get-intrinsic "^1.1.3" - has "^1.0.3" + get-intrinsic "^1.2.2" has-tostringtag "^1.0.0" + hasown "^2.0.0" es-shim-unscopables@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/es-shim-unscopables/-/es-shim-unscopables-1.0.0.tgz#702e632193201e3edf8713635d083d378e510241" - integrity sha512-Jm6GPcCdC30eMLbZ2x8z2WuRwAws3zTBBKuusffYVUrNj/GVSUAZ+xKMaUpfNDR5IbyNA5LJbaecoUVbmUcB1w== + version "1.0.2" + resolved "https://registry.yarnpkg.com/es-shim-unscopables/-/es-shim-unscopables-1.0.2.tgz#1f6942e71ecc7835ed1c8a83006d8771a63a3763" + integrity sha512-J3yBRXCzDu4ULnQwxyToo/OjdMx6akgVC7K6few0a7F/0wLtmKKN7I73AH5T2836UuXRqN7Qg+IIUw/+YJksRw== dependencies: - has "^1.0.3" + hasown "^2.0.0" es-to-primitive@^1.2.1: version "1.2.1" @@ -2355,7 +2395,7 @@ escape-string-regexp@^4.0.0: resolved "https://registry.yarnpkg.com/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz#14ba83a5d373e3d311e5afca29cf5bfad965bf34" integrity sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA== -eslint-import-resolver-node@^0.3.7: +eslint-import-resolver-node@^0.3.9: version "0.3.9" resolved "https://registry.yarnpkg.com/eslint-import-resolver-node/-/eslint-import-resolver-node-0.3.9.tgz#d4eaac52b8a2e7c3cd1903eb00f7e053356118ac" integrity sha512-WFj2isz22JahUv+B788TlO3N6zL3nNJGU8CcZbPZvVEkBPaJdCV4vy5wyghty5ROFbCRnm132v8BScu5/1BQ8g== @@ -2372,25 +2412,25 @@ eslint-module-utils@^2.8.0: debug "^3.2.7" eslint-plugin-import@^2: - version "2.28.1" - resolved "https://registry.yarnpkg.com/eslint-plugin-import/-/eslint-plugin-import-2.28.1.tgz#63b8b5b3c409bfc75ebaf8fb206b07ab435482c4" - integrity sha512-9I9hFlITvOV55alzoKBI+K9q74kv0iKMeY6av5+umsNwayt59fz692daGyjR+oStBQgx6nwR9rXldDev3Clw+A== - dependencies: - array-includes "^3.1.6" - array.prototype.findlastindex "^1.2.2" - array.prototype.flat "^1.3.1" - array.prototype.flatmap "^1.3.1" + version "2.29.0" + resolved "https://registry.yarnpkg.com/eslint-plugin-import/-/eslint-plugin-import-2.29.0.tgz#8133232e4329ee344f2f612885ac3073b0b7e155" + integrity sha512-QPOO5NO6Odv5lpoTkddtutccQjysJuFxoPS7fAHO+9m9udNHvTCPSAMW9zGAYj8lAIdr40I8yPCdUYrncXtrwg== + dependencies: + array-includes "^3.1.7" + array.prototype.findlastindex "^1.2.3" + array.prototype.flat "^1.3.2" + array.prototype.flatmap "^1.3.2" debug "^3.2.7" doctrine "^2.1.0" - eslint-import-resolver-node "^0.3.7" + eslint-import-resolver-node "^0.3.9" eslint-module-utils "^2.8.0" - has "^1.0.3" - is-core-module "^2.13.0" + hasown "^2.0.0" + is-core-module "^2.13.1" is-glob "^4.0.3" minimatch "^3.1.2" - object.fromentries "^2.0.6" - object.groupby "^1.0.0" - object.values "^1.1.6" + object.fromentries "^2.0.7" + object.groupby "^1.0.1" + object.values "^1.1.7" semver "^6.3.1" tsconfig-paths "^3.14.2" @@ -2460,17 +2500,18 @@ eslint-visitor-keys@^3.3.0, eslint-visitor-keys@^3.4.1, eslint-visitor-keys@^3.4 integrity sha512-wpc+LXeiyiisxPlEkUzU6svyS1frIO3Mgxj1fdy7Pm8Ygzguax2N3Fa/D/ag1WqbOprdI+uY6wMUl8/a2G+iag== eslint@^8: - version "8.51.0" - resolved "https://registry.yarnpkg.com/eslint/-/eslint-8.51.0.tgz#4a82dae60d209ac89a5cff1604fea978ba4950f3" - integrity sha512-2WuxRZBrlwnXi+/vFSJyjMqrNjtJqiasMzehF0shoLaW7DzS3/9Yvrmq5JiT66+pNjiX4UBnLDiKHcWAr/OInA== + version "8.52.0" + resolved "https://registry.yarnpkg.com/eslint/-/eslint-8.52.0.tgz#d0cd4a1fac06427a61ef9242b9353f36ea7062fc" + integrity sha512-zh/JHnaixqHZsolRB/w9/02akBk9EPrOs9JwcTP2ek7yL5bVvXuRariiaAjjoJ5DvuwQ1WAE/HsMz+w17YgBCg== dependencies: "@eslint-community/eslint-utils" "^4.2.0" "@eslint-community/regexpp" "^4.6.1" "@eslint/eslintrc" "^2.1.2" - "@eslint/js" "8.51.0" - "@humanwhocodes/config-array" "^0.11.11" + "@eslint/js" "8.52.0" + "@humanwhocodes/config-array" "^0.11.13" "@humanwhocodes/module-importer" "^1.0.1" "@nodelib/fs.walk" "^1.2.8" + "@ungap/structured-clone" "^1.2.0" ajv "^6.12.4" chalk "^4.0.0" cross-spawn "^7.0.2" @@ -2799,12 +2840,12 @@ fsevents@^2.3.2, fsevents@~2.3.2: resolved "https://registry.yarnpkg.com/fsevents/-/fsevents-2.3.3.tgz#cac6407785d03675a2a5e1a5305c697b347d90d6" integrity sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw== -function-bind@^1.1.1: - version "1.1.1" - resolved "https://registry.yarnpkg.com/function-bind/-/function-bind-1.1.1.tgz#a56899d3ea3c9bab874bb9773b7c5ede92f4895d" - integrity sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A== +function-bind@^1.1.1, function-bind@^1.1.2: + version "1.1.2" + resolved "https://registry.yarnpkg.com/function-bind/-/function-bind-1.1.2.tgz#2c02d864d97f3ea6c8830c464cbd11ab6eab7a1c" + integrity sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA== -function.prototype.name@^1.1.5: +function.prototype.name@^1.1.5, function.prototype.name@^1.1.6: version "1.1.6" resolved "https://registry.yarnpkg.com/function.prototype.name/-/function.prototype.name-1.1.6.tgz#cdf315b7d90ee77a4c6ee216c3c3362da07533fd" integrity sha512-Z5kx79swU5P27WEayXM1tBi5Ze/lbIyiNgU3qyXUOf9b2rgXYyF9Dy9Cx+IQv/Lc8WCG6L82zwUPpSS9hGehIg== @@ -2829,15 +2870,15 @@ get-caller-file@^2.0.5: resolved "https://registry.yarnpkg.com/get-caller-file/-/get-caller-file-2.0.5.tgz#4f94412a82db32f36e3b0b9741f8a97feb031f7e" integrity sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg== -get-intrinsic@^1.0.2, get-intrinsic@^1.1.1, get-intrinsic@^1.1.3, get-intrinsic@^1.2.0, get-intrinsic@^1.2.1: - version "1.2.1" - resolved "https://registry.yarnpkg.com/get-intrinsic/-/get-intrinsic-1.2.1.tgz#d295644fed4505fc9cde952c37ee12b477a83d82" - integrity sha512-2DcsyfABl+gVHEfCOaTrWgyt+tb6MSEGmKq+kI5HwLbIYgjgmMcV8KQ41uaKz1xxUcn9tJtgFbQUEVcEbd0FYw== +get-intrinsic@^1.0.2, get-intrinsic@^1.1.1, get-intrinsic@^1.1.3, get-intrinsic@^1.2.0, get-intrinsic@^1.2.1, get-intrinsic@^1.2.2: + version "1.2.2" + resolved "https://registry.yarnpkg.com/get-intrinsic/-/get-intrinsic-1.2.2.tgz#281b7622971123e1ef4b3c90fd7539306da93f3b" + integrity sha512-0gSo4ml/0j98Y3lngkFEot/zhiCeWsbYIlZ+uZOVgzLyLaUw7wxUL+nCTP0XJvJg1AXulJRI3UJi8GsbDuxdGA== dependencies: - function-bind "^1.1.1" - has "^1.0.3" + function-bind "^1.1.2" has-proto "^1.0.1" has-symbols "^1.0.3" + hasown "^2.0.0" get-nonce@^1.0.0: version "1.0.1" @@ -2959,11 +3000,11 @@ has-flag@^4.0.0: integrity sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ== has-property-descriptors@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/has-property-descriptors/-/has-property-descriptors-1.0.0.tgz#610708600606d36961ed04c196193b6a607fa861" - integrity sha512-62DVLZGoiEBDHQyqG4w9xCuZ7eJEwNmJRWw2VY84Oedb7WFcA27fiEVe8oUQx9hAUJ4ekurquucTGwsyO1XGdQ== + version "1.0.1" + resolved "https://registry.yarnpkg.com/has-property-descriptors/-/has-property-descriptors-1.0.1.tgz#52ba30b6c5ec87fd89fa574bc1c39125c6f65340" + integrity sha512-VsX8eaIewvas0xnvinAe9bw4WfIeODpGYikiWYLH+dma0Jw6KHYqWiWfhQlgOVK8D6PvjubK5Uc4P0iIhIcNVg== dependencies: - get-intrinsic "^1.1.1" + get-intrinsic "^1.2.2" has-proto@^1.0.1: version "1.0.1" @@ -3014,11 +3055,16 @@ has-values@^1.0.0: kind-of "^4.0.0" has@^1.0.3: - version "1.0.3" - resolved "https://registry.yarnpkg.com/has/-/has-1.0.3.tgz#722d7cbfc1f6aa8241f16dd814e011e1f41e8796" - integrity sha512-f2dvO0VU6Oej7RkWJGrehjbzMAjFp5/VKPp5tTpWIV4JHHZK1/BxbFRtf/siA2SWTe09caDmVtYYzWEIbBS4zw== + version "1.0.4" + resolved "https://registry.yarnpkg.com/has/-/has-1.0.4.tgz#2eb2860e000011dae4f1406a86fe80e530fb2ec6" + integrity sha512-qdSAmqLF6209RFj4VVItywPMbm3vWylknmB3nvNiUIs72xAimcM8nVYxYr7ncvZq5qzk9MKIZR8ijqD/1QuYjQ== + +hasown@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/hasown/-/hasown-2.0.0.tgz#f4c513d454a57b7c7e1650778de226b11700546c" + integrity sha512-vUptKVTpIJhcczKBbgnS+RtcuYMB8+oNzPK2/Hp3hanz8JmpATdmmgLgSaadVREkDm+e2giHwY3ZRkyjSIDDFA== dependencies: - function-bind "^1.1.1" + function-bind "^1.1.2" hoist-non-react-statics@^3.3.1: version "3.3.2" @@ -3091,7 +3137,7 @@ inherits@2: resolved "https://registry.yarnpkg.com/inherits/-/inherits-2.0.4.tgz#0fa2c64f932917c3433a0ded55363aae37416b7c" integrity sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ== -internal-slot@^1.0.3, internal-slot@^1.0.5: +internal-slot@^1.0.3: version "1.0.5" resolved "https://registry.yarnpkg.com/internal-slot/-/internal-slot-1.0.5.tgz#f2a2ee21f668f8627a4667f309dc0f4fb6674986" integrity sha512-Y+R5hJrzs52QCG2laLn4udYVnxsfny9CpOhNhUvk/SSSVyF6T27FzRbF0sroPidSu3X8oEAkOn2K804mjpt6UQ== @@ -3100,6 +3146,15 @@ internal-slot@^1.0.3, internal-slot@^1.0.5: has "^1.0.3" side-channel "^1.0.4" +internal-slot@^1.0.5: + version "1.0.6" + resolved "https://registry.yarnpkg.com/internal-slot/-/internal-slot-1.0.6.tgz#37e756098c4911c5e912b8edbf71ed3aa116f930" + integrity sha512-Xj6dv+PsbtwyPpEflsejS+oIZxmMlV44zAhG479uYu89MsjcYOhCFnNyKrkJrihbsiasQyY0afoCl/9BLR65bg== + dependencies: + get-intrinsic "^1.2.2" + hasown "^2.0.0" + side-channel "^1.0.4" + invariant@^2.2.4: version "2.2.4" resolved "https://registry.yarnpkg.com/invariant/-/invariant-2.2.4.tgz#610f3c92c9359ce1db616e538008d23ff35158e6" @@ -3174,13 +3229,20 @@ is-ci@^2.0.0: dependencies: ci-info "^2.0.0" -is-core-module@^2.11.0, is-core-module@^2.13.0, is-core-module@^2.9.0: +is-core-module@^2.11.0, is-core-module@^2.9.0: version "2.13.0" resolved "https://registry.yarnpkg.com/is-core-module/-/is-core-module-2.13.0.tgz#bb52aa6e2cbd49a30c2ba68c42bf3435ba6072db" integrity sha512-Z7dk6Qo8pOCp3l4tsX2C5ZVas4V+UxwQodwZhLopL91TX8UyyHEXafPcyoeeWuLrwzHcr3igO78wNLwHJHsMCQ== dependencies: has "^1.0.3" +is-core-module@^2.13.0, is-core-module@^2.13.1: + version "2.13.1" + resolved "https://registry.yarnpkg.com/is-core-module/-/is-core-module-2.13.1.tgz#ad0d7532c6fea9da1ebdc82742d74525c6273384" + integrity sha512-hHrIjvZsftOsvKSn2TRYl63zvxsgE0K+0mYMoH6gD4omR5IWB2KynivBQczo3+wF1cCkjzvptnI9Q0sPU66ilw== + dependencies: + hasown "^2.0.0" + is-data-descriptor@^0.1.4: version "0.1.4" resolved "https://registry.yarnpkg.com/is-data-descriptor/-/is-data-descriptor-0.1.4.tgz#0b5ee648388e2c860282e793f1856fec3f301b56" @@ -3375,7 +3437,7 @@ is-symbol@^1.0.2, is-symbol@^1.0.3: dependencies: has-symbols "^1.0.2" -is-typed-array@^1.1.10, is-typed-array@^1.1.9: +is-typed-array@^1.1.10, is-typed-array@^1.1.12, is-typed-array@^1.1.9: version "1.1.12" resolved "https://registry.yarnpkg.com/is-typed-array/-/is-typed-array-1.1.12.tgz#d0bab5686ef4a76f7a73097b95470ab199c57d4a" integrity sha512-Z14TF2JNG8Lss5/HMqt0//T9JeHXttXy5pH/DBU4vi98ozO2btxzq9MwYDZYnKwU8nRsz/+GVFVRDq3DkVuSPg== @@ -3991,9 +4053,9 @@ json5@^2.2.2, json5@^2.2.3: object.values "^1.1.6" keyv@^4.5.3: - version "4.5.3" - resolved "https://registry.yarnpkg.com/keyv/-/keyv-4.5.3.tgz#00873d2b046df737963157bd04f294ca818c9c25" - integrity sha512-QCiSav9WaX1PgETJ+SpNnx2PRRapJ/oRSXM4VO5OGYGSjrxbKPVFVhB3l2OCbLCk329N8qyAtsJjSjvVBWzEug== + version "4.5.4" + resolved "https://registry.yarnpkg.com/keyv/-/keyv-4.5.4.tgz#a879a99e29452f942439f2a405e3af8b31d4de93" + integrity sha512-oxVHkHR/EJf2CNXnWxRLW6mg7JyCCUcG0DtEGmL2ctUo1PNTin1PUil+r/+4r5MpVgC/fn1kjsx7mjSujKqIpw== dependencies: json-buffer "3.0.1" @@ -4302,10 +4364,10 @@ object-copy@^0.1.0: define-property "^0.2.5" kind-of "^3.0.3" -object-inspect@^1.12.3, object-inspect@^1.9.0: - version "1.12.3" - resolved "https://registry.yarnpkg.com/object-inspect/-/object-inspect-1.12.3.tgz#ba62dffd67ee256c8c086dfae69e016cd1f198b9" - integrity sha512-geUvdk7c+eizMNUDkRpW1wJwgfOiOeHbxBR/hLXK1aT6zmVSO0jsQcs7fj6MGw89jC/cjGfLcNOrtMYtGqm81g== +object-inspect@^1.13.1, object-inspect@^1.9.0: + version "1.13.1" + resolved "https://registry.yarnpkg.com/object-inspect/-/object-inspect-1.13.1.tgz#b96c6109324ccfef6b12216a956ca4dc2ff94bc2" + integrity sha512-5qoj1RUiKOMsCCNLV1CBiPYE10sziTsnmNxkAI/rZhiD63CF7IqdFGC/XzjWjpSgLf0LxXX3bDFIh0E18f6UhQ== object-keys@^1.1.1: version "1.1.1" @@ -4347,7 +4409,16 @@ object.fromentries@^2.0.6: define-properties "^1.1.4" es-abstract "^1.20.4" -object.groupby@^1.0.0: +object.fromentries@^2.0.7: + version "2.0.7" + resolved "https://registry.yarnpkg.com/object.fromentries/-/object.fromentries-2.0.7.tgz#71e95f441e9a0ea6baf682ecaaf37fa2a8d7e616" + integrity sha512-UPbPHML6sL8PI/mOqPwsH4G6iyXcCGzLin8KvEPenOZN5lpCNBZZQ+V62vdjB1mQHrmqGQt5/OJzemUA+KJmEA== + dependencies: + call-bind "^1.0.2" + define-properties "^1.2.0" + es-abstract "^1.22.1" + +object.groupby@^1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/object.groupby/-/object.groupby-1.0.1.tgz#d41d9f3c8d6c778d9cbac86b4ee9f5af103152ee" integrity sha512-HqaQtqLnp/8Bn4GL16cj+CUYbnpe1bh0TtEaWvybszDG4tgxCJuRpV8VGuvNaI1fAnI4lUJzDG55MXcOH4JZcQ== @@ -4372,7 +4443,7 @@ object.pick@^1.3.0: dependencies: isobject "^3.0.1" -object.values@^1.1.6: +object.values@^1.1.6, object.values@^1.1.7: version "1.1.7" resolved "https://registry.yarnpkg.com/object.values/-/object.values-1.1.7.tgz#617ed13272e7e1071b43973aa1655d9291b8442a" integrity sha512-aU6xnDFYT3x17e/f0IiiwlGPTy2jzMySGfUB4fq6z7CV8l85CWHDk5ErhyhpfDHhrOMwGFhSQkhMGHaIotA6Ng== @@ -4676,19 +4747,19 @@ react-remove-scroll@^2.5.5: use-sidecar "^1.1.2" react-router-dom@^6: - version "6.16.0" - resolved "https://registry.yarnpkg.com/react-router-dom/-/react-router-dom-6.16.0.tgz#86f24658da35eb66727e75ecbb1a029e33ee39d9" - integrity sha512-aTfBLv3mk/gaKLxgRDUPbPw+s4Y/O+ma3rEN1u8EgEpLpPe6gNjIsWt9rxushMHHMb7mSwxRGdGlGdvmFsyPIg== + version "6.17.0" + resolved "https://registry.yarnpkg.com/react-router-dom/-/react-router-dom-6.17.0.tgz#ea73f89186546c1cf72b10fcb7356d874321b2ad" + integrity sha512-qWHkkbXQX+6li0COUUPKAUkxjNNqPJuiBd27dVwQGDNsuFBdMbrS6UZ0CLYc4CsbdLYTckn4oB4tGDuPZpPhaQ== dependencies: - "@remix-run/router" "1.9.0" - react-router "6.16.0" + "@remix-run/router" "1.10.0" + react-router "6.17.0" -react-router@6.16.0: - version "6.16.0" - resolved "https://registry.yarnpkg.com/react-router/-/react-router-6.16.0.tgz#abbf3d5bdc9c108c9b822a18be10ee004096fb81" - integrity sha512-VT4Mmc4jj5YyjpOi5jOf0I+TYzGpvzERy4ckNSvSh2RArv8LLoCxlsZ2D+tc7zgjxcY34oTz2hZaeX5RVprKqA== +react-router@6.17.0: + version "6.17.0" + resolved "https://registry.yarnpkg.com/react-router/-/react-router-6.17.0.tgz#7b680c4cefbc425b57537eb9c73bedecbdc67c1e" + integrity sha512-YJR3OTJzi3zhqeJYADHANCGPUu9J+6fT5GLv82UWRGSxu6oJYCKVmxUcaBQuGm9udpWmPsvpme/CdHumqgsoaA== dependencies: - "@remix-run/router" "1.9.0" + "@remix-run/router" "1.10.0" react-style-singleton@^2.2.1: version "2.2.1" @@ -4755,7 +4826,7 @@ regex-not@^1.0.0, regex-not@^1.0.2: extend-shallow "^3.0.2" safe-regex "^1.1.0" -regexp.prototype.flags@^1.4.3, regexp.prototype.flags@^1.5.0: +regexp.prototype.flags@^1.4.3: version "1.5.0" resolved "https://registry.yarnpkg.com/regexp.prototype.flags/-/regexp.prototype.flags-1.5.0.tgz#fe7ce25e7e4cca8db37b6634c8a2c7009199b9cb" integrity sha512-0SutC3pNudRKgquxGoRGIz946MZVHqbNfPjBdxeOhBrdgDKlRoXmYLQN9xRbrR09ZXWeGAdPuif7egofn6v5LA== @@ -4764,6 +4835,15 @@ regexp.prototype.flags@^1.4.3, regexp.prototype.flags@^1.5.0: define-properties "^1.2.0" functions-have-names "^1.2.3" +regexp.prototype.flags@^1.5.1: + version "1.5.1" + resolved "https://registry.yarnpkg.com/regexp.prototype.flags/-/regexp.prototype.flags-1.5.1.tgz#90ce989138db209f81492edd734183ce99f9677e" + integrity sha512-sy6TXMN+hnP/wMy+ISxg3krXx7BAtWVO4UouuCN/ziM9UEne0euamVNafDfvC83bRNr95y0V5iijeDQFUNpvrg== + dependencies: + call-bind "^1.0.2" + define-properties "^1.2.0" + set-function-name "^2.0.0" + remove-trailing-separator@^1.0.1: version "1.1.0" resolved "https://registry.yarnpkg.com/remove-trailing-separator/-/remove-trailing-separator-1.1.0.tgz#c24bce2a283adad5bc3f58e0d48249b92379d8ef" @@ -4820,7 +4900,7 @@ resolve@^1.19.0: path-parse "^1.0.7" supports-preserve-symlinks-flag "^1.0.0" -resolve@^1.20.0, resolve@^1.22.4: +resolve@^1.20.0: version "1.22.4" resolved "https://registry.yarnpkg.com/resolve/-/resolve-1.22.4.tgz#1dc40df46554cdaf8948a486a10f6ba1e2026c34" integrity sha512-PXNdCiPqDqeUou+w1C2eTQbNfxKSuMxqTCuvlmmMsk1NWHL5fRrhY6Pl0qEYYc6+QqGClco1Qj8XnjPego4wfg== @@ -4829,6 +4909,15 @@ resolve@^1.20.0, resolve@^1.22.4: path-parse "^1.0.7" supports-preserve-symlinks-flag "^1.0.0" +resolve@^1.22.4: + version "1.22.8" + resolved "https://registry.yarnpkg.com/resolve/-/resolve-1.22.8.tgz#b6c87a9f2aa06dfab52e3d70ac8cde321fa5a48d" + integrity sha512-oKWePCxqpd6FlLvGV1VU0x7bkPmmCNolxzjMf4NczoDnQcIWrAF+cPtZn5i6n+RfD2d9i0tzpKnG6Yk168yIyw== + dependencies: + is-core-module "^2.13.0" + path-parse "^1.0.7" + supports-preserve-symlinks-flag "^1.0.0" + resolve@^2.0.0-next.4: version "2.0.0-next.4" resolved "https://registry.yarnpkg.com/resolve/-/resolve-2.0.0-next.4.tgz#3d37a113d6429f496ec4752d2a2e58efb1fd4660" @@ -4881,13 +4970,13 @@ run-parallel@^1.1.9: dependencies: queue-microtask "^1.2.2" -safe-array-concat@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/safe-array-concat/-/safe-array-concat-1.0.0.tgz#2064223cba3c08d2ee05148eedbc563cd6d84060" - integrity sha512-9dVEFruWIsnie89yym+xWTAYASdpw3CJV7Li/6zBewGf9z2i1j31rP6jnY0pHEO4QZh6N0K11bFjWmdR8UGdPQ== +safe-array-concat@^1.0.0, safe-array-concat@^1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/safe-array-concat/-/safe-array-concat-1.0.1.tgz#91686a63ce3adbea14d61b14c99572a8ff84754c" + integrity sha512-6XbUAseYE2KtOuGueyeobCySj9L4+66Tn6KQMOPQJrAJEowYKW/YR/MGJZl7FdydUdaFu4LYyDZjxf4/Nmo23Q== dependencies: call-bind "^1.0.2" - get-intrinsic "^1.2.0" + get-intrinsic "^1.2.1" has-symbols "^1.0.3" isarray "^2.0.5" @@ -4946,6 +5035,25 @@ semver@^7.5.3, semver@^7.5.4: dependencies: lru-cache "^6.0.0" +set-function-length@^1.1.1: + version "1.1.1" + resolved "https://registry.yarnpkg.com/set-function-length/-/set-function-length-1.1.1.tgz#4bc39fafb0307224a33e106a7d35ca1218d659ed" + integrity sha512-VoaqjbBJKiWtg4yRcKBQ7g7wnGnLV3M8oLvVWwOk2PdYY6PEFegR1vezXR0tw6fZGF9csVakIRjrJiy2veSBFQ== + dependencies: + define-data-property "^1.1.1" + get-intrinsic "^1.2.1" + gopd "^1.0.1" + has-property-descriptors "^1.0.0" + +set-function-name@^2.0.0: + version "2.0.1" + resolved "https://registry.yarnpkg.com/set-function-name/-/set-function-name-2.0.1.tgz#12ce38b7954310b9f61faa12701620a0c882793a" + integrity sha512-tMNCiqYVkXIZgc2Hnoy2IvC/f8ezc5koaRFkCjrpWzGpCd3qbZXPzVy9MAZzK1ch/X0jvSkojys3oqJN0qCmdA== + dependencies: + define-data-property "^1.0.1" + functions-have-names "^1.2.3" + has-property-descriptors "^1.0.0" + set-value@^2.0.0, set-value@^2.0.1: version "2.0.1" resolved "https://registry.yarnpkg.com/set-value/-/set-value-2.0.1.tgz#a18d40530e6f07de4228c7defe4227af8cad005b" @@ -5131,32 +5239,32 @@ string.prototype.matchall@^4.0.8: regexp.prototype.flags "^1.4.3" side-channel "^1.0.4" -string.prototype.trim@^1.2.7: - version "1.2.7" - resolved "https://registry.yarnpkg.com/string.prototype.trim/-/string.prototype.trim-1.2.7.tgz#a68352740859f6893f14ce3ef1bb3037f7a90533" - integrity sha512-p6TmeT1T3411M8Cgg9wBTMRtY2q9+PNy9EV1i2lIXUN/btt763oIfxwN3RR8VU6wHX8j/1CFy0L+YuThm6bgOg== +string.prototype.trim@^1.2.8: + version "1.2.8" + resolved "https://registry.yarnpkg.com/string.prototype.trim/-/string.prototype.trim-1.2.8.tgz#f9ac6f8af4bd55ddfa8895e6aea92a96395393bd" + integrity sha512-lfjY4HcixfQXOfaqCvcBuOIapyaroTXhbkfJN3gcB1OtyupngWK4sEET9Knd0cXd28kTUqu/kHoV4HKSJdnjiQ== dependencies: call-bind "^1.0.2" - define-properties "^1.1.4" - es-abstract "^1.20.4" + define-properties "^1.2.0" + es-abstract "^1.22.1" -string.prototype.trimend@^1.0.6: - version "1.0.6" - resolved "https://registry.yarnpkg.com/string.prototype.trimend/-/string.prototype.trimend-1.0.6.tgz#c4a27fa026d979d79c04f17397f250a462944533" - integrity sha512-JySq+4mrPf9EsDBEDYMOb/lM7XQLulwg5R/m1r0PXEFqrV0qHvl58sdTilSXtKOflCsK2E8jxf+GKC0T07RWwQ== +string.prototype.trimend@^1.0.7: + version "1.0.7" + resolved "https://registry.yarnpkg.com/string.prototype.trimend/-/string.prototype.trimend-1.0.7.tgz#1bb3afc5008661d73e2dc015cd4853732d6c471e" + integrity sha512-Ni79DqeB72ZFq1uH/L6zJ+DKZTkOtPIHovb3YZHQViE+HDouuU4mBrLOLDn5Dde3RF8qw5qVETEjhu9locMLvA== dependencies: call-bind "^1.0.2" - define-properties "^1.1.4" - es-abstract "^1.20.4" + define-properties "^1.2.0" + es-abstract "^1.22.1" -string.prototype.trimstart@^1.0.6: - version "1.0.6" - resolved "https://registry.yarnpkg.com/string.prototype.trimstart/-/string.prototype.trimstart-1.0.6.tgz#e90ab66aa8e4007d92ef591bbf3cd422c56bdcf4" - integrity sha512-omqjMDaY92pbn5HOX7f9IccLA+U1tA9GvtU4JrodiXFfYB7jPzzHpRzpglLAjtUV6bB557zwClJezTqnAiYnQA== +string.prototype.trimstart@^1.0.7: + version "1.0.7" + resolved "https://registry.yarnpkg.com/string.prototype.trimstart/-/string.prototype.trimstart-1.0.7.tgz#d4cdb44b83a4737ffbac2d406e405d43d0184298" + integrity sha512-NGhtDFu3jCEm7B4Fy0DpLewdJQOZcQ0rGbwQ/+stjnrp2i+rlKeCvos9hOIeCmqwratM47OBxY7uFZzjxHXmrg== dependencies: call-bind "^1.0.2" - define-properties "^1.1.4" - es-abstract "^1.20.4" + define-properties "^1.2.0" + es-abstract "^1.22.1" strip-ansi@^6.0.0, strip-ansi@^6.0.1: version "6.0.1" @@ -5488,9 +5596,9 @@ v8-to-istanbul@^9.0.1: convert-source-map "^1.6.0" vite@^4: - version "4.4.11" - resolved "https://registry.yarnpkg.com/vite/-/vite-4.4.11.tgz#babdb055b08c69cfc4c468072a2e6c9ca62102b0" - integrity sha512-ksNZJlkcU9b0lBwAGZGGaZHCMqHsc8OpgtoYhsQ4/I2v5cnpmmmqe5pM4nv/4Hn6G/2GhTdj0DhZh2e+Er1q5A== + version "4.5.0" + resolved "https://registry.yarnpkg.com/vite/-/vite-4.5.0.tgz#ec406295b4167ac3bc23e26f9c8ff559287cff26" + integrity sha512-ulr8rNLA6rkyFAlVWw2q5YJ91v098AFQ2R0PRFwPzREXOUJQPtFUG0t+/ZikhaOCDqFoDhN6/v8Sq0o4araFAw== dependencies: esbuild "^0.18.10" postcss "^8.4.27" @@ -5544,7 +5652,18 @@ which-collection@^1.0.1: is-weakmap "^2.0.1" is-weakset "^2.0.1" -which-typed-array@^1.1.10, which-typed-array@^1.1.11, which-typed-array@^1.1.9: +which-typed-array@^1.1.11, which-typed-array@^1.1.13: + version "1.1.13" + resolved "https://registry.yarnpkg.com/which-typed-array/-/which-typed-array-1.1.13.tgz#870cd5be06ddb616f504e7b039c4c24898184d36" + integrity sha512-P5Nra0qjSncduVPEAr7xhoF5guty49ArDTwzJ/yNuPIbZppyRxFQsRCWrocxIY+CnMVG+qfbU2FmDKyvSGClow== + dependencies: + available-typed-arrays "^1.0.5" + call-bind "^1.0.4" + for-each "^0.3.3" + gopd "^1.0.1" + has-tostringtag "^1.0.0" + +which-typed-array@^1.1.9: version "1.1.11" resolved "https://registry.yarnpkg.com/which-typed-array/-/which-typed-array-1.1.11.tgz#99d691f23c72aab6768680805a271b69761ed61a" integrity sha512-qe9UWWpkeG5yzZ0tNYxDmd7vo58HDBc39mZ0xWWpolAGADdFOzkfamWLDxkOWcvHQKVmdTyQdLD4NOfjLWTKew== diff --git a/config-model-api/abi-spec.json b/config-model-api/abi-spec.json index b28401f1873..ce73f8300db 100644 --- a/config-model-api/abi-spec.json +++ b/config-model-api/abi-spec.json @@ -1420,6 +1420,31 @@ ], "fields" : [ ] }, + "com.yahoo.config.model.api.OnnxMemoryStats" : { + "superClass" : "java.lang.Record", + "interfaces" : [ ], + "attributes" : [ + "public", + "final", + "record" + ], + "methods" : [ + "public void <init>(long, long, long, long)", + "public static com.yahoo.config.model.api.OnnxMemoryStats fromJson(com.fasterxml.jackson.databind.JsonNode)", + "public static com.yahoo.config.model.api.OnnxMemoryStats fromJson(com.yahoo.config.application.api.ApplicationFile)", + "public static com.yahoo.path.Path memoryStatsFilePath(com.yahoo.path.Path)", + "public long peakMemoryUsage()", + "public com.fasterxml.jackson.databind.JsonNode toJson()", + "public final java.lang.String toString()", + "public final int hashCode()", + "public final boolean equals(java.lang.Object)", + "public long vmSize()", + "public long vmRss()", + "public long mallocPeak()", + "public long mallocCurrent()" + ], + "fields" : [ ] + }, "com.yahoo.config.model.api.OnnxModelCost$Calculator" : { "superClass" : "java.lang.Object", "interfaces" : [ ], @@ -1431,11 +1456,28 @@ "methods" : [ "public abstract long aggregatedModelCostInBytes()", "public abstract void registerModel(com.yahoo.config.application.api.ApplicationFile)", - "public abstract void registerModel(com.yahoo.config.ModelReference)", "public abstract void registerModel(java.net.URI)" ], "fields" : [ ] }, + "com.yahoo.config.model.api.OnnxModelCost$DisabledOnnxModelCost" : { + "superClass" : "java.lang.Object", + "interfaces" : [ + "com.yahoo.config.model.api.OnnxModelCost", + "com.yahoo.config.model.api.OnnxModelCost$Calculator" + ], + "attributes" : [ + "public" + ], + "methods" : [ + "public void <init>()", + "public com.yahoo.config.model.api.OnnxModelCost$Calculator newCalculator(com.yahoo.config.application.api.ApplicationPackage, com.yahoo.config.provision.ApplicationId)", + "public long aggregatedModelCostInBytes()", + "public void registerModel(com.yahoo.config.application.api.ApplicationFile)", + "public void registerModel(java.net.URI)" + ], + "fields" : [ ] + }, "com.yahoo.config.model.api.OnnxModelCost" : { "superClass" : "java.lang.Object", "interfaces" : [ ], @@ -1445,7 +1487,8 @@ "abstract" ], "methods" : [ - "public abstract com.yahoo.config.model.api.OnnxModelCost$Calculator newCalculator(com.yahoo.config.application.api.ApplicationPackage, com.yahoo.config.application.api.DeployLogger)", + "public com.yahoo.config.model.api.OnnxModelCost$Calculator newCalculator(com.yahoo.config.application.api.ApplicationPackage, com.yahoo.config.application.api.DeployLogger)", + "public abstract com.yahoo.config.model.api.OnnxModelCost$Calculator newCalculator(com.yahoo.config.application.api.ApplicationPackage, com.yahoo.config.provision.ApplicationId)", "public static com.yahoo.config.model.api.OnnxModelCost disabled()" ], "fields" : [ ] diff --git a/config-model-api/src/main/java/com/yahoo/config/application/api/ApplicationFile.java b/config-model-api/src/main/java/com/yahoo/config/application/api/ApplicationFile.java index 36d6efdf59b..d262c7bc862 100644 --- a/config-model-api/src/main/java/com/yahoo/config/application/api/ApplicationFile.java +++ b/config-model-api/src/main/java/com/yahoo/config/application/api/ApplicationFile.java @@ -27,21 +27,21 @@ public abstract class ApplicationFile implements Comparable<ApplicationFile> { } /** - * Check whether or not this file is a directory. + * Checks whether this file is a directory. * * @return true if it is, false if not. */ public abstract boolean isDirectory(); /** - * Test whether or not this file exists. + * Tests whether this file exists. * * @return true if it exists, false if not. */ public abstract boolean exists(); /** - * Create a {@link Reader} for the contents of this file. + * Creates a {@link Reader} for the contents of this file. * * @return A {@link Reader} that should be closed after use. * @throws FileNotFoundException if the file is not found. @@ -50,7 +50,7 @@ public abstract class ApplicationFile implements Comparable<ApplicationFile> { /** - * Create an {@link InputStream} for the contents of this file. + * Creates an {@link InputStream} for the contents of this file. * * @return An {@link InputStream} that should be closed after use. * @throws FileNotFoundException if the file is not found. @@ -58,7 +58,7 @@ public abstract class ApplicationFile implements Comparable<ApplicationFile> { public abstract InputStream createInputStream() throws FileNotFoundException; /** - * Create a directory at the path represented by this file. Parent directories will + * Creates a directory at the path represented by this file. Parent directories will * be automatically created. * * @return this @@ -67,7 +67,7 @@ public abstract class ApplicationFile implements Comparable<ApplicationFile> { public abstract ApplicationFile createDirectory(); /** - * Write the contents from this reader to this file. Any existing content will be overwritten! + * Writes the contents from supplied reader to this file. Any existing content will be overwritten! * * @param input A reader pointing to the content that should be written. * @return this @@ -82,7 +82,7 @@ public abstract class ApplicationFile implements Comparable<ApplicationFile> { public abstract ApplicationFile appendFile(String value); /** - * List the files under this directory. If this is file, an empty list is returned. + * Lists the files under this directory. If this is file, an empty list is returned. * Only immediate files/subdirectories are returned. * * @return a list of files in this directory. @@ -92,7 +92,7 @@ public abstract class ApplicationFile implements Comparable<ApplicationFile> { } /** - * List the files under this directory. If this is file, an empty list is returned. + * Lists the files under this directory. If this is a file, an empty list is returned. * Only immediate files/subdirectories are returned. * * @param filter A filter functor for filtering path names @@ -101,7 +101,7 @@ public abstract class ApplicationFile implements Comparable<ApplicationFile> { public abstract List<ApplicationFile> listFiles(PathFilter filter); /** - * List the files in this directory, optionally list files for subdirectories recursively as well. + * Lists the files in this directory, optionally lists files for subdirectories recursively as well. * * @param recurse Set to true if all files in the directory tree should be returned. * @return a list of files in this directory. @@ -121,7 +121,7 @@ public abstract class ApplicationFile implements Comparable<ApplicationFile> { } /** - * Delete the file pointed to by this. If it is a non-empty directory, the operation will throw. + * Deletes the file pointed to by this. If this is a non-empty directory, the operation will throw. * * @return this. * @throws RuntimeException if the file is a directory and not empty. @@ -129,7 +129,7 @@ public abstract class ApplicationFile implements Comparable<ApplicationFile> { public abstract ApplicationFile delete(); /** - * Get the path that this file represents. + * Gets the path that this file represents. * * @return a Path */ diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/OnnxMemoryStats.java b/config-model-api/src/main/java/com/yahoo/config/model/api/OnnxMemoryStats.java new file mode 100644 index 00000000000..4e660c6fe73 --- /dev/null +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/OnnxMemoryStats.java @@ -0,0 +1,49 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package com.yahoo.config.model.api; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.yahoo.config.application.api.ApplicationFile; +import com.yahoo.config.application.api.ApplicationPackage; +import com.yahoo.path.Path; + +import java.io.IOException; +import java.util.Optional; + +/** + * Memory statistics as reported by vespa-analyze-onnx-model. + * + * @author bjorncs + */ +public record OnnxMemoryStats(long vmSize, long vmRss, long mallocPeak, long mallocCurrent) { + private static final String VM_SIZE_FIELD = "vm_size", VM_RSS_FIELD = "vm_rss", + MALLOC_PEAK_FIELD = "malloc_peak", MALLOC_CURRENT_FIELD = "malloc_current"; + private static final ObjectMapper jsonParser = new ObjectMapper(); + + /** Parse output from `vespa-analyze-onnx-model --probe-types` */ + public static OnnxMemoryStats fromJson(JsonNode json) { + return new OnnxMemoryStats(json.get(VM_SIZE_FIELD).asLong(), json.get(VM_RSS_FIELD).asLong(), + // Temporarily allow missing fields until old config model versions are gone + Optional.ofNullable(json.get(MALLOC_PEAK_FIELD)).map(JsonNode::asLong).orElse(0L), + Optional.ofNullable(json.get(MALLOC_CURRENT_FIELD)).map(JsonNode::asLong).orElse(0L)); + } + + /** @see #fromJson(JsonNode) */ + public static OnnxMemoryStats fromJson(ApplicationFile file) throws IOException { + return fromJson(jsonParser.readTree(file.createReader())); + } + + public static Path memoryStatsFilePath(Path modelPath) { + var fileName = modelPath.getRelative().replaceAll("[^\\w\\d\\$@_]", "_") + ".memory_stats"; + return ApplicationPackage.MODELS_GENERATED_REPLICATED_DIR.append(fileName); + } + + public long peakMemoryUsage() { return Long.max(vmSize, Long.max(vmRss, Long.max(mallocPeak, mallocCurrent))); } + + public JsonNode toJson() { + return jsonParser.createObjectNode().put(VM_SIZE_FIELD, vmSize).put(VM_RSS_FIELD, vmRss) + .put(MALLOC_PEAK_FIELD, mallocPeak).put(MALLOC_CURRENT_FIELD, mallocCurrent); + } +} + diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/OnnxModelCost.java b/config-model-api/src/main/java/com/yahoo/config/model/api/OnnxModelCost.java index e6fe3ce18b5..abfddfe40be 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/OnnxModelCost.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/OnnxModelCost.java @@ -2,10 +2,10 @@ package com.yahoo.config.model.api; -import com.yahoo.config.ModelReference; import com.yahoo.config.application.api.ApplicationFile; import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.config.provision.ApplicationId; import java.net.URI; @@ -14,21 +14,26 @@ import java.net.URI; */ public interface OnnxModelCost { - Calculator newCalculator(ApplicationPackage appPkg, DeployLogger logger); + // TODO: Remove when 8.250 is oldest model in use + default Calculator newCalculator(ApplicationPackage appPkg, DeployLogger deployLogger) { + return newCalculator(appPkg, ApplicationId.defaultId()); + } + + Calculator newCalculator(ApplicationPackage appPkg, ApplicationId applicationId); interface Calculator { long aggregatedModelCostInBytes(); void registerModel(ApplicationFile path); - @Deprecated(forRemoval = true) void registerModel(ModelReference ref); // TODO(bjorncs): remove once no longer in use by old config models void registerModel(URI uri); } - static OnnxModelCost disabled() { - return (__, ___) -> new Calculator() { - @Override public long aggregatedModelCostInBytes() { return 0; } - @Override public void registerModel(ApplicationFile path) {} - @SuppressWarnings("removal") @Override public void registerModel(ModelReference ref) {} - @Override public void registerModel(URI uri) {} - }; + static OnnxModelCost disabled() { return new DisabledOnnxModelCost(); } + + class DisabledOnnxModelCost implements OnnxModelCost, Calculator { + @Override public Calculator newCalculator(ApplicationPackage appPkg, ApplicationId applicationId) { return this; } + @Override public long aggregatedModelCostInBytes() {return 0;} + @Override public void registerModel(ApplicationFile path) {} + @Override public void registerModel(URI uri) {} } + } diff --git a/config-model/src/main/java/com/yahoo/config/model/provision/InMemoryProvisioner.java b/config-model/src/main/java/com/yahoo/config/model/provision/InMemoryProvisioner.java index 16affbd7b0e..c8f088509c5 100644 --- a/config-model/src/main/java/com/yahoo/config/model/provision/InMemoryProvisioner.java +++ b/config-model/src/main/java/com/yahoo/config/model/provision/InMemoryProvisioner.java @@ -13,6 +13,7 @@ import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.HostSpec; import com.yahoo.config.provision.NodeResources; +import com.yahoo.config.provision.NodeResources.DiskSpeed; import com.yahoo.config.provision.ProvisionLogger; import java.util.ArrayList; @@ -234,7 +235,7 @@ public class InMemoryProvisioner implements HostProvisioner { // Minimal capacity policies private NodeResources decideResources(NodeResources resources) { if (defaultNodeResources.isUnspecified()) return resources; - return resources.withUnspecifiedNumbersFrom(defaultNodeResources); + return resources.withUnspecifiedFieldsFrom(defaultNodeResources); } private List<HostSpec> allocateHostGroup(ClusterSpec clusterGroup, NodeResources requestedResourcesOrUnspecified, diff --git a/config-model/src/main/java/com/yahoo/schema/RankProfile.java b/config-model/src/main/java/com/yahoo/schema/RankProfile.java index 6007a1cf4b1..1ff85c9c89f 100644 --- a/config-model/src/main/java/com/yahoo/schema/RankProfile.java +++ b/config-model/src/main/java/com/yahoo/schema/RankProfile.java @@ -22,6 +22,7 @@ import com.yahoo.searchlib.rankingexpression.FeatureList; import com.yahoo.searchlib.rankingexpression.RankingExpression; import com.yahoo.searchlib.rankingexpression.Reference; import com.yahoo.searchlib.rankingexpression.rule.Arguments; +import com.yahoo.searchlib.rankingexpression.rule.ExpressionNode; import com.yahoo.searchlib.rankingexpression.rule.ReferenceNode; import com.yahoo.tensor.Tensor; import com.yahoo.tensor.TensorType; @@ -30,6 +31,7 @@ import java.io.IOException; import java.io.Reader; import java.io.StringReader; import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; @@ -109,7 +111,7 @@ public class RankProfile implements Cloneable { private String inheritedSummaryFeaturesProfileName; private Set<ReferenceNode> matchFeatures; - private Set<String> hiddenMatchFeatures; + private Set<ReferenceNode> hiddenMatchFeatures; private String inheritedMatchFeaturesProfileName; private Set<ReferenceNode> rankFeatures; @@ -607,7 +609,7 @@ public class RankProfile implements Cloneable { .orElse(Set.of()); } - public Set<String> getHiddenMatchFeatures() { + public Set<ReferenceNode> getHiddenMatchFeatures() { if (hiddenMatchFeatures != null) return Collections.unmodifiableSet(hiddenMatchFeatures); return uniquelyInherited(p -> p.getHiddenMatchFeatures(), f -> ! f.isEmpty(), "hidden match features") .orElse(Set.of()); @@ -626,15 +628,13 @@ public class RankProfile implements Cloneable { } private void addImplicitMatchFeatures(List<FeatureList> list) { - if (matchFeatures == null) - matchFeatures = new LinkedHashSet<>(); if (hiddenMatchFeatures == null) hiddenMatchFeatures = new LinkedHashSet<>(); + var current = getMatchFeatures(); for (var features : list) { for (ReferenceNode feature : features) { - if (! matchFeatures.contains(feature)) { - matchFeatures.add(feature); - hiddenMatchFeatures.add(feature.toString()); + if (! current.contains(feature)) { + hiddenMatchFeatures.add(feature); } } } @@ -1058,21 +1058,45 @@ public class RankProfile implements Cloneable { functions = compileFunctions(this::getFunctions, queryProfiles, featureTypes, importedModels, inlineFunctions, expressionTransforms); allFunctionsCached = null; + var context = new RankProfileTransformContext(this, + queryProfiles, + featureTypes, + importedModels, + constants(), + inlineFunctions); + var allNormalizers = getFeatureNormalizers(); + verifyNoNormalizers("first-phase expression", firstPhaseRanking, allNormalizers, context); + verifyNoNormalizers("second-phase expression", secondPhaseRanking, allNormalizers, context); + for (ReferenceNode mf : getMatchFeatures()) { + verifyNoNormalizers("match-feature " + mf, mf, allNormalizers, context); + } + for (ReferenceNode sf : getSummaryFeatures()) { + verifyNoNormalizers("summary-feature " + sf, sf, allNormalizers, context); + } if (globalPhaseRanking != null) { - var context = new RankProfileTransformContext(this, - queryProfiles, - featureTypes, - importedModels, - constants(), - inlineFunctions); var needInputs = new HashSet<String>(); + Set<String> userDeclaredMatchFeatures = new HashSet<>(); + for (ReferenceNode mf : getMatchFeatures()) { + userDeclaredMatchFeatures.add(mf.toString()); + } var recorder = new InputRecorder(needInputs); - if (matchFeatures != null) { - for (ReferenceNode mf : matchFeatures) { - recorder.alreadyHandled(mf.toString()); + recorder.alreadyMatchFeatures(userDeclaredMatchFeatures); + recorder.addKnownNormalizers(allNormalizers.keySet()); + recorder.process(globalPhaseRanking.function().getBody(), context); + for (var normalizerName : recorder.normalizersUsed()) { + var normalizer = allNormalizers.get(normalizerName); + var func = functions.get(normalizer.input()); + if (func != null) { + verifyNoNormalizers("normalizer input " + normalizer.input(), func, allNormalizers, context); + if (! userDeclaredMatchFeatures.contains(normalizer.input())) { + var subRecorder = new InputRecorder(needInputs); + subRecorder.alreadyMatchFeatures(userDeclaredMatchFeatures); + subRecorder.process(func.function().getBody(), context); + } + } else { + needInputs.add(normalizer.input()); } } - recorder.process(globalPhaseRanking.function().getBody(), context); List<FeatureList> addIfMissing = new ArrayList<>(); for (String input : needInputs) { if (input.startsWith("constant(") || input.startsWith("query(")) { @@ -1630,4 +1654,70 @@ public class RankProfile implements Cloneable { } + public static record RankFeatureNormalizer(Reference original, String name, String input, String algo, double kparam) { + @Override + public String toString() { + return "normalizer{name=" + name + ",input=" + input + ",algo=" + algo + ",k=" + kparam + "}"; + } + private static long hash(String s) { + int bob = com.yahoo.collections.BobHash.hash(s); + return bob + 0x100000000L; + } + public static RankFeatureNormalizer linear(Reference original, Reference inputRef) { + long h = hash(original.toString()); + String name = "normalize@" + h + "@linear"; + return new RankFeatureNormalizer(original, name, inputRef.toString(), "LINEAR", 0.0); + } + public static RankFeatureNormalizer rrank(Reference original, Reference inputRef, double k) { + long h = hash(original.toString()); + String name = "normalize@" + h + "@rrank"; + return new RankFeatureNormalizer(original, name, inputRef.toString(), "RRANK", k); + } + } + + private List<RankFeatureNormalizer> featureNormalizers = new ArrayList<>(); + + public Map<String, RankFeatureNormalizer> getFeatureNormalizers() { + Map<String, RankFeatureNormalizer> all = new LinkedHashMap<>(); + for (var inheritedProfile : inherited()) { + all.putAll(inheritedProfile.getFeatureNormalizers()); + } + for (var n : featureNormalizers) { + all.put(n.name(), n); + } + return all; + } + + public void addFeatureNormalizer(RankFeatureNormalizer n) { + if (functions.get(n.name()) != null) { + throw new IllegalArgumentException("cannot use name '" + name + "' for both function and normalizer"); + } + featureNormalizers.add(n); + } + + private void verifyNoNormalizers(String where, RankingExpressionFunction f, Map<String, RankFeatureNormalizer> allNormalizers, RankProfileTransformContext context) { + if (f == null) return; + verifyNoNormalizers(where, f.function(), allNormalizers, context); + } + + private void verifyNoNormalizers(String where, ExpressionFunction func, Map<String, RankFeatureNormalizer> allNormalizers, RankProfileTransformContext context) { + if (func == null) return; + var body = func.getBody(); + if (body == null) return; + verifyNoNormalizers(where, body.getRoot(), allNormalizers, context); + } + + private void verifyNoNormalizers(String where, ExpressionNode node, Map<String, RankFeatureNormalizer> allNormalizers, RankProfileTransformContext context) { + var needInputs = new HashSet<String>(); + var recorder = new InputRecorder(needInputs); + recorder.process(node, context); + for (var input : needInputs) { + var normalizer = allNormalizers.get(input); + if (normalizer != null) { + throw new IllegalArgumentException("Cannot use " + normalizer.original() + " from " + where + ", only valid in global-phase expression"); + } + } + } + + } diff --git a/config-model/src/main/java/com/yahoo/schema/derived/RawRankProfile.java b/config-model/src/main/java/com/yahoo/schema/derived/RawRankProfile.java index 05e5f17ea3d..68fa0fe6de9 100644 --- a/config-model/src/main/java/com/yahoo/schema/derived/RawRankProfile.java +++ b/config-model/src/main/java/com/yahoo/schema/derived/RawRankProfile.java @@ -54,6 +54,7 @@ public class RawRankProfile implements RankProfilesConfig.Producer { private final String name; private final Compressor.Compression compressedProperties; + private final Map<String, RankProfile.RankFeatureNormalizer> featureNormalizers; /** The compiled profile this is created from. */ private final Collection<RankProfile.Constant> constants; @@ -66,13 +67,14 @@ public class RawRankProfile implements RankProfilesConfig.Producer { this.name = rankProfile.name(); /* * Forget the RankProfiles as soon as possible. They can become very large and memory hungry - * Especially do not refer then through any member variables due to the RawRankProfile living forever. + * Especially do not refer them through any member variables due to the RawRankProfile living forever. */ RankProfile compiled = rankProfile.compile(queryProfiles, importedModels); constants = compiled.constants().values(); onnxModels = compiled.onnxModels().values(); - compressedProperties = compress(new Deriver(compiled, attributeFields, deployProperties, queryProfiles) - .derive(largeExpressions)); + var deriver = new Deriver(compiled, attributeFields, deployProperties, queryProfiles); + compressedProperties = compress(deriver.derive(largeExpressions)); + this.featureNormalizers = compiled.getFeatureNormalizers(); } public Collection<RankProfile.Constant> constants() { return constants; } @@ -111,6 +113,18 @@ public class RawRankProfile implements RankProfilesConfig.Producer { b.fef(fefB); } + private void buildNormalizers(RankProfilesConfig.Rankprofile.Builder b) { + for (var normalizer : featureNormalizers.values()) { + var nBuilder = new RankProfilesConfig.Rankprofile.Normalizer.Builder(); + nBuilder.name(normalizer.name()); + nBuilder.input(normalizer.input()); + var algo = RankProfilesConfig.Rankprofile.Normalizer.Algo.Enum.valueOf(normalizer.algo()); + nBuilder.algo(algo); + nBuilder.kparam(normalizer.kparam()); + b.normalizer(nBuilder); + } + } + /** * Returns the properties of this as an unmodifiable list. * Note: This method is expensive. @@ -121,6 +135,7 @@ public class RawRankProfile implements RankProfilesConfig.Producer { public void getConfig(RankProfilesConfig.Builder builder) { RankProfilesConfig.Rankprofile.Builder b = new RankProfilesConfig.Rankprofile.Builder().name(getName()); getRankProperties(b); + buildNormalizers(b); builder.rankprofile(b); } @@ -134,7 +149,7 @@ public class RawRankProfile implements RankProfilesConfig.Producer { private final Map<String, FieldRankSettings> fieldRankSettings = new java.util.LinkedHashMap<>(); private final Set<ReferenceNode> summaryFeatures; private final Set<ReferenceNode> matchFeatures; - private final Collection<String> hiddenMatchFeatures; + private final Set<ReferenceNode> hiddenMatchFeatures; private final Set<ReferenceNode> rankFeatures; private final Map<String, String> featureRenames = new java.util.LinkedHashMap<>(); private final List<RankProfile.RankProperty> rankProperties; @@ -188,6 +203,7 @@ public class RawRankProfile implements RankProfilesConfig.Producer { summaryFeatures = new LinkedHashSet<>(compiled.getSummaryFeatures()); matchFeatures = new LinkedHashSet<>(compiled.getMatchFeatures()); hiddenMatchFeatures = compiled.getHiddenMatchFeatures(); + matchFeatures.addAll(hiddenMatchFeatures); rankFeatures = compiled.getRankFeatures(); rerankCount = compiled.getRerankCount(); globalPhaseRerankCount = compiled.getGlobalPhaseRerankCount(); @@ -414,8 +430,8 @@ public class RawRankProfile implements RankProfilesConfig.Producer { for (ReferenceNode feature : matchFeatures) { properties.add(new Pair<>("vespa.match.feature", feature.toString())); } - for (String feature : hiddenMatchFeatures) { - properties.add(new Pair<>("vespa.hidden.matchfeature", feature)); + for (ReferenceNode feature : hiddenMatchFeatures) { + properties.add(new Pair<>("vespa.hidden.matchfeature", feature.toString())); } for (ReferenceNode feature : rankFeatures) { properties.add(new Pair<>("vespa.dump.feature", feature.toString())); diff --git a/config-model/src/main/java/com/yahoo/schema/derived/SummaryClass.java b/config-model/src/main/java/com/yahoo/schema/derived/SummaryClass.java index ddb6b004070..300a55e521a 100644 --- a/config-model/src/main/java/com/yahoo/schema/derived/SummaryClass.java +++ b/config-model/src/main/java/com/yahoo/schema/derived/SummaryClass.java @@ -155,7 +155,8 @@ public class SummaryClass extends Derived { summaryField.getTransform() == SummaryTransform.GEOPOS || summaryField.getTransform() == SummaryTransform.POSITIONS || summaryField.getTransform() == SummaryTransform.MATCHED_ELEMENTS_FILTER || - summaryField.getTransform() == SummaryTransform.MATCHED_ATTRIBUTE_ELEMENTS_FILTER) + summaryField.getTransform() == SummaryTransform.MATCHED_ATTRIBUTE_ELEMENTS_FILTER || + summaryField.getTransform() == SummaryTransform.TOKENS) { return summaryField.getSingleSource(); } else if (summaryField.getTransform().isDynamic()) { diff --git a/config-model/src/main/java/com/yahoo/schema/derived/SummaryClassField.java b/config-model/src/main/java/com/yahoo/schema/derived/SummaryClassField.java index c1e6dd2aea3..2f60cd8eb06 100644 --- a/config-model/src/main/java/com/yahoo/schema/derived/SummaryClassField.java +++ b/config-model/src/main/java/com/yahoo/schema/derived/SummaryClassField.java @@ -92,6 +92,8 @@ public class SummaryClassField { return Type.FEATUREDATA; } else if (transform != null && transform.equals(SummaryTransform.SUMMARYFEATURES)) { return Type.FEATUREDATA; + } else if (transform != null && transform.equals(SummaryTransform.TOKENS)) { + return Type.JSONSTRING; } else { return Type.LONGSTRING; } diff --git a/config-model/src/main/java/com/yahoo/schema/document/Ranking.java b/config-model/src/main/java/com/yahoo/schema/document/Ranking.java index d00abfcb9aa..2a2b1431057 100644 --- a/config-model/src/main/java/com/yahoo/schema/document/Ranking.java +++ b/config-model/src/main/java/com/yahoo/schema/document/Ranking.java @@ -44,9 +44,8 @@ public class Ranking implements Cloneable, Serializable { /** Returns true if the given rank settings are the same */ @Override public boolean equals(Object o) { - if ( ! (o instanceof Ranking)) return false; + if ( ! (o instanceof Ranking other)) return false; - Ranking other=(Ranking)o; if (this.filter != other.filter) return false; if (this.literal != other.literal) return false; if (this.normal != other.normal) return false; diff --git a/config-model/src/main/java/com/yahoo/schema/expressiontransforms/ExpressionTransforms.java b/config-model/src/main/java/com/yahoo/schema/expressiontransforms/ExpressionTransforms.java index cf46bedf223..42c8147b3dc 100644 --- a/config-model/src/main/java/com/yahoo/schema/expressiontransforms/ExpressionTransforms.java +++ b/config-model/src/main/java/com/yahoo/schema/expressiontransforms/ExpressionTransforms.java @@ -35,7 +35,8 @@ public class ExpressionTransforms { new FunctionShadower(), new TensorMaxMinTransformer(), new Simplifier(), - new BooleanExpressionTransformer()); + new BooleanExpressionTransformer(), + new NormalizerFunctionExpander()); } public RankingExpression transform(RankingExpression expression, RankProfileTransformContext context) { diff --git a/config-model/src/main/java/com/yahoo/schema/expressiontransforms/InputRecorder.java b/config-model/src/main/java/com/yahoo/schema/expressiontransforms/InputRecorder.java index 1128aaf3681..ab18f9c83db 100644 --- a/config-model/src/main/java/com/yahoo/schema/expressiontransforms/InputRecorder.java +++ b/config-model/src/main/java/com/yahoo/schema/expressiontransforms/InputRecorder.java @@ -14,6 +14,7 @@ import com.yahoo.searchlib.rankingexpression.transform.ExpressionTransformer; import com.yahoo.tensor.functions.Generate; import java.io.StringReader; +import java.util.Collection; import java.util.HashSet; import java.util.Set; import java.util.logging.Logger; @@ -29,19 +30,35 @@ public class InputRecorder extends ExpressionTransformer<InputRecorderContext> { private final Set<String> neededInputs; private final Set<String> handled = new HashSet<>(); + private final Set<String> availableNormalizers = new HashSet<>(); + private final Set<String> usedNormalizers = new HashSet<>(); public InputRecorder(Set<String> target) { this.neededInputs = target; } public void process(RankingExpression expression, RankProfileTransformContext context) { - transform(expression.getRoot(), new InputRecorderContext(context)); + process(expression.getRoot(), context); } - public void alreadyHandled(String name) { - handled.add(name); + public void process(ExpressionNode node, RankProfileTransformContext context) { + transform(node, new InputRecorderContext(context)); } + public void alreadyMatchFeatures(Collection<String> matchFeatures) { + for (String mf : matchFeatures) { + handled.add(mf); + } + } + + public void addKnownNormalizers(Collection<String> names) { + for (String name : names) { + availableNormalizers.add(name); + } + } + + public Set<String> normalizersUsed() { return this.usedNormalizers; } + @Override public ExpressionNode transform(ExpressionNode node, InputRecorderContext context) { if (node instanceof ReferenceNode r) { @@ -77,6 +94,10 @@ public class InputRecorder extends ExpressionTransformer<InputRecorderContext> { if (simpleFunctionOrIdentifier && context.localVariables().contains(name)) { return; } + if (simpleFunctionOrIdentifier && availableNormalizers.contains(name)) { + usedNormalizers.add(name); + return; + } if (ref.isSimpleRankingExpressionWrapper()) { name = ref.simpleArgument().get(); simpleFunctionOrIdentifier = true; @@ -113,13 +134,21 @@ public class InputRecorder extends ExpressionTransformer<InputRecorderContext> { } } if ("onnx".equals(name)) { - if (args.size() != 1) { + if (args.size() < 1) { throw new IllegalArgumentException("expected name of ONNX model as argument: " + feature); } var arg = args.expressions().get(0); var models = context.rankProfile().onnxModels(); var model = models.get(arg.toString()); if (model == null) { + var tmp = OnnxModelTransformer.transformFeature(feature, context.rankProfile()); + if (tmp instanceof ReferenceNode newRefNode) { + args = newRefNode.getArguments(); + arg = args.expressions().get(0); + model = models.get(arg.toString()); + } + } + if (model == null) { throw new IllegalArgumentException("missing onnx model: " + arg); } model.getInputMap().forEach((__, onnxInput) -> { diff --git a/config-model/src/main/java/com/yahoo/schema/expressiontransforms/NormalizerFunctionExpander.java b/config-model/src/main/java/com/yahoo/schema/expressiontransforms/NormalizerFunctionExpander.java new file mode 100644 index 00000000000..a8fee966656 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/expressiontransforms/NormalizerFunctionExpander.java @@ -0,0 +1,134 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.expressiontransforms; + +import com.yahoo.schema.FeatureNames; +import com.yahoo.schema.RankProfile.RankFeatureNormalizer; +import com.yahoo.searchlib.rankingexpression.evaluation.BooleanValue; +import com.yahoo.searchlib.rankingexpression.rule.OperationNode; +import com.yahoo.searchlib.rankingexpression.rule.Operator; +import com.yahoo.searchlib.rankingexpression.rule.CompositeNode; +import com.yahoo.searchlib.rankingexpression.rule.ConstantNode; +import com.yahoo.searchlib.rankingexpression.rule.ExpressionNode; +import com.yahoo.searchlib.rankingexpression.rule.IfNode; +import com.yahoo.searchlib.rankingexpression.transform.ExpressionTransformer; +import com.yahoo.searchlib.rankingexpression.transform.TransformContext; +import com.yahoo.searchlib.rankingexpression.RankingExpression; +import com.yahoo.searchlib.rankingexpression.Reference; +import com.yahoo.searchlib.rankingexpression.parser.ParseException; +import com.yahoo.searchlib.rankingexpression.rule.CompositeNode; +import com.yahoo.searchlib.rankingexpression.rule.ConstantNode; +import com.yahoo.searchlib.rankingexpression.rule.ExpressionNode; +import com.yahoo.searchlib.rankingexpression.rule.ReferenceNode; +import com.yahoo.searchlib.rankingexpression.rule.TensorFunctionNode; +import com.yahoo.searchlib.rankingexpression.transform.ExpressionTransformer; +import com.yahoo.tensor.functions.Generate; + +import java.io.StringReader; +import java.util.HashSet; +import java.util.Set; +import java.util.logging.Logger; + +import java.util.ArrayDeque; +import java.util.Deque; +import java.util.Iterator; +import java.util.List; +import java.util.ArrayList; + +/** + * Recognizes pseudo-functions and creates global-phase normalizers + * @author arnej + */ +public class NormalizerFunctionExpander extends ExpressionTransformer<RankProfileTransformContext> { + + public final static String NORMALIZE_LINEAR = "normalize_linear"; + public final static String RECIPROCAL_RANK = "reciprocal_rank"; + public final static String RECIPROCAL_RANK_FUSION = "reciprocal_rank_fusion"; + + @Override + public ExpressionNode transform(ExpressionNode node, RankProfileTransformContext context) { + if (node instanceof ReferenceNode r) { + node = transformReference(r, context); + } + if (node instanceof CompositeNode composite) { + node = transformChildren(composite, context); + } + return node; + } + + private ExpressionNode transformReference(ReferenceNode node, RankProfileTransformContext context) { + Reference ref = node.reference(); + String name = ref.name(); + if (ref.output() != null) { + return node; + } + var f = context.rankProfile().getFunctions().get(name); + if (f != null) { + // never transform declared functions + return node; + } + return switch(name) { + case RECIPROCAL_RANK_FUSION -> transform(expandRRF(ref), context); + case NORMALIZE_LINEAR -> transformNormLin(ref, context); + case RECIPROCAL_RANK -> transformRRank(ref, context); + default -> node; + }; + } + + private ExpressionNode expandRRF(Reference ref) { + var args = ref.arguments(); + if (args.size() < 2) { + throw new IllegalArgumentException("must have at least 2 arguments: " + ref); + } + List<ExpressionNode> children = new ArrayList<>(); + List<Operator> operators = new ArrayList<>(); + for (var arg : args.expressions()) { + if (! children.isEmpty()) operators.add(Operator.plus); + children.add(new ReferenceNode(RECIPROCAL_RANK, List.of(arg), null)); + } + // must be further transformed (see above) + return new OperationNode(children, operators); + } + + private ExpressionNode transformNormLin(Reference ref, RankProfileTransformContext context) { + var args = ref.arguments(); + if (args.size() != 1) { + throw new IllegalArgumentException("must have exactly 1 argument: " + ref); + } + var input = args.expressions().get(0); + if (input instanceof ReferenceNode inputRefNode) { + var inputRef = inputRefNode.reference(); + RankFeatureNormalizer normalizer = RankFeatureNormalizer.linear(ref, inputRef); + context.rankProfile().addFeatureNormalizer(normalizer); + var newRef = Reference.fromIdentifier(normalizer.name()); + return new ReferenceNode(newRef); + } else { + throw new IllegalArgumentException("the first argument must be a simple feature: " + ref + " => " + input.getClass()); + } + } + + private ExpressionNode transformRRank(Reference ref, RankProfileTransformContext context) { + var args = ref.arguments(); + if (args.size() < 1 || args.size() > 2) { + throw new IllegalArgumentException("must have 1 or 2 arguments: " + ref); + } + double k = 60.0; + if (args.size() == 2) { + var kArg = args.expressions().get(1); + if (kArg instanceof ConstantNode kNode) { + k = kNode.getValue().asDouble(); + } else { + throw new IllegalArgumentException("the second argument (k) must be a constant in: " + ref); + } + } + var input = args.expressions().get(0); + if (input instanceof ReferenceNode inputRefNode) { + var inputRef = inputRefNode.reference(); + RankFeatureNormalizer normalizer = RankFeatureNormalizer.rrank(ref, inputRef, k); + context.rankProfile().addFeatureNormalizer(normalizer); + var newRef = Reference.fromIdentifier(normalizer.name()); + return new ReferenceNode(newRef); + } else { + throw new IllegalArgumentException("the first argument must be a simple feature: " + ref); + } + } +} diff --git a/config-model/src/main/java/com/yahoo/schema/parser/ConvertParsedFields.java b/config-model/src/main/java/com/yahoo/schema/parser/ConvertParsedFields.java index 7c6d62580cb..bc0e16abbe3 100644 --- a/config-model/src/main/java/com/yahoo/schema/parser/ConvertParsedFields.java +++ b/config-model/src/main/java/com/yahoo/schema/parser/ConvertParsedFields.java @@ -20,6 +20,7 @@ import com.yahoo.vespa.documentmodel.SummaryTransform; import java.util.Locale; import java.util.Map; +import java.util.logging.Level; /** * Helper for converting ParsedField etc to SDField with settings @@ -137,7 +138,7 @@ public class ConvertParsedFields { } // from grammar, things that can be inside struct-field block - private void convertCommonFieldSettings(SDField field, ParsedField parsed) { + private void convertCommonFieldSettings(Schema schema, SDField field, ParsedField parsed) { convertMatchSettings(field, parsed.matchSettings()); var indexing = parsed.getIndexing(); if (indexing.isPresent()) { @@ -152,7 +153,12 @@ public class ConvertParsedFields { for (var summaryField : parsed.getSummaryFields()) { var dataType = field.getDataType(); var otherType = summaryField.getType(); - if (otherType != null) { + if (otherType != null && summaryField.getHasExplicitType()) { + schema.getDeployLogger().log(Level.FINE, () -> "For " + schema.getName() + + ", field '" + field.getName() + + "', summary '" + summaryField.name() + + "': Specifying the type is deprecated, ignored and will be an error in Vespa 9." + + " Remove the type specification to silence this warning."); dataType = context.resolveType(otherType); } convertSummaryField(field, summaryField, dataType); @@ -161,7 +167,7 @@ public class ConvertParsedFields { field.addQueryCommand(command); } for (var structField : parsed.getStructFields()) { - convertStructField(field, structField); + convertStructField(schema, field, structField); } if (parsed.hasLiteral()) { field.getRanking().setLiteral(true); @@ -174,13 +180,13 @@ public class ConvertParsedFields { } } - private void convertStructField(SDField field, ParsedField parsed) { + private void convertStructField(Schema schema, SDField field, ParsedField parsed) { SDField structField = field.getStructField(parsed.name()); if (structField == null ) { throw new IllegalArgumentException("Struct field '" + parsed.name() + "' has not been defined in struct " + "for field '" + field.getName() + "'."); } - convertCommonFieldSettings(structField, parsed); + convertCommonFieldSettings(schema, structField, parsed); } private void convertExtraFieldSettings(SDField field, ParsedField parsed) { @@ -217,6 +223,8 @@ public class ConvertParsedFields { transform = SummaryTransform.MATCHED_ELEMENTS_FILTER; } else if (parsed.getDynamic()) { transform = SummaryTransform.DYNAMICTEASER; + } else if (parsed.getTokens()) { + transform = SummaryTransform.TOKENS; } if (parsed.getBolded()) { transform = transform.bold(); @@ -278,7 +286,7 @@ public class ConvertParsedFields { String name = parsed.name(); DataType dataType = context.resolveType(parsed.getType()); var field = new SDField(document, name, dataType); - convertCommonFieldSettings(field, parsed); + convertCommonFieldSettings(schema, field, parsed); convertExtraFieldSettings(field, parsed); document.addField(field); return field; @@ -288,7 +296,7 @@ public class ConvertParsedFields { String name = parsed.name(); DataType dataType = context.resolveType(parsed.getType()); var field = new SDField(schema.getDocument(), name, dataType); - convertCommonFieldSettings(field, parsed); + convertCommonFieldSettings(schema, field, parsed); convertExtraFieldSettings(field, parsed); schema.addExtraField(field); } @@ -305,7 +313,7 @@ public class ConvertParsedFields { for (var parsedField : parsed.getFields()) { var fieldType = context.resolveType(parsedField.getType()); var field = new SDField(document, parsedField.name(), fieldType); - convertCommonFieldSettings(field, parsedField); + convertCommonFieldSettings(schema, field, parsedField); structProxy.addField(field); if (parsedField.hasIdOverride()) { structProxy.setFieldId(field, parsedField.idOverride()); diff --git a/config-model/src/main/java/com/yahoo/schema/parser/ConvertParsedSchemas.java b/config-model/src/main/java/com/yahoo/schema/parser/ConvertParsedSchemas.java index 9145934501c..7e19cb4a0ae 100644 --- a/config-model/src/main/java/com/yahoo/schema/parser/ConvertParsedSchemas.java +++ b/config-model/src/main/java/com/yahoo/schema/parser/ConvertParsedSchemas.java @@ -11,11 +11,13 @@ import com.yahoo.config.model.deploy.TestProperties; import com.yahoo.config.model.test.MockApplicationPackage; import com.yahoo.document.DataType; import com.yahoo.document.DocumentTypeManager; +import com.yahoo.document.PositionDataType; import com.yahoo.schema.DefaultRankProfile; import com.yahoo.schema.DocumentOnlySchema; import com.yahoo.schema.RankProfileRegistry; import com.yahoo.schema.Schema; import com.yahoo.schema.UnrankedRankProfile; +import com.yahoo.schema.derived.SummaryClass; import com.yahoo.schema.document.SDDocumentType; import com.yahoo.schema.document.SDField; import com.yahoo.schema.document.TemporaryImportedField; @@ -25,7 +27,9 @@ import com.yahoo.vespa.documentmodel.SummaryField; import java.util.ArrayList; import java.util.LinkedHashMap; +import java.util.LinkedHashSet; import java.util.List; +import java.util.logging.Level; import java.util.Map; import java.util.Optional; @@ -137,7 +141,86 @@ public class ConvertParsedSchemas { schema.addDocument(document); } - private void convertDocumentSummary(Schema schema, ParsedDocumentSummary parsed, TypeResolver typeContext) { + /* + * Helper class for resolving data type for a document summary. Summary type is still + * used internally in config model when generating and processing indexing scripts. + * See DynamicSummaryTransformUtils class comment for more details. + * + * This kind of resolving is a temporary measure until the use of summary fields have + * been eliminated from indexing scripts and are no longer used to extend the document + * type. At that time, the data type of a summary field is no longer relevant. + */ + private class SummaryFieldTypeResolver { + + private final Schema schema; + private final Map<String, ParsedSummaryField> summaryFields = new LinkedHashMap<String, ParsedSummaryField>(); + private static final String zCurveSuffix = new String("_zcurve"); + + public SummaryFieldTypeResolver(Schema schema, List<ParsedDocumentSummary> parsed) { + this.schema = schema; + for (var docsum : parsed) { + for (var field : docsum.getSummaryFields()) { + summaryFields.put(field.name(), field); + } + } + } + + private boolean isPositionAttribute(Schema schema, String sourceFieldName) { + if (!sourceFieldName.endsWith(zCurveSuffix)) { + return false; + } + var name = sourceFieldName.substring(0, sourceFieldName.length() - zCurveSuffix.length()); + var field = schema.getField(name); + return (field.getDataType().equals(PositionDataType.INSTANCE)); + } + + + private String getSingleSource(ParsedSummaryField parsedField) { + if (parsedField.getSources().size() == 1) { + return parsedField.getSources().get(0); + } + return parsedField.name(); + } + + public DataType resolve(ParsedDocumentSummary docsum, ParsedSummaryField parsedField) { + var seen = new LinkedHashSet<String>(); + var origName = parsedField.name(); + while (true) { + if (seen.contains(parsedField.name())) { + throw new IllegalArgumentException("For schema '" + schema.getName() + + "' summary class '" + docsum.name() + + "' summary field '" + origName + + "': Source loop detected for summary field '" + parsedField.name() + "'"); + } + seen.add(parsedField.name()); + if (parsedField.getSources().size() >= 2) { + return DataType.STRING; // Flattening, streaming search + } + var source = getSingleSource(parsedField); + if (source.equals(SummaryClass.DOCUMENT_ID_FIELD)) { + return DataType.STRING; // Reserved source field name + } else if (isPositionAttribute(schema, source)) { + return DataType.LONG; // Extra field with suffix is added later for positions + } + var field = schema.getField(source); + if (field != null) { + return field.getDataType(); + } else if (schema.temporaryImportedFields().isPresent() && + schema.temporaryImportedFields().get().hasField(source)) { + return null; // Imported field, cannot resolve now + } else if (source.equals(parsedField.name()) || !summaryFields.containsKey(source)) { + throw new IllegalArgumentException("For schema '" + schema.getName() + + "', summary class '" + docsum.name() + + "', summary field '" + parsedField.name() + + "': there is no valid source '" + source + "'."); + } + parsedField = summaryFields.get(source); + } + } + } + + private void convertDocumentSummary(Schema schema, ParsedDocumentSummary parsed, TypeResolver typeContext, + SummaryFieldTypeResolver sfResolver) { var docsum = new DocumentSummary(parsed.name(), schema); parsed.getInherited().forEach(inherited -> docsum.addInherited(inherited)); if (parsed.getFromDisk()) { @@ -148,10 +231,17 @@ public class ConvertParsedSchemas { } for (var parsedField : parsed.getSummaryFields()) { var parsedType = parsedField.getType(); + if (parsedType != null) { + var log = schema.getDeployLogger(); + log.log(Level.FINE, () -> "For " + schema.getName() + + ", document-summary '" + parsed.name() + + "', summary field '" + parsedField.name() + + "': Specifying the type is deprecated, ignored and will be an error in Vespa 9." + + " Remove the type specification to silence this warning."); + } DataType dataType = (parsedType != null) ? typeContext.resolveType(parsedType) : null; - var existingField = schema.getField(parsedField.name()); - if (existingField != null) { - var existingType = existingField.getDataType(); + DataType existingType = sfResolver.resolve(parsed, parsedField); + if (existingType != null) { if (dataType == null) { dataType = existingType; } else if (!dataType.equals(existingType)) { @@ -161,10 +251,9 @@ public class ConvertParsedSchemas { } } } - if (dataType == null) { - throw new IllegalArgumentException("Missing data-type for summary field " + parsedField.name() + " in document-summary " + parsed.name()); - } - var summaryField = new SummaryField(parsedField.name(), dataType); + var summaryField = (dataType == null) ? + SummaryField.createWithUnresolvedType(parsedField.name()) : + new SummaryField(parsedField.name(), dataType); // XXX does not belong here: summaryField.setVsmCommand(SummaryField.VsmCommand.FLATTENSPACE); ConvertParsedFields.convertSummaryFieldSettings(summaryField, parsedField); @@ -206,6 +295,7 @@ public class ConvertParsedSchemas { } parsed.getRawAsBase64().ifPresent(value -> schema.enableRawAsBase64(value)); var typeContext = typeConverter.makeContext(parsed.getDocument()); + var sfResolver = new SummaryFieldTypeResolver(schema, parsed.getDocumentSummaries()); var fieldConverter = new ConvertParsedFields(typeContext, convertedStructs); convertDocument(schema, parsed.getDocument(), fieldConverter); for (var field : parsed.getFields()) { @@ -214,12 +304,12 @@ public class ConvertParsedSchemas { for (var index : parsed.getIndexes()) { fieldConverter.convertExtraIndex(schema, index); } - for (var docsum : parsed.getDocumentSummaries()) { - convertDocumentSummary(schema, docsum, typeContext); - } for (var importedField : parsed.getImportedFields()) { convertImportField(schema, importedField); } + for (var docsum : parsed.getDocumentSummaries()) { + convertDocumentSummary(schema, docsum, typeContext, sfResolver); + } for (var fieldSet : parsed.getFieldSets()) { convertFieldSet(schema, fieldSet); } diff --git a/config-model/src/main/java/com/yahoo/schema/parser/ParsedSummaryField.java b/config-model/src/main/java/com/yahoo/schema/parser/ParsedSummaryField.java index 1d5d73635e7..8f9733d2595 100644 --- a/config-model/src/main/java/com/yahoo/schema/parser/ParsedSummaryField.java +++ b/config-model/src/main/java/com/yahoo/schema/parser/ParsedSummaryField.java @@ -18,6 +18,8 @@ class ParsedSummaryField extends ParsedBlock { private boolean isMEO = false; private boolean isFull = false; private boolean isBold = false; + private boolean isTokens = false; + private boolean hasExplicitType = false; private final List<String> sources = new ArrayList<>(); private final List<String> destinations = new ArrayList<>(); @@ -37,6 +39,8 @@ class ParsedSummaryField extends ParsedBlock { boolean getDynamic() { return isDyn; } boolean getFull() { return isFull; } boolean getMatchedElementsOnly() { return isMEO; } + boolean getTokens() { return isTokens; } + boolean getHasExplicitType() { return hasExplicitType; } void addDestination(String dst) { destinations.add(dst); } void addSource(String src) { sources.add(src); } @@ -44,6 +48,8 @@ class ParsedSummaryField extends ParsedBlock { void setDynamic() { this.isDyn = true; } void setFull() { this.isFull = true; } void setMatchedElementsOnly() { this.isMEO = true; } + void setTokens() { this.isTokens = true; } + void setHasExplicitType() { this.hasExplicitType = true; } void setType(ParsedType value) { verifyThat(type == null, "Cannot change type from ", type, "to", value); this.type = value; diff --git a/config-model/src/main/java/com/yahoo/schema/processing/AddAttributeTransformToSummaryOfImportedFields.java b/config-model/src/main/java/com/yahoo/schema/processing/AddDataTypeAndTransformToSummaryOfImportedFields.java index 5b72381bfb1..762279e3871 100644 --- a/config-model/src/main/java/com/yahoo/schema/processing/AddAttributeTransformToSummaryOfImportedFields.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/AddDataTypeAndTransformToSummaryOfImportedFields.java @@ -2,6 +2,8 @@ package com.yahoo.schema.processing; import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.document.DataType; +import com.yahoo.document.PositionDataType; import com.yahoo.schema.RankProfileRegistry; import com.yahoo.schema.Schema; import com.yahoo.schema.document.ImmutableImportedComplexSDField; @@ -13,17 +15,17 @@ import com.yahoo.vespa.model.container.search.QueryProfiles; import java.util.stream.Stream; /** - * Adds the attribute summary transform ({@link SummaryTransform#ATTRIBUTE} to all {@link SummaryField} having an imported + * Adds the data type and attribute summary transform ({@link SummaryTransform#ATTRIBUTE} to all {@link SummaryField} having an imported * field as source. * * @author bjorncs */ -public class AddAttributeTransformToSummaryOfImportedFields extends Processor { +public class AddDataTypeAndTransformToSummaryOfImportedFields extends Processor { - public AddAttributeTransformToSummaryOfImportedFields(Schema schema, - DeployLogger deployLogger, - RankProfileRegistry rankProfileRegistry, - QueryProfiles queryProfiles) { + public AddDataTypeAndTransformToSummaryOfImportedFields(Schema schema, + DeployLogger deployLogger, + RankProfileRegistry rankProfileRegistry, + QueryProfiles queryProfiles) { super(schema, deployLogger, rankProfileRegistry, queryProfiles); } @@ -39,19 +41,29 @@ public class AddAttributeTransformToSummaryOfImportedFields extends Processor { private void setTransform(ImmutableSDField field) { if (field instanceof ImmutableImportedComplexSDField) { - getSummaryFieldsForImportedField(field).forEach(AddAttributeTransformToSummaryOfImportedFields::setAttributeCombinerTransform); + getSummaryFieldsForImportedField(field).forEach(summaryField -> setAttributeCombinerTransform(field, summaryField)); } else { - getSummaryFieldsForImportedField(field).forEach(AddAttributeTransformToSummaryOfImportedFields::setAttributeTransform); + getSummaryFieldsForImportedField(field).forEach(summaryField -> setAttributeTransform(field, summaryField)); } } - private static void setAttributeTransform(SummaryField summaryField) { + private static void setAttributeTransform(ImmutableSDField field, SummaryField summaryField) { + if (summaryField.hasUnresolvedType()) { + if (field.getDataType().equals(DataType.LONG) && summaryField.getTransform().equals(SummaryTransform.GEOPOS)) { + summaryField.setResolvedDataType(PositionDataType.INSTANCE); + } else { + summaryField.setResolvedDataType(field.getDataType()); + } + } if (summaryField.getTransform() == SummaryTransform.NONE) { summaryField.setTransform(SummaryTransform.ATTRIBUTE); } } - private static void setAttributeCombinerTransform(SummaryField summaryField) { + private static void setAttributeCombinerTransform(ImmutableSDField field, SummaryField summaryField) { + if (summaryField.hasUnresolvedType()) { + summaryField.setResolvedDataType(field.getDataType()); + } if (summaryField.getTransform() == SummaryTransform.MATCHED_ATTRIBUTE_ELEMENTS_FILTER) { // This field already has the correct transform. return; diff --git a/config-model/src/main/java/com/yahoo/schema/processing/AdjustSummaryTransforms.java b/config-model/src/main/java/com/yahoo/schema/processing/AdjustSummaryTransforms.java new file mode 100644 index 00000000000..dd6f118d113 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/processing/AdjustSummaryTransforms.java @@ -0,0 +1,82 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.processing; + +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.schema.RankProfileRegistry; +import com.yahoo.schema.Schema; +import com.yahoo.schema.derived.SummaryClass; +import com.yahoo.schema.document.Attribute; +import com.yahoo.schema.document.ImmutableSDField; +import com.yahoo.vespa.documentmodel.SummaryField; +import com.yahoo.vespa.documentmodel.SummaryTransform; +import com.yahoo.vespa.model.container.search.QueryProfiles; + +import static com.yahoo.schema.document.ComplexAttributeFieldUtils.isComplexFieldWithOnlyStructFieldAttributes; + +/** + * Adds the corresponding summary transform for all "documentid" summary fields. + * For summary fields without an existing transform: + * - Adds the attribute transforms where the source field has an attribute vector. + * - Adds the attribute combiner transform where the source field is a struct field where all subfields have attribute + * vector. + * - Add the copy transform where the source field is a struct or map field with a different name. + * + * @author geirst + */ +public class AdjustSummaryTransforms extends Processor { + + public AdjustSummaryTransforms(Schema schema, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { + super(schema, deployLogger, rankProfileRegistry, queryProfiles); + } + + @Override + public void process(boolean validate, boolean documentsOnly) { + for (var summary : schema.getSummaries().values()) { + for (var summaryField : summary.getSummaryFields().values()) { + makeDocumentIdTransformIfAppropriate(summaryField); + makeAttributeTransformIfAppropriate(summaryField, schema); + makeAttributeCombinerTransformIfAppropriate(summaryField, schema); + makeCopyTransformIfAppropriate(summaryField, schema); + } + } + } + + private void makeDocumentIdTransformIfAppropriate(SummaryField summaryField) + { + if (summaryField.getName().equals(SummaryClass.DOCUMENT_ID_FIELD)) { + summaryField.setTransform(SummaryTransform.DOCUMENT_ID); + } + } + + /** If the source is an attribute, make this use the attribute transform */ + private void makeAttributeTransformIfAppropriate(SummaryField summaryField, Schema schema) { + if (summaryField.getTransform() != SummaryTransform.NONE) return; + Attribute attribute = schema.getAttribute(summaryField.getSingleSource()); + if (attribute == null) return; + summaryField.setTransform(SummaryTransform.ATTRIBUTE); + } + + /** If the source is a complex field with only struct field attributes then make this use the attribute combiner transform */ + private void makeAttributeCombinerTransformIfAppropriate(SummaryField summaryField, Schema schema) { + if (summaryField.getTransform() == SummaryTransform.NONE) { + String sourceFieldName = summaryField.getSingleSource(); + ImmutableSDField source = schema.getField(sourceFieldName); + if (source != null && isComplexFieldWithOnlyStructFieldAttributes(source)) { + summaryField.setTransform(SummaryTransform.ATTRIBUTECOMBINER); + } + } + } + + /* + * This function must be called after makeAttributeCombinerTransformIfAppropriate(). + */ + private void makeCopyTransformIfAppropriate(SummaryField summaryField, Schema schema) { + if (summaryField.getTransform() == SummaryTransform.NONE) { + String sourceFieldName = summaryField.getSingleSource(); + ImmutableSDField source = schema.getField(sourceFieldName); + if (source != null && source.usesStructOrMap() && summaryField.hasExplicitSingleSource()) { + summaryField.setTransform(SummaryTransform.COPY); + } + } + } +} diff --git a/config-model/src/main/java/com/yahoo/schema/processing/IndexingOutputs.java b/config-model/src/main/java/com/yahoo/schema/processing/IndexingOutputs.java index 1d279242895..e4116c3f9d5 100644 --- a/config-model/src/main/java/com/yahoo/schema/processing/IndexingOutputs.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/IndexingOutputs.java @@ -78,7 +78,8 @@ public class IndexingOutputs extends Processor { return; } dynamicSummary.add(summaryName); - } else if (summaryTransform != SummaryTransform.ATTRIBUTE) { + } else if (summaryTransform != SummaryTransform.ATTRIBUTE && + summaryTransform != SummaryTransform.TOKENS) { staticSummary.add(summaryName); } } diff --git a/config-model/src/main/java/com/yahoo/schema/processing/MakeDefaultSummaryTheSuperSet.java b/config-model/src/main/java/com/yahoo/schema/processing/MakeDefaultSummaryTheSuperSet.java index 610021c510d..420df3ee575 100644 --- a/config-model/src/main/java/com/yahoo/schema/processing/MakeDefaultSummaryTheSuperSet.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/MakeDefaultSummaryTheSuperSet.java @@ -41,6 +41,7 @@ public class MakeDefaultSummaryTheSuperSet extends Processor { if (summaryField.getTransform() == SummaryTransform.ATTRIBUTE) continue; if (summaryField.getTransform() == SummaryTransform.ATTRIBUTECOMBINER) continue; if (summaryField.getTransform() == SummaryTransform.MATCHED_ATTRIBUTE_ELEMENTS_FILTER) continue; + if (summaryField.getTransform() == SummaryTransform.TOKENS) continue; defaultSummary.add(summaryField.clone()); } diff --git a/config-model/src/main/java/com/yahoo/schema/processing/Processing.java b/config-model/src/main/java/com/yahoo/schema/processing/Processing.java index 89e6a1533d0..c23d87e9eba 100644 --- a/config-model/src/main/java/com/yahoo/schema/processing/Processing.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/Processing.java @@ -49,15 +49,16 @@ public class Processing { DictionaryProcessor::new, WordMatch::new, ImportedFieldsResolver::new, + AddDataTypeAndTransformToSummaryOfImportedFields::new, ImplicitSummaries::new, ImplicitSummaryFields::new, AdjustPositionSummaryFields::new, - SummaryTransformForDocumentId::new, SummaryConsistency::new, + AdjustSummaryTransforms::new, SummaryNamesFieldCollisions::new, SummaryFieldsMustHaveValidSource::new, + TokensTransformValidator::new, MatchedElementsOnlyResolver::new, - AddAttributeTransformToSummaryOfImportedFields::new, MakeDefaultSummaryTheSuperSet::new, Bolding::new, AttributeProperties::new, diff --git a/config-model/src/main/java/com/yahoo/schema/processing/ReservedFunctionNames.java b/config-model/src/main/java/com/yahoo/schema/processing/ReservedFunctionNames.java index f4d2faf9444..0987521831b 100644 --- a/config-model/src/main/java/com/yahoo/schema/processing/ReservedFunctionNames.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/ReservedFunctionNames.java @@ -9,6 +9,7 @@ import com.yahoo.searchlib.rankingexpression.parser.RankingExpressionParserConst import com.yahoo.vespa.model.container.search.QueryProfiles; import java.util.Arrays; +import java.util.HashSet; import java.util.Set; import java.util.logging.Level; import java.util.stream.Collectors; @@ -46,8 +47,28 @@ public class ReservedFunctionNames extends Processor { } private static Set<String> getReservedNames() { - return Arrays.stream(RankingExpressionParserConstants.tokenImage) - .map(token -> token.substring(1, token.length()-1)).collect(Collectors.toUnmodifiableSet()); + Set<String> temp = new HashSet<>(); + Arrays.stream(RankingExpressionParserConstants.tokenImage) + .map(token -> token.substring(1, token.length()-1)).forEach(name -> temp.add(name)); + temp.add("attribute"); + temp.add("constant"); + temp.add("customTokenInputIds"); + temp.add("firstphase"); + temp.add("globalphase"); + temp.add("normalize_linear"); + temp.add("onnx"); + temp.add("onnx_vespa"); + temp.add("query"); + temp.add("reciprocal_rank"); + temp.add("reciprocal_rank_fusion"); + temp.add("secondphase"); + temp.add("tensor"); + temp.add("tokenAttentionMask"); + temp.add("tokenInputIds"); + temp.add("tokenTypeIds"); + temp.add("value"); + temp.add("xgboost"); + return Set.copyOf(temp); } } diff --git a/config-model/src/main/java/com/yahoo/schema/processing/SummaryConsistency.java b/config-model/src/main/java/com/yahoo/schema/processing/SummaryConsistency.java index a7ff50ffcca..4b214e00d65 100644 --- a/config-model/src/main/java/com/yahoo/schema/processing/SummaryConsistency.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/SummaryConsistency.java @@ -8,14 +8,11 @@ import com.yahoo.schema.RankProfileRegistry; import com.yahoo.schema.Schema; import com.yahoo.schema.document.Attribute; import com.yahoo.document.WeightedSetDataType; -import com.yahoo.schema.document.ImmutableSDField; import com.yahoo.vespa.documentmodel.DocumentSummary; import com.yahoo.vespa.documentmodel.SummaryField; import com.yahoo.vespa.documentmodel.SummaryTransform; import com.yahoo.vespa.model.container.search.QueryProfiles; -import static com.yahoo.schema.document.ComplexAttributeFieldUtils.isComplexFieldWithOnlyStructFieldAttributes; - /** * Ensure that summary field transforms for fields having the same name * are consistent across summary classes @@ -35,9 +32,6 @@ public class SummaryConsistency extends Processor { for (SummaryField summaryField : summary.getSummaryFields().values()) { assertConsistency(summaryField, schema, validate); - makeAttributeTransformIfAppropriate(summaryField, schema); - makeAttributeCombinerTransformIfAppropriate(summaryField, schema); - makeCopyTransformIfAppropriate(summaryField, schema); } } } @@ -60,38 +54,6 @@ public class SummaryConsistency extends Processor { } } - /** If the source is an attribute, make this use the attribute transform */ - private void makeAttributeTransformIfAppropriate(SummaryField summaryField, Schema schema) { - if (summaryField.getTransform() != SummaryTransform.NONE) return; - Attribute attribute = schema.getAttribute(summaryField.getSingleSource()); - if (attribute == null) return; - summaryField.setTransform(SummaryTransform.ATTRIBUTE); - } - - /** If the source is a complex field with only struct field attributes then make this use the attribute combiner transform */ - private void makeAttributeCombinerTransformIfAppropriate(SummaryField summaryField, Schema schema) { - if (summaryField.getTransform() == SummaryTransform.NONE) { - String sourceFieldName = summaryField.getSingleSource(); - ImmutableSDField source = schema.getField(sourceFieldName); - if (source != null && isComplexFieldWithOnlyStructFieldAttributes(source)) { - summaryField.setTransform(SummaryTransform.ATTRIBUTECOMBINER); - } - } - } - - /* - * This function must be called after makeAttributeCombinerTransformIfAppropriate(). - */ - private void makeCopyTransformIfAppropriate(SummaryField summaryField, Schema schema) { - if (summaryField.getTransform() == SummaryTransform.NONE) { - String sourceFieldName = summaryField.getSingleSource(); - ImmutableSDField source = schema.getField(sourceFieldName); - if (source != null && source.usesStructOrMap() && summaryField.hasExplicitSingleSource()) { - summaryField.setTransform(SummaryTransform.COPY); - } - } - } - private void assertConsistentTypes(SummaryField existing, SummaryField seen) { if (existing.getDataType() instanceof WeightedSetDataType && seen.getDataType() instanceof WeightedSetDataType && ((WeightedSetDataType)existing.getDataType()).getNestedType().equals(((WeightedSetDataType)seen.getDataType()).getNestedType())) diff --git a/config-model/src/main/java/com/yahoo/schema/processing/SummaryTransformForDocumentId.java b/config-model/src/main/java/com/yahoo/schema/processing/SummaryTransformForDocumentId.java deleted file mode 100644 index 388aa93e81c..00000000000 --- a/config-model/src/main/java/com/yahoo/schema/processing/SummaryTransformForDocumentId.java +++ /dev/null @@ -1,32 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.schema.processing; - -import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.schema.RankProfileRegistry; -import com.yahoo.schema.Schema; -import com.yahoo.schema.derived.SummaryClass; -import com.yahoo.vespa.documentmodel.SummaryTransform; -import com.yahoo.vespa.model.container.search.QueryProfiles; - -/** - * Adds the corresponding summary transform for all "documentid" summary fields. - * - * @author geirst - */ -public class SummaryTransformForDocumentId extends Processor { - - public SummaryTransformForDocumentId(Schema schema, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { - super(schema, deployLogger, rankProfileRegistry, queryProfiles); - } - - @Override - public void process(boolean validate, boolean documentsOnly) { - for (var summary : schema.getSummaries().values()) { - for (var summaryField : summary.getSummaryFields().values()) { - if (summaryField.getName().equals(SummaryClass.DOCUMENT_ID_FIELD)) { - summaryField.setTransform(SummaryTransform.DOCUMENT_ID); - } - } - } - } -} diff --git a/config-model/src/main/java/com/yahoo/schema/processing/TokensTransformValidator.java b/config-model/src/main/java/com/yahoo/schema/processing/TokensTransformValidator.java new file mode 100644 index 00000000000..7988a0b9ceb --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/processing/TokensTransformValidator.java @@ -0,0 +1,50 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.processing; + +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.document.DataType; +import com.yahoo.schema.RankProfileRegistry; +import com.yahoo.schema.Schema; +import com.yahoo.vespa.documentmodel.SummaryTransform; +import com.yahoo.vespa.model.container.search.QueryProfiles; + +/* + * Check that summary fields with summary transform 'tokens' have a source field with a data type that is one of + * string, array<string> or weightedset<string>. + */ +public class TokensTransformValidator extends Processor { + public TokensTransformValidator(Schema schema, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { + super(schema, deployLogger, rankProfileRegistry, queryProfiles); + } + + @Override + public void process(boolean validate, boolean documentsOnly) { + if (!validate || documentsOnly) { + return; + } + for (var summary : schema.getSummaries().values()) { + for (var summaryField : summary.getSummaryFields().values()) { + if (summaryField.getTransform().isTokens()) { + var source = summaryField.getSingleSource(); + if (source != null) { + var field = schema.getField(source); + if (field != null) { + var type = field.getDataType(); + var innerType = type.getPrimitiveType(); + if (innerType != DataType.STRING) { + throw new IllegalArgumentException("For schema '" + schema.getName() + + "', document-summary '" + summary.getName() + + "', summary field '" + summaryField.getName() + + "', source field '" + field.getName() + + "', source field type '" + type.getName() + + "': transform '" + SummaryTransform.TOKENS.getName() + + "' is only allowed for fields of type" + + " string, array<string> or weightedset<string>"); + } + } + } + } + } + } + } +} diff --git a/config-model/src/main/java/com/yahoo/schema/processing/ValidateFieldTypes.java b/config-model/src/main/java/com/yahoo/schema/processing/ValidateFieldTypes.java index dd2fd72b280..662f3fc970b 100644 --- a/config-model/src/main/java/com/yahoo/schema/processing/ValidateFieldTypes.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/ValidateFieldTypes.java @@ -49,7 +49,9 @@ public class ValidateFieldTypes extends Processor { final protected void verifySummaryFields(String searchName, Map<String, DataType> seenFields) { for (DocumentSummary summary : schema.getSummaries().values()) { for (SummaryField field : summary.getSummaryFields().values()) { - checkFieldType(searchName, "summary field", field.getName(), field.getDataType(), seenFields); + if (!field.hasUnresolvedType()) { + checkFieldType(searchName, "summary field", field.getName(), field.getDataType(), seenFields); + } } } } diff --git a/config-model/src/main/java/com/yahoo/vespa/documentmodel/SummaryField.java b/config-model/src/main/java/com/yahoo/vespa/documentmodel/SummaryField.java index 2a316c8af60..1c53ee36497 100644 --- a/config-model/src/main/java/com/yahoo/vespa/documentmodel/SummaryField.java +++ b/config-model/src/main/java/com/yahoo/vespa/documentmodel/SummaryField.java @@ -66,6 +66,7 @@ public class SummaryField extends Field implements Cloneable, TypedKey { /** True if this field was defined implicitly */ private boolean implicit = false; + private boolean unresolvedType = false; /** Creates a summary field with NONE as transform */ public SummaryField(String name, DataType type) { @@ -87,10 +88,24 @@ public class SummaryField extends Field implements Cloneable, TypedKey { this.transform=transform; } + public static SummaryField createWithUnresolvedType(String name) { + /* + * Data type is not available during conversion of + * parsed schema to schema. Use a placeholder data type and tag the summary + * field as having an unresolved type. + */ + var summaryField = new SummaryField(name, DataType.NONE); + summaryField.unresolvedType = true; + return summaryField; + } + + public void setImplicit(boolean implicit) { this.implicit=implicit; } public boolean isImplicit() { return implicit; } + public boolean hasUnresolvedType() { return unresolvedType; } + public void setTransform(SummaryTransform transform) { this.transform = transform; if (SummaryTransform.DYNAMICTEASER.equals(transform) || SummaryTransform.BOLDED.equals(transform)) { @@ -246,6 +261,7 @@ public class SummaryField extends Field implements Cloneable, TypedKey { clone.sources = new LinkedHashSet<>(this.sources); if (this.destinations != null) clone.destinations = new LinkedHashSet<>(destinations); + clone.unresolvedType = unresolvedType; return clone; } catch (CloneNotSupportedException e) { @@ -272,6 +288,14 @@ public class SummaryField extends Field implements Cloneable, TypedKey { return true; } + public void setResolvedDataType(DataType type) { + this.dataType = type; + if (!hasForcedId()) { + this.fieldId = calculateIdV7(null); + } + unresolvedType = false; + } + public VsmCommand getVsmCommand() { return vsmCommand; } diff --git a/config-model/src/main/java/com/yahoo/vespa/documentmodel/SummaryTransform.java b/config-model/src/main/java/com/yahoo/vespa/documentmodel/SummaryTransform.java index 575a3a748e6..58f47680f9f 100644 --- a/config-model/src/main/java/com/yahoo/vespa/documentmodel/SummaryTransform.java +++ b/config-model/src/main/java/com/yahoo/vespa/documentmodel/SummaryTransform.java @@ -23,7 +23,8 @@ public enum SummaryTransform { MATCHED_ELEMENTS_FILTER("matchedelementsfilter"), MATCHED_ATTRIBUTE_ELEMENTS_FILTER("matchedattributeelementsfilter"), COPY("copy"), - DOCUMENT_ID("documentid"); + DOCUMENT_ID("documentid"), + TOKENS("tokens"); private final String name; @@ -68,6 +69,8 @@ public enum SummaryTransform { return this==DYNAMICBOLDED || this==DYNAMICTEASER; } + public boolean isTokens() { return this == TOKENS; } + /** Returns whether this transform always gets its value by accessing memory only */ public boolean isInMemory() { return switch (this) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java index 9fab6a2b17b..348b84367d5 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java @@ -48,6 +48,7 @@ import java.util.stream.Stream; import static ai.vespa.metrics.set.DefaultMetrics.defaultMetricSet; import static ai.vespa.metrics.set.MetricSet.empty; import static ai.vespa.metrics.set.SystemMetrics.systemMetricSet; +import static ai.vespa.metrics.set.Vespa9DefaultMetricSet.vespa9defaultMetricSet; import static com.yahoo.vespa.model.admin.metricsproxy.ConsumersConfigGenerator.addMetrics; import static com.yahoo.vespa.model.admin.metricsproxy.ConsumersConfigGenerator.generateConsumers; import static com.yahoo.vespa.model.admin.metricsproxy.ConsumersConfigGenerator.toConsumerBuilder; @@ -169,8 +170,7 @@ public class MetricsProxyContainerCluster extends ContainerCluster<MetricsProxyC public MetricsConsumer newDefaultConsumer() { if (isHostedVespa()) { - // TODO: use different metric set for hosted vespa. - return MetricsConsumer.consumer(NEW_DEFAULT_CONSUMER_ID, defaultMetricSet, systemMetricSet); + return MetricsConsumer.consumer(NEW_DEFAULT_CONSUMER_ID, vespa9defaultMetricSet, systemMetricSet); } return MetricsConsumer.consumer(NEW_DEFAULT_CONSUMER_ID, defaultMetricSet, systemMetricSet); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/JvmHeapSizeValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/JvmHeapSizeValidator.java index 510fff66c10..425a662bb2d 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/JvmHeapSizeValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/JvmHeapSizeValidator.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.model.application.validation; import com.yahoo.config.model.deploy.DeployState; +import com.yahoo.text.Text; import com.yahoo.vespa.model.VespaModel; import java.util.logging.Level; @@ -32,21 +33,20 @@ public class JvmHeapSizeValidator extends Validator { double gbLimit = 0.6; double availableMemoryGb = mp.availableMemoryGb().getAsDouble(); double modelCostGb = jvmModelCost / (1024D * 1024 * 1024); - ds.getDeployLogger().log(Level.FINE, () -> "JVM: %d%% (limit: %d%%), %.2fGB (limit: %.2fGB), ONNX: %.2fGB" - .formatted(mp.percentage(), percentLimit, availableMemoryGb, gbLimit, modelCostGb)); + ds.getDeployLogger().log(Level.FINE, () -> Text.format("JVM: %d%% (limit: %d%%), %.2fGB (limit: %.2fGB), ONNX: %.2fGB", + mp.percentage(), percentLimit, availableMemoryGb, gbLimit, modelCostGb)); if (mp.percentage() < percentLimit) { - throw new IllegalArgumentException( - ("Allocated percentage of memory of JVM in cluster '%s' is too low (%d%% < %d%%). " + + throw new IllegalArgumentException(Text.format("Allocated percentage of memory of JVM in cluster '%s' is too low (%d%% < %d%%). " + "Estimated cost of ONNX models is %.2fGB. Either use a node flavor with more memory or use less expensive models. " + - "You may override this validation by specifying 'allocated-memory' (https://docs.vespa.ai/en/performance/container-tuning.html#jvm-heap-size).") - .formatted(clusterId, mp.percentage(), percentLimit, modelCostGb)); + "You may override this validation by specifying 'allocated-memory' (https://docs.vespa.ai/en/performance/container-tuning.html#jvm-heap-size).", + clusterId, mp.percentage(), percentLimit, modelCostGb)); } if (availableMemoryGb < gbLimit) { throw new IllegalArgumentException( - ("Allocated memory to JVM in cluster '%s' is too low (%.2fGB < %.2fGB). " + + Text.format("Allocated memory to JVM in cluster '%s' is too low (%.2fGB < %.2fGB). " + "Estimated cost of ONNX models is %.2fGB. Either use a node flavor with more memory or use less expensive models. " + - "You may override this validation by specifying 'allocated-memory' (https://docs.vespa.ai/en/performance/container-tuning.html#jvm-heap-size).") - .formatted(clusterId, availableMemoryGb, gbLimit, modelCostGb)); + "You may override this validation by specifying 'allocated-memory' (https://docs.vespa.ai/en/performance/container-tuning.html#jvm-heap-size).", + clusterId, availableMemoryGb, gbLimit, modelCostGb)); } } }); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomSearchTuningBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomSearchTuningBuilder.java index 50e6cace8b8..273e5580403 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomSearchTuningBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomSearchTuningBuilder.java @@ -1,6 +1,7 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.builder.xml.dom; +import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.text.XML; import com.yahoo.config.model.producer.AnyConfigProducer; @@ -8,6 +9,8 @@ import com.yahoo.config.model.producer.TreeConfigProducer; import com.yahoo.vespa.model.search.Tuning; import org.w3c.dom.Element; +import java.util.logging.Level; + /** * Builder for the tuning config for a search cluster. * @@ -20,7 +23,7 @@ public class DomSearchTuningBuilder extends VespaDomBuilder.DomConfigProducerBui Tuning tuning = new Tuning(parent); for (Element e : XML.getChildren(spec)) { if (equals("searchnode", e)) - handleSearchNode(e, tuning); + handleSearchNode(deployState.getDeployLogger(), e, tuning); } return tuning; } @@ -45,7 +48,7 @@ public class DomSearchTuningBuilder extends VespaDomBuilder.DomConfigProducerBui return Double.parseDouble(e.getFirstChild().getNodeValue()); } - private void handleSearchNode(Element spec, Tuning t) { + private void handleSearchNode(DeployLogger deployLogger, Element spec, Tuning t) { t.searchNode = new Tuning.SearchNode(); for (Element e : XML.getChildren(spec)) { if (equals("requestthreads", e)) { @@ -53,13 +56,13 @@ public class DomSearchTuningBuilder extends VespaDomBuilder.DomConfigProducerBui } else if (equals("flushstrategy", e)) { handleFlushStrategy(e, t.searchNode); } else if (equals("resizing", e)) { - handleResizing(e, t.searchNode); + handleResizing(deployLogger, e, t.searchNode); } else if (equals("index", e)) { - handleIndex(e, t.searchNode); + handleIndex(deployLogger, e, t.searchNode); } else if (equals("attribute", e)) { - handleAttribute(e, t.searchNode); + deployLogger.logApplicationPackage(Level.WARNING, "searchnode.attribute is deprecated and ignored."); } else if (equals("summary", e)) { - handleSummary(e, t.searchNode); + handleSummary(deployLogger, e, t.searchNode); } else if (equals("initialize", e)) { handleInitialize(e, t.searchNode); } else if (equals("feeding", e)) { @@ -161,18 +164,19 @@ public class DomSearchTuningBuilder extends VespaDomBuilder.DomConfigProducerBui } } - private void handleResizing(Element spec, Tuning.SearchNode sn) { + private void handleResizing(DeployLogger deployLogger, Element spec, Tuning.SearchNode sn) { sn.resizing = new Tuning.SearchNode.Resizing(); for (Element e : XML.getChildren(spec)) { if (equals("initialdocumentcount", e)) { + deployLogger.logApplicationPackage(Level.WARNING, "resizing.initialdocumentcount is deprecated."); sn.resizing.initialDocumentCount = asInt(e); } else if (equals("amortize-count", e)) { - sn.resizing.amortizeCount = asInt(e); + deployLogger.logApplicationPackage(Level.WARNING, "resizing.amortize-count is deprecated and ignored"); } } } - private void handleIndex(Element spec, Tuning.SearchNode sn) { + private void handleIndex(DeployLogger deployLogger, Element spec, Tuning.SearchNode sn) { sn.index = new Tuning.SearchNode.Index(); for (Element e : XML.getChildren(spec)) { if (equals("io", e)) { @@ -180,9 +184,9 @@ public class DomSearchTuningBuilder extends VespaDomBuilder.DomConfigProducerBui Tuning.SearchNode.Index.Io io = sn.index.io; for (Element e2 : XML.getChildren(e)) { if (equals("write", e2)) { - io.write = Tuning.SearchNode.IoType.fromString(asString(e2)); + deployLogger.logApplicationPackage(Level.WARNING, "index.io.write is deprecated and ignored."); } else if (equals("read", e2)) { - io.read = Tuning.SearchNode.IoType.fromString(asString(e2)); + deployLogger.logApplicationPackage(Level.WARNING, "index.io.read is deprecated and ignored."); } else if (equals("search", e2)) { io.search = Tuning.SearchNode.IoType.fromString(asString(e2)); } @@ -201,28 +205,14 @@ public class DomSearchTuningBuilder extends VespaDomBuilder.DomConfigProducerBui } } - private void handleAttribute(Element spec, Tuning.SearchNode sn) { - sn.attribute = new Tuning.SearchNode.Attribute(); - for (Element e : XML.getChildren(spec)) { - if (equals("io", e)) { - sn.attribute.io = new Tuning.SearchNode.Attribute.Io(); - for (Element e2 : XML.getChildren(e)) { - if (equals("write", e2)) { - sn.attribute.io.write = Tuning.SearchNode.IoType.fromString(asString(e2)); - } - } - } - } - } - - private void handleSummary(Element spec, Tuning.SearchNode sn) { + private void handleSummary(DeployLogger deployLogger, Element spec, Tuning.SearchNode sn) { sn.summary = new Tuning.SearchNode.Summary(); for (Element e : XML.getChildren(spec)) { if (equals("io", e)) { sn.summary.io = new Tuning.SearchNode.Summary.Io(); for (Element e2 : XML.getChildren(e)) { if (equals("write", e2)) { - sn.summary.io.write = Tuning.SearchNode.IoType.fromString(asString(e2)); + deployLogger.logApplicationPackage(Level.WARNING, "summary.io.write is deprecated and ignored."); } else if (equals("read", e2)) { sn.summary.io.read = Tuning.SearchNode.IoType.fromString(asString(e2)); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java index 9821f3b9568..e04711a1c56 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java @@ -49,10 +49,10 @@ import java.util.LinkedHashSet; import java.util.List; import java.util.Optional; import java.util.Set; -import java.util.logging.Level; import java.util.stream.Collectors; import static com.yahoo.vespa.model.container.docproc.DocprocChains.DOCUMENT_TYPE_MANAGER_CLASS; +import static java.util.logging.Level.FINE; /** * A container cluster that is typically set up from the user application. @@ -137,7 +137,7 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat ? Math.min(99, deployState.featureFlags().heapSizePercentage()) : defaultHeapSizePercentageOfAvailableMemory; onnxModelCost = deployState.onnxModelCost().newCalculator( - deployState.getApplicationPackage(), deployState.getDeployLogger()); + deployState.getApplicationPackage(), deployState.getProperties().applicationId()); logger = deployState.getDeployLogger(); } @@ -166,7 +166,8 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat UserConfiguredFiles files = new UserConfiguredFiles(deployState.getFileRegistry(), deployState.getDeployLogger(), deployState.featureFlags(), - userConfiguredUrls); + userConfiguredUrls, + deployState.getApplicationPackage()); for (Component<?, ?> component : getAllComponents()) { files.register(component); } @@ -217,8 +218,8 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat double jvmHeapDeductionGb = dynamicHeapSize ? onnxModelCost.aggregatedModelCostInBytes() / (1024D * 1024 * 1024) : 0; double availableMemory = Math.max(0, totalMemory - Host.memoryOverheadGb - jvmHeapDeductionGb); int memoryPercentage = (int) (availableMemory / totalMemory * availableMemoryPercentage); - logger.log(Level.FINE, () -> "memoryPercentage=%d, availableMemory=%f, totalMemory=%f, availableMemoryPercentage=%d, jvmHeapDeductionGb=%f" - .formatted(memoryPercentage, availableMemory, totalMemory, availableMemoryPercentage, jvmHeapDeductionGb)); + logger.log(FINE, () -> "cluster id '%s': memoryPercentage=%d, availableMemory=%f, totalMemory=%f, availableMemoryPercentage=%d, jvmHeapDeductionGb=%f" + .formatted(id(), memoryPercentage, availableMemory, totalMemory, availableMemoryPercentage, jvmHeapDeductionGb)); return Optional.of(JvmMemoryPercentage.of(memoryPercentage, availableMemory)); } return Optional.empty(); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java b/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java index aa4aa6af32c..4e3b3d1d8cb 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java @@ -109,7 +109,7 @@ public abstract class Container extends AbstractService implements addChild(new SimpleComponent("com.yahoo.container.jdisc.ConfiguredApplication$ApplicationContext")); appendJvmOptions(jvmOmitStackTraceInFastThrowOption(deployState.featureFlags())); - addEnvironmentVariable("VESPA_MALLOC_MMAP_THRESHOLD","0x200000"); + addEnvironmentVariable("VESPA_MALLOC_MMAP_THRESHOLD","0x800000"); } protected String jvmOmitStackTraceInFastThrowOption(ModelContext.FeatureFlags featureFlags) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/component/Handler.java b/config-model/src/main/java/com/yahoo/vespa/model/container/component/Handler.java index 0af970e016a..099255975b6 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/component/Handler.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/component/Handler.java @@ -64,8 +64,8 @@ public class Handler extends Component<Component<?, ?>, ComponentModel> { clientBindings.addAll(Arrays.asList(bindings)); } - public final Set<BindingPattern> getServerBindings() { - return Collections.unmodifiableSet(serverBindings); + public final Collection<BindingPattern> getServerBindings() { + return List.copyOf(serverBindings); } public final List<BindingPattern> getClientBindings() { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/http/JettyHttpServer.java b/config-model/src/main/java/com/yahoo/vespa/model/container/http/JettyHttpServer.java index d2faff7850b..b14495756c3 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/http/JettyHttpServer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/http/JettyHttpServer.java @@ -9,6 +9,7 @@ import com.yahoo.jdisc.http.ServerConfig; import com.yahoo.osgi.provider.model.ComponentModel; import com.yahoo.vespa.model.container.ApplicationContainerCluster; import com.yahoo.vespa.model.container.ContainerCluster; +import com.yahoo.vespa.model.container.component.ConnectionLogComponent; import com.yahoo.vespa.model.container.component.SimpleComponent; import java.util.ArrayList; @@ -24,13 +25,11 @@ import java.util.TreeSet; public class JettyHttpServer extends SimpleComponent implements ServerConfig.Producer { private final ContainerCluster<?> cluster; - private volatile boolean isHostedVespa; private final List<ConnectorFactory> connectorFactories = new ArrayList<>(); private final SortedSet<String> ignoredUserAgentsList = new TreeSet<>(); public JettyHttpServer(String componentId, ContainerCluster<?> cluster, DeployState deployState) { super(new ComponentModel(componentId, com.yahoo.jdisc.http.server.jetty.JettyHttpServer.class.getName(), null)); - this.isHostedVespa = deployState.isHosted(); this.cluster = cluster; FilterBindingsProviderComponent filterBindingsProviderComponent = new FilterBindingsProviderComponent(componentId); addChild(filterBindingsProviderComponent); @@ -42,8 +41,6 @@ public class JettyHttpServer extends SimpleComponent implements ServerConfig.Pro } } - public void setHostedVespa(boolean isHostedVespa) { this.isHostedVespa = isHostedVespa; } - public void addConnector(ConnectorFactory connectorFactory) { connectorFactories.add(connectorFactory); addChild(connectorFactory); @@ -64,10 +61,8 @@ public class JettyHttpServer extends SimpleComponent implements ServerConfig.Pro .ignoredUserAgents(ignoredUserAgentsList) .searchHandlerPaths(List.of("/search")) ); - if (isHostedVespa) { - // Enable connection log hosted Vespa + if (cluster.getAllComponents().stream().anyMatch(c -> c instanceof ConnectionLogComponent)) builder.connectionLog(new ServerConfig.ConnectionLog.Builder().enabled(true)); - } configureJettyThreadpool(builder); builder.stopTimeout(300); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/http/ssl/HostedSslConnectorFactory.java b/config-model/src/main/java/com/yahoo/vespa/model/container/http/ssl/HostedSslConnectorFactory.java index c75aca7a5fa..08b0398a98f 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/http/ssl/HostedSslConnectorFactory.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/http/ssl/HostedSslConnectorFactory.java @@ -9,7 +9,10 @@ import com.yahoo.vespa.model.container.http.ConnectorFactory; import java.time.Duration; import java.util.ArrayList; import java.util.Collection; +import java.util.Collections; import java.util.List; +import java.util.Set; +import java.util.TreeSet; /** * Component specification for {@link com.yahoo.jdisc.http.server.jetty.ConnectorFactory} with hosted specific configuration. @@ -25,6 +28,7 @@ public class HostedSslConnectorFactory extends ConnectorFactory { private final Duration endpointConnectionTtl; private final List<String> remoteAddressHeaders; private final List<String> remotePortHeaders; + private final Set<String> knownServerNames; public static Builder builder(String name, int listenPort) { return new Builder(name, listenPort); } @@ -37,6 +41,7 @@ public class HostedSslConnectorFactory extends ConnectorFactory { this.endpointConnectionTtl = builder.endpointConnectionTtl; this.remoteAddressHeaders = List.copyOf(builder.remoteAddressHeaders); this.remotePortHeaders = List.copyOf(builder.remotePortHeaders); + this.knownServerNames = Collections.unmodifiableSet(new TreeSet<>(builder.knownServerNames)); } private static SslProvider createSslProvider(Builder builder) { @@ -70,7 +75,8 @@ public class HostedSslConnectorFactory extends ConnectorFactory { .maxConnectionLife(endpointConnectionTtl != null ? endpointConnectionTtl.toSeconds() : 0) .accessLog(new ConnectorConfig.AccessLog.Builder() .remoteAddressHeaders(remoteAddressHeaders) - .remotePortHeaders(remotePortHeaders)); + .remotePortHeaders(remotePortHeaders)) + .serverName.known(knownServerNames); } @@ -89,6 +95,7 @@ public class HostedSslConnectorFactory extends ConnectorFactory { String tlsCaCertificatesPem; String tlsCaCertificatesPath; boolean tokenEndpoint; + Set<String> knownServerNames = Set.of(); private Builder(String name, int port) { this.name = name; this.port = port; } public Builder clientAuth(SslClientAuth auth) { clientAuth = auth; return this; } @@ -101,7 +108,7 @@ public class HostedSslConnectorFactory extends ConnectorFactory { public Builder tokenEndpoint(boolean enable) { this.tokenEndpoint = enable; return this; } public Builder remoteAddressHeader(String header) { this.remoteAddressHeaders.add(header); return this; } public Builder remotePortHeader(String header) { this.remotePortHeaders.add(header); return this; } - + public Builder knownServerNames(Set<String> knownServerNames) { this.knownServerNames = Set.copyOf(knownServerNames); return this; } public HostedSslConnectorFactory build() { return new HostedSslConnectorFactory(this); } } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ConfigServerContainerModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ConfigServerContainerModelBuilder.java index 7653d814d8a..119a3ad18c2 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ConfigServerContainerModelBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ConfigServerContainerModelBuilder.java @@ -3,19 +3,14 @@ package com.yahoo.vespa.model.container.xml; import com.yahoo.config.model.ConfigModelContext; import com.yahoo.config.model.deploy.DeployState; -import com.yahoo.container.logging.AccessLog; import com.yahoo.container.logging.FileConnectionLog; -import com.yahoo.jdisc.http.server.jetty.VoidRequestLog; import com.yahoo.vespa.model.container.ApplicationContainerCluster; import com.yahoo.vespa.model.container.ContainerModel; -import com.yahoo.vespa.model.container.component.AccessLogComponent; import com.yahoo.vespa.model.container.component.ConnectionLogComponent; import com.yahoo.vespa.model.container.configserver.ConfigserverCluster; import com.yahoo.vespa.model.container.configserver.option.CloudConfigOptions; import org.w3c.dom.Element; -import static com.yahoo.vespa.model.container.component.AccessLogComponent.AccessLogType.jsonAccessLog; - /** * Builds the config model for the standalone config server. * @@ -57,12 +52,6 @@ public class ConfigServerContainerModelBuilder extends ContainerModelBuilder { } @Override - protected void addHttp(DeployState deployState, Element spec, ApplicationContainerCluster cluster, ConfigModelContext context) { - super.addHttp(deployState, spec, cluster, context); - cluster.getHttp().getHttpServer().get().setHostedVespa(isHosted()); - } - - @Override protected void addModelEvaluationRuntime(ApplicationContainerCluster cluster) { // Model evaluation bundles are pre-installed in the standalone container. } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java index 830440aaf8e..18020f5df5d 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java @@ -574,7 +574,12 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { Reader reader = file.createReader(); String certPem = IOUtils.readAll(reader); reader.close(); - List<X509Certificate> x509Certificates = X509CertificateUtils.certificateListFromPem(certPem); + List<X509Certificate> x509Certificates; + try { + x509Certificates = X509CertificateUtils.certificateListFromPem(certPem); + } catch (IllegalArgumentException e) { + throw new IllegalArgumentException("File %s contains an invalid certificate".formatted(file.getPath().getRelative()), e); + } if (x509Certificates.isEmpty()) { throw new IllegalArgumentException("File %s does not contain any certificates.".formatted(file.getPath().getRelative())); } @@ -601,6 +606,11 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { var endpointCert = state.endpointCertificateSecrets().orElse(null); if (endpointCert != null) { builder.endpointCertificate(endpointCert); + Set<String> mtlsEndpointNames = state.getEndpoints().stream() + .filter(endpoint -> endpoint.authMethod() == ApplicationClusterEndpoint.AuthMethod.mtls) + .flatMap(endpoint -> endpoint.names().stream()) + .collect(Collectors.toSet()); + builder.knownServerNames(mtlsEndpointNames); boolean isPublic = state.zone().system().isPublic(); List<X509Certificate> clientCertificates = getClientCertificates(cluster); if (isPublic) { @@ -654,6 +664,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { .remoteAddressHeader("X-Forwarded-For") .remotePortHeader("X-Forwarded-Port") .clientAuth(SslClientAuth.NEED) + .knownServerNames(tokenEndpoints) .build(); server.addConnector(connector); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/UserConfiguredFiles.java b/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/UserConfiguredFiles.java index e4eaa02acd5..9729d7d806b 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/UserConfiguredFiles.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/UserConfiguredFiles.java @@ -3,6 +3,8 @@ package com.yahoo.vespa.model.filedistribution; import com.yahoo.config.FileReference; import com.yahoo.config.ModelReference; +import com.yahoo.config.application.api.ApplicationFile; +import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.application.api.FileRegistry; import com.yahoo.config.model.api.ModelContext; @@ -12,18 +14,17 @@ import com.yahoo.path.Path; import com.yahoo.vespa.config.ConfigDefinition; import com.yahoo.vespa.config.ConfigDefinitionKey; import com.yahoo.vespa.config.ConfigPayloadBuilder; - import com.yahoo.yolean.Exceptions; -import java.io.File; import java.io.Serializable; import java.util.Collection; import java.util.HashMap; import java.util.Map; import java.util.Optional; -import java.util.logging.Level; import static com.yahoo.vespa.model.container.ApplicationContainerCluster.UserConfiguredUrls; +import static java.util.logging.Level.INFO; +import static java.util.logging.Level.WARNING; /** * Utility methods for registering file distribution of files/paths/urls/models defined by the user. @@ -37,14 +38,17 @@ public class UserConfiguredFiles implements Serializable { private final DeployLogger logger; private final UserConfiguredUrls userConfiguredUrls; private final String unknownConfigDefinition; + private final ApplicationPackage applicationPackage; public UserConfiguredFiles(FileRegistry fileRegistry, DeployLogger logger, ModelContext.FeatureFlags featureFlags, - UserConfiguredUrls userConfiguredUrls) { + UserConfiguredUrls userConfiguredUrls, + ApplicationPackage applicationPackage) { this.fileRegistry = fileRegistry; this.logger = logger; this.userConfiguredUrls = userConfiguredUrls; this.unknownConfigDefinition = featureFlags.unknownConfigDefinition(); + this.applicationPackage = applicationPackage; } /** @@ -69,8 +73,7 @@ public class UserConfiguredFiles implements Serializable { if (configDefinition == null) { String message = "Unable to find config definition " + key + ". Will not register files for file distribution for this config"; switch (unknownConfigDefinition) { - case "log" -> logger.logApplicationPackage(Level.INFO, message); - case "warning" -> logger.logApplicationPackage(Level.WARNING, message); + case "warning" -> logger.logApplicationPackage(WARNING, message); case "fail" -> throw new IllegalArgumentException("Unable to find config definition for " + key); } return; @@ -156,9 +159,9 @@ public class UserConfiguredFiles implements Serializable { path = Path.fromString(builder.getValue()); } - File file = path.toFile(); - if (file.isDirectory() && (file.listFiles() == null || file.listFiles().length == 0)) - throw new IllegalArgumentException("Directory '" + path.getRelative() + "' is empty"); + ApplicationFile file = applicationPackage.getFile(path); + if (file.isDirectory() && (file.listFiles() == null || file.listFiles().isEmpty())) + logger.logApplicationPackage(INFO, "Directory '" + path.getRelative() + "' is empty"); FileReference reference = registeredFiles.get(path); if (reference == null) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/ml/OnnxModelProbe.java b/config-model/src/main/java/com/yahoo/vespa/model/ml/OnnxModelProbe.java index 5649cd51c95..0f89a839a26 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/ml/OnnxModelProbe.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/ml/OnnxModelProbe.java @@ -8,6 +8,7 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.yahoo.config.application.api.ApplicationFile; import com.yahoo.config.application.api.ApplicationPackage; +import com.yahoo.config.model.api.OnnxMemoryStats; import com.yahoo.io.IOUtils; import com.yahoo.path.Path; import com.yahoo.tensor.TensorType; @@ -45,7 +46,7 @@ public class OnnxModelProbe { String jsonInput = createJsonInput(app.getFileReference(modelPath).getAbsolutePath(), inputTypes); var jsonOutput = callVespaAnalyzeOnnxModel(jsonInput); outputType = outputTypeFromJson(jsonOutput, outputName); - writeMemoryStats(app, modelPath, MemoryStats.fromJson(jsonOutput)); + writeMemoryStats(app, modelPath, OnnxMemoryStats.fromJson(jsonOutput)); if ( ! outputType.equals(TensorType.empty)) { writeProbedOutputType(app, modelPath, contextKey, outputType); } @@ -56,16 +57,11 @@ public class OnnxModelProbe { return outputType; } - private static void writeMemoryStats(ApplicationPackage app, Path modelPath, MemoryStats memoryStats) throws IOException { - String path = app.getFileReference(memoryStatsPath(modelPath)).getAbsolutePath(); + private static void writeMemoryStats(ApplicationPackage app, Path modelPath, OnnxMemoryStats memoryStats) throws IOException { + String path = app.getFileReference(OnnxMemoryStats.memoryStatsFilePath(modelPath)).getAbsolutePath(); IOUtils.writeFile(path, memoryStats.toJson().toPrettyString(), false); } - private static Path memoryStatsPath(Path modelPath) { - var fileName = OnnxModelInfo.asValidIdentifier(modelPath.getRelative()) + ".memory_stats"; - return ApplicationPackage.MODELS_GENERATED_REPLICATED_DIR.append(fileName); - } - private static String createContextKey(String onnxName, Map<String, TensorType> inputTypes) { StringBuilder key = new StringBuilder().append(onnxName).append(":"); inputTypes.entrySet().stream().sorted(Map.Entry.comparingByKey()) @@ -161,14 +157,4 @@ public class OnnxModelProbe { } return jsonParser.readTree(output.toString()); } - - public record MemoryStats(long vmSize, long vmRss) { - static MemoryStats fromJson(JsonNode json) { - return new MemoryStats(json.get("vm_size").asLong(), json.get("vm_rss").asLong()); - } - JsonNode toJson() { - return jsonParser.createObjectNode().put("vm_size", vmSize).put("vm_rss", vmRss); - } - } - } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/NodeResourcesTuning.java b/config-model/src/main/java/com/yahoo/vespa/model/search/NodeResourcesTuning.java index 003cbbe78a8..2beec421faa 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/NodeResourcesTuning.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/NodeResourcesTuning.java @@ -24,7 +24,7 @@ public class NodeResourcesTuning implements ProtonConfig.Producer { final static long MB = 1024 * 1024; public final static long GB = MB * 1024; // This is an approximate number based on observation of a node using 33G memory with 765M docs - private final static long MEMORY_COST_PER_DOCUMENT_STORE_ONLY = 46L; + private final static long MEMORY_COST_PER_DOCUMENT_DB_ONLY = 46L; private final NodeResources resources; private final int threadsPerSearch; private final double fractionOfMemoryReserved; @@ -58,9 +58,10 @@ public class NodeResourcesTuning implements ProtonConfig.Producer { } private void getConfig(ProtonConfig.Documentdb.Builder builder) { + // TODO => Move this to backend to enable ignoring this setting. ProtonConfig.Documentdb dbCfg = builder.build(); if (dbCfg.mode() != ProtonConfig.Documentdb.Mode.Enum.INDEX) { - long numDocs = (long)usableMemoryGb() * GB / MEMORY_COST_PER_DOCUMENT_STORE_ONLY; + long numDocs = (long)usableMemoryGb() * GB / MEMORY_COST_PER_DOCUMENT_DB_ONLY; builder.allocation.initialnumdocs(numDocs/redundancy.readyCopies()); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/Tuning.java b/config-model/src/main/java/com/yahoo/vespa/model/search/Tuning.java index e8d42b701ef..9621ddd1374 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/Tuning.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/Tuning.java @@ -120,8 +120,8 @@ public class Tuning extends AnyConfigProducer implements ProtonConfig.Producer { } public static class Resizing implements ProtonConfig.Producer { + // TODO GC as soon as resource computation is moved to backend. public Integer initialDocumentCount = null; - public Integer amortizeCount = null; @Override public void getConfig(ProtonConfig.Builder builder) { @@ -130,29 +130,16 @@ public class Tuning extends AnyConfigProducer implements ProtonConfig.Producer { db.allocation.initialnumdocs(initialDocumentCount); } } - if (amortizeCount !=null) { - for (ProtonConfig.Documentdb.Builder db : builder.documentdb) { - db.allocation.amortizecount(amortizeCount); - } - } } } public static class Index implements ProtonConfig.Producer { public static class Io implements ProtonConfig.Producer { - public IoType write = null; - public IoType read = null; public IoType search = null; @Override public void getConfig(ProtonConfig.Builder builder) { - if (write != null) { - builder.indexing.write.io(ProtonConfig.Indexing.Write.Io.Enum.valueOf(write.name)); - } - if (read != null) { - builder.indexing.read.io(ProtonConfig.Indexing.Read.Io.Enum.valueOf(read.name)); - } if (search != null) { if (search.equals(IoType.POPULATE)) { builder.search.mmap.options.add(ProtonConfig.Search.Mmap.Options.POPULATE); @@ -184,38 +171,11 @@ public class Tuning extends AnyConfigProducer implements ProtonConfig.Producer { } } - public static class Attribute implements ProtonConfig.Producer { - public static class Io implements ProtonConfig.Producer { - public IoType write = null; - - public Io() {} - - @Override - public void getConfig(ProtonConfig.Builder builder) { - if (write != null) { - builder.attribute.write.io(ProtonConfig.Attribute.Write.Io.Enum.valueOf(write.name)); - } - } - } - - public Io io; - - @Override - public void getConfig(ProtonConfig.Builder builder) { - if (io != null) io.getConfig(builder); - } - - } - public static class Summary implements ProtonConfig.Producer { public static class Io { - public IoType write = null; public IoType read = null; public void getConfig(ProtonConfig.Summary.Builder builder) { - if (write != null) { - builder.write.io(ProtonConfig.Summary.Write.Io.Enum.valueOf(write.name)); - } if (read != null) { if (read.equals(IoType.POPULATE)) { builder.read.io(ProtonConfig.Summary.Read.Io.MMAP); @@ -389,7 +349,6 @@ public class Tuning extends AnyConfigProducer implements ProtonConfig.Producer { public FlushStrategy strategy = null; public Resizing resizing = null; public Index index = null; - public Attribute attribute = null; public Summary summary = null; public Initialize initialize = null; public Feeding feeding = null; @@ -402,7 +361,6 @@ public class Tuning extends AnyConfigProducer implements ProtonConfig.Producer { if (strategy != null) strategy.getConfig(builder); if (resizing != null) resizing.getConfig(builder); if (index != null) index.getConfig(builder); - if (attribute != null) attribute.getConfig(builder); if (summary != null) summary.getConfig(builder); if (initialize != null) initialize.getConfig(builder); if (feeding != null) feeding.getConfig(builder); diff --git a/config-model/src/main/javacc/SchemaParser.jj b/config-model/src/main/javacc/SchemaParser.jj index ae4c3b365d8..aef91e34239 100644 --- a/config-model/src/main/javacc/SchemaParser.jj +++ b/config-model/src/main/javacc/SchemaParser.jj @@ -201,6 +201,7 @@ TOKEN : | < FULL: "full" > | < STATIC: "static" > | < DYNAMIC: "dynamic" > +| < TOKENS: "tokens" > | < MATCHED_ELEMENTS_ONLY: "matched-elements-only" > | < SSCONTEXTUAL: "contextual" > | < SSOVERRIDE: "override" > @@ -1089,6 +1090,9 @@ void summaryInDocument(ParsedDocumentSummary docsum) : (<TYPE> type = dataType())? lbrace() { psf = new ParsedSummaryField(name, type); + if (type != null) { + psf.setHasExplicitType(); + } } (summaryItem(psf) (<NL>)*)* <RBRACE> { @@ -1128,6 +1132,7 @@ void summaryInFieldShort(ParsedField field) : <COLON> ( <DYNAMIC> { psf.setDynamic(); } | <MATCHED_ELEMENTS_ONLY> { psf.setMatchedElementsOnly(); } | (<FULL> | <STATIC>) { psf.setFull(); } + | <TOKENS> { psf.setTokens(); } ) } @@ -1138,13 +1143,17 @@ void summaryInFieldLong(ParsedField field) : { String name = field.name(); ParsedType type = field.getType(); + boolean explicitType = false; ParsedSummaryField psf; } { - ( [ name = identifier() [ <TYPE> type = dataType() ] ] + ( [ name = identifier() [ <TYPE> { type = dataType(); explicitType = true; } ] ] lbrace() { psf = field.summaryFieldFor(name, type); + if (explicitType) { + psf.setHasExplicitType(); + } } (summaryItem(psf) (<NL>)*)* <RBRACE> ) } @@ -1173,6 +1182,7 @@ void summaryTransform(ParsedSummaryField field) : { } ( <DYNAMIC> { field.setDynamic(); } | <MATCHED_ELEMENTS_ONLY> { field.setMatchedElementsOnly(); } | (<FULL> | <STATIC>) { field.setFull(); } + | <TOKENS> { field.setTokens(); } ) } @@ -2765,6 +2775,7 @@ String identifier() : { } | <TERTIARY> | <TEXT> | <TO> + | <TOKENS> | <TRUE> | <TYPE> | <UCA> diff --git a/config-model/src/main/resources/schema/content.rnc b/config-model/src/main/resources/schema/content.rnc index 5382e27e0b2..520f41609b2 100644 --- a/config-model/src/main/resources/schema/content.rnc +++ b/config-model/src/main/resources/schema/content.rnc @@ -150,10 +150,12 @@ Dispatch = element dispatch { DispatchGroup* } +# TODO: Deprecated, remove in Vespa 9 DispatchGroup = element group { DispatchNode+ } +# TODO: Deprecated, remove in Vespa 9 DispatchNode = element node { attribute distribution-key { xsd:nonNegativeInteger } } @@ -293,6 +295,8 @@ Group = element group { } Tuning = element tuning { + # TODO: Deprecated, remove in Vespa 9 + # Use the one under the content tag. element dispatch { element max-hits-per-partition { xsd:nonNegativeInteger }? }? & @@ -326,11 +330,13 @@ Tuning = element tuning { }? }? & element resizing { + # resizing is deprecated and will be gone on vespa 9 element initialdocumentcount { xsd:nonNegativeInteger }? & element amortize-count { xsd:nonNegativeInteger }? }? & element index { element io { + # io.read and io.write is deprecated and will be gone on vespa 9 element write { TuningIoOptionsLight }? & element read { TuningIoOptionsLight }? & element search { TuningIoOptionsSearch }? @@ -341,12 +347,14 @@ Tuning = element tuning { }? }? & element attribute { + # attribute element is deprecated and will be gone on vespa 9 element io { element write { TuningIoOptionsLight }? } }? & element summary { element io { + # summary.io.write is deprecated and will be gone on vespa 9 element write { TuningIoOptionsLight }? & element read { TuningIoOptionsFull }? }? & @@ -354,6 +362,7 @@ Tuning = element tuning { element cache { element maxsize { xsd:nonNegativeInteger }? & element maxsize-percent { xsd:double { minInclusive = "0.0" maxInclusive = "50.0" } }? & + # initialentries is deprecated and will be gone on vespa 9 element initialentries { xsd:nonNegativeInteger }? & TuningCompression? }? & diff --git a/config-model/src/test/derived/array_of_struct_attribute/test.sd b/config-model/src/test/derived/array_of_struct_attribute/test.sd index ce6e3db7310..3e46aea986a 100644 --- a/config-model/src/test/derived/array_of_struct_attribute/test.sd +++ b/config-model/src/test/derived/array_of_struct_attribute/test.sd @@ -17,6 +17,6 @@ schema test { } } document-summary rename { - summary new_elem_array type array<elem> { source: elem_array } + summary new_elem_array { source: elem_array } } } diff --git a/config-model/src/test/derived/bolding_dynamic_summary/test.sd b/config-model/src/test/derived/bolding_dynamic_summary/test.sd index bf7455df3c9..3d054c65839 100644 --- a/config-model/src/test/derived/bolding_dynamic_summary/test.sd +++ b/config-model/src/test/derived/bolding_dynamic_summary/test.sd @@ -31,19 +31,19 @@ schema test { } } document-summary dyn { - summary str_3_dyn type string { + summary str_3_dyn { source: str_3 dynamic } - summary arr_3_dyn type array<string> { + summary arr_3_dyn { source: arr_3 dynamic } - summary str_4_bold type string { + summary str_4_bold { source: str_4 bolding: on } - summary arr_4_bold type array<string> { + summary arr_4_bold { source: arr_4 bolding: on } diff --git a/config-model/src/test/derived/flickr/flickrphotos.sd b/config-model/src/test/derived/flickr/flickrphotos.sd index 514df5e76ed..df80ea3f96f 100755 --- a/config-model/src/test/derived/flickr/flickrphotos.sd +++ b/config-model/src/test/derived/flickr/flickrphotos.sd @@ -4,7 +4,7 @@ schema flickrphotos{ #Document summary to use for attribute-prefetching with many hits document-summary mapcluster { - summary distance type int {} + summary distance {} } document flickrphotos{ diff --git a/config-model/src/test/derived/globalphase_token_functions/rank-profiles.cfg b/config-model/src/test/derived/globalphase_token_functions/rank-profiles.cfg index 37d84c1a2d9..d0336e31744 100644 --- a/config-model/src/test/derived/globalphase_token_functions/rank-profiles.cfg +++ b/config-model/src/test/derived/globalphase_token_functions/rank-profiles.cfg @@ -49,3 +49,46 @@ rankprofile[].fef.property[].name "vespa.type.attribute.tokens" rankprofile[].fef.property[].value "tensor(d0[128])" rankprofile[].fef.property[].name "vespa.type.query.input" rankprofile[].fef.property[].value "tensor(d0[32])" +rankprofile[].name "with-fun" +rankprofile[].fef.property[].name "rankingExpression(use_model).rankingScript" +rankprofile[].fef.property[].value "attribute(outputidx) + 1.0" +rankprofile[].fef.property[].name "vespa.rank.globalphase" +rankprofile[].fef.property[].value "rankingExpression(use_model)" +rankprofile[].fef.property[].name "vespa.match.feature" +rankprofile[].fef.property[].value "attribute(outputidx)" +rankprofile[].fef.property[].name "vespa.hidden.matchfeature" +rankprofile[].fef.property[].value "attribute(outputidx)" +rankprofile[].fef.property[].name "vespa.type.attribute.tokens" +rankprofile[].fef.property[].value "tensor(d0[128])" +rankprofile[].name "with-fun-mf" +rankprofile[].fef.property[].name "rankingExpression(use_model).rankingScript" +rankprofile[].fef.property[].value "attribute(outputidx) + 1.0" +rankprofile[].fef.property[].name "vespa.rank.firstphase" +rankprofile[].fef.property[].value "nativeRank" +rankprofile[].fef.property[].name "vespa.rank.globalphase" +rankprofile[].fef.property[].value "rankingExpression(use_model)" +rankprofile[].fef.property[].name "vespa.match.feature" +rankprofile[].fef.property[].value "rankingExpression(use_model)" +rankprofile[].fef.property[].name "vespa.feature.rename" +rankprofile[].fef.property[].value "rankingExpression(use_model)" +rankprofile[].fef.property[].name "vespa.feature.rename" +rankprofile[].fef.property[].value "use_model" +rankprofile[].fef.property[].name "vespa.type.attribute.tokens" +rankprofile[].fef.property[].value "tensor(d0[128])" +rankprofile[].name "fun-mf-child" +rankprofile[].fef.property[].name "rankingExpression(use_model).rankingScript" +rankprofile[].fef.property[].value "attribute(outputidx) + 1.0" +rankprofile[].fef.property[].name "vespa.rank.firstphase" +rankprofile[].fef.property[].value "rankingExpression(firstphase)" +rankprofile[].fef.property[].name "rankingExpression(firstphase).rankingScript" +rankprofile[].fef.property[].value "42 * attribute(outputidx)" +rankprofile[].fef.property[].name "vespa.rank.globalphase" +rankprofile[].fef.property[].value "rankingExpression(use_model)" +rankprofile[].fef.property[].name "vespa.match.feature" +rankprofile[].fef.property[].value "rankingExpression(use_model)" +rankprofile[].fef.property[].name "vespa.feature.rename" +rankprofile[].fef.property[].value "rankingExpression(use_model)" +rankprofile[].fef.property[].name "vespa.feature.rename" +rankprofile[].fef.property[].value "use_model" +rankprofile[].fef.property[].name "vespa.type.attribute.tokens" +rankprofile[].fef.property[].value "tensor(d0[128])" diff --git a/config-model/src/test/derived/globalphase_token_functions/test.sd b/config-model/src/test/derived/globalphase_token_functions/test.sd index 5e849772249..511e09948b4 100644 --- a/config-model/src/test/derived/globalphase_token_functions/test.sd +++ b/config-model/src/test/derived/globalphase_token_functions/test.sd @@ -42,4 +42,23 @@ schema test { } } + rank-profile with-fun { + function use_model() { + expression: attribute(outputidx) + 1.0 + } + global-phase { + expression: use_model + } + } + rank-profile with-fun-mf inherits with-fun { + first-phase { + expression: nativeRank + } + match-features: use_model + } + rank-profile fun-mf-child inherits with-fun-mf { + first-phase { + expression: 42 * attribute(outputidx) + } + } } diff --git a/config-model/src/test/derived/imported_position_field_summary/child.sd b/config-model/src/test/derived/imported_position_field_summary/child.sd index 34aa6a19920..0efa2a8905c 100644 --- a/config-model/src/test/derived/imported_position_field_summary/child.sd +++ b/config-model/src/test/derived/imported_position_field_summary/child.sd @@ -8,6 +8,6 @@ schema child { import field parent_ref.pos as my_pos {} document-summary mysummary { - summary my_pos type position { } + summary my_pos { } } } diff --git a/config-model/src/test/derived/imported_struct_fields/child.sd b/config-model/src/test/derived/imported_struct_fields/child.sd index 63a4efee6f2..09eb08d3534 100644 --- a/config-model/src/test/derived/imported_struct_fields/child.sd +++ b/config-model/src/test/derived/imported_struct_fields/child.sd @@ -10,22 +10,22 @@ schema child { import field parent_ref.str_int_map as my_str_int_map {} document-summary mysummary { - summary documentid type string {} - summary my_elem_array type array<elem> {} - summary my_elem_map type map<string, elem> {} - summary my_str_int_map type map<string, int> {} + summary documentid {} + summary my_elem_array {} + summary my_elem_map {} + summary my_str_int_map {} } document-summary filtered { - summary elem_array_filtered type array<elem> { + summary elem_array_filtered { source: my_elem_array matched-elements-only } - summary elem_map_filtered type map<string, elem> { + summary elem_map_filtered { source: my_elem_map matched-elements-only } - summary str_int_map_filtered type map<string, int> { + summary str_int_map_filtered { source: my_str_int_map matched-elements-only } diff --git a/config-model/src/test/derived/importedfields/child.sd b/config-model/src/test/derived/importedfields/child.sd index 540c3b87751..d3be011f5fb 100644 --- a/config-model/src/test/derived/importedfields/child.sd +++ b/config-model/src/test/derived/importedfields/child.sd @@ -22,14 +22,14 @@ schema child { } document-summary mysummary { - summary a_ref type reference<parent_a> {} - summary b_ref_with_summary type reference<parent_b> {} - summary my_int_field type int {} - summary my_string_field type string {} - summary my_int_array_field type array<int> {} - summary my_int_wset_field type weightedset<int> {} - summary my_ancient_int_field type int {} - summary my_filtered_int_array_field type array<int> { + summary a_ref {} + summary b_ref_with_summary {} + summary my_int_field {} + summary my_string_field {} + summary my_int_array_field {} + summary my_int_wset_field {} + summary my_ancient_int_field {} + summary my_filtered_int_array_field { source: my_int_array_field matched-elements-only } diff --git a/config-model/src/test/derived/map_of_struct_attribute/test.sd b/config-model/src/test/derived/map_of_struct_attribute/test.sd index 7001b95d09f..617f761c7e7 100644 --- a/config-model/src/test/derived/map_of_struct_attribute/test.sd +++ b/config-model/src/test/derived/map_of_struct_attribute/test.sd @@ -30,7 +30,7 @@ schema test { } } document-summary rename { - summary new_str_elem_map type map<string,elem> { source: str_elem_map } - summary new_int_elem_map type map<int,elem> { source: int_elem_map } + summary new_str_elem_map { source: str_elem_map } + summary new_int_elem_map { source: int_elem_map } } } diff --git a/config-model/src/test/derived/multiplesummaries/multiplesummaries.sd b/config-model/src/test/derived/multiplesummaries/multiplesummaries.sd index b19b04c8222..28eaec22784 100644 --- a/config-model/src/test/derived/multiplesummaries/multiplesummaries.sd +++ b/config-model/src/test/derived/multiplesummaries/multiplesummaries.sd @@ -80,35 +80,35 @@ schema multiplesummaries { document-summary third { - summary a type string { + summary a { } - summary adynamic type string { + summary adynamic { } - summary d type string { + summary d { } - summary e type string { + summary e { } summary f { } - summary g type array<int> { + summary g { } - summary h type weightedset<string> { + summary h { } } document-summary attributesonly1 { - summary a type string { + summary a { } - summary c type string { + summary c { } } @@ -116,10 +116,10 @@ schema multiplesummaries { # Since a here is a dynamic summary field, it will be fetched from disk document-summary notattributesonly1 { - summary adynamic type string { # Should still be dynamic here + summary adynamic { # Should still be dynamic here } - summary c type string { + summary c { } } @@ -127,25 +127,25 @@ schema multiplesummaries { # Since a here is a dynamic summary, it will be fetched from disk document-summary anothernotattributesonly2 { - summary adynamic2 type string { # Should still be dynamic here + summary adynamic2 { # Should still be dynamic here source: a dynamic } - summary c type string { + summary c { } - summary alltags type array<string> { + summary alltags { source: mytags } - summary sometags type array<string> { + summary sometags { source: mytags matched-elements-only } - summary anothera type string { + summary anothera { source: a } - summary anotherb type string { + summary anotherb { source: b } } @@ -153,24 +153,24 @@ schema multiplesummaries { # Not attributes only because d is bolded document-summary notattributesonly3 { - summary a type string { + summary a { } - summary d type string { + summary d { } } document-summary attributesonly2 { - summary anotdynamic type string { # Should not be dynamic here + summary anotdynamic { # Should not be dynamic here source: adynamic } - summary c type string { + summary c { } - summary loc_position type long { + summary loc_position { source: loc_pos_zcurve } @@ -178,33 +178,33 @@ schema multiplesummaries { document-summary attributesonly3 { - summary a type string { + summary a { } - summary anotbolded type string { + summary anotbolded { source: a } - summary loc_pos_zcurve type long { + summary loc_pos_zcurve { } } document-summary notattributesonly4 { - summary abolded2 type string { + summary abolded2 { source: a bolding: on } - summary c type string { + summary c { } } document-summary notattributesonly5 { - summary aboldeddynamic type string { + summary aboldeddynamic { source: a dynamic bolding: on diff --git a/config-model/src/test/derived/nearestneighbor/test.sd b/config-model/src/test/derived/nearestneighbor/test.sd index 7d08a5279bc..5b891049480 100644 --- a/config-model/src/test/derived/nearestneighbor/test.sd +++ b/config-model/src/test/derived/nearestneighbor/test.sd @@ -23,6 +23,6 @@ schema test { } } document-summary minimal { - summary id type int {} + summary id {} } } diff --git a/config-model/src/test/derived/ngram/chunk.sd b/config-model/src/test/derived/ngram/chunk.sd index ab309f57548..84d806ef074 100644 --- a/config-model/src/test/derived/ngram/chunk.sd +++ b/config-model/src/test/derived/ngram/chunk.sd @@ -12,7 +12,7 @@ schema chunk { } document-summary content-summary inherits default { - summary content_dynamic type string { + summary content_dynamic { source: content dynamic } diff --git a/config-model/src/test/derived/rankingexpression/rank-profiles.cfg b/config-model/src/test/derived/rankingexpression/rank-profiles.cfg index b0f7d0f2477..b3257c962dd 100644 --- a/config-model/src/test/derived/rankingexpression/rank-profiles.cfg +++ b/config-model/src/test/derived/rankingexpression/rank-profiles.cfg @@ -520,3 +520,65 @@ rankprofile[].fef.property[].name "vespa.type.attribute.t1" rankprofile[].fef.property[].value "tensor(m{},v[3])" rankprofile[].fef.property[].name "vespa.type.query.v" rankprofile[].fef.property[].value "tensor(v[3])" +rankprofile[].name "withnorm" +rankprofile[].fef.property[].name "rankingExpression(normBar).rankingScript" +rankprofile[].fef.property[].value "attribute(foo1) + attribute(year)" +rankprofile[].fef.property[].name "vespa.rank.firstphase" +rankprofile[].fef.property[].value "attribute(foo1)" +rankprofile[].fef.property[].name "vespa.rank.globalphase" +rankprofile[].fef.property[].value "rankingExpression(globalphase)" +rankprofile[].fef.property[].name "rankingExpression(globalphase).rankingScript" +rankprofile[].fef.property[].value "normalize@3551296680@linear + normalize@2879443254@rrank" +rankprofile[].fef.property[].name "vespa.match.feature" +rankprofile[].fef.property[].value "nativeRank" +rankprofile[].fef.property[].name "vespa.match.feature" +rankprofile[].fef.property[].value "attribute(year)" +rankprofile[].fef.property[].name "vespa.match.feature" +rankprofile[].fef.property[].value "attribute(foo1)" +rankprofile[].fef.property[].name "vespa.hidden.matchfeature" +rankprofile[].fef.property[].value "attribute(year)" +rankprofile[].fef.property[].name "vespa.hidden.matchfeature" +rankprofile[].fef.property[].value "attribute(foo1)" +rankprofile[].fef.property[].name "vespa.globalphase.rerankcount" +rankprofile[].fef.property[].value "123" +rankprofile[].fef.property[].name "vespa.type.attribute.t1" +rankprofile[].fef.property[].value "tensor(m{},v[3])" +rankprofile[].normalizer[].name "normalize@3551296680@linear" +rankprofile[].normalizer[].input "nativeRank" +rankprofile[].normalizer[].algo LINEAR +rankprofile[].normalizer[].kparam 0.0 +rankprofile[].normalizer[].name "normalize@2879443254@rrank" +rankprofile[].normalizer[].input "normBar" +rankprofile[].normalizer[].algo RRANK +rankprofile[].normalizer[].kparam 42.0 +rankprofile[].name "withfusion" +rankprofile[].fef.property[].name "rankingExpression(normBar).rankingScript" +rankprofile[].fef.property[].value "attribute(foo1) + attribute(year)" +rankprofile[].fef.property[].name "vespa.rank.firstphase" +rankprofile[].fef.property[].value "attribute(foo1)" +rankprofile[].fef.property[].name "vespa.rank.globalphase" +rankprofile[].fef.property[].value "rankingExpression(globalphase)" +rankprofile[].fef.property[].name "rankingExpression(globalphase).rankingScript" +rankprofile[].fef.property[].value "normalize@5385018767@rrank + normalize@3221316369@rrank" +rankprofile[].fef.property[].name "vespa.match.feature" +rankprofile[].fef.property[].value "nativeRank" +rankprofile[].fef.property[].name "vespa.match.feature" +rankprofile[].fef.property[].value "attribute(year)" +rankprofile[].fef.property[].name "vespa.match.feature" +rankprofile[].fef.property[].value "attribute(foo1)" +rankprofile[].fef.property[].name "vespa.hidden.matchfeature" +rankprofile[].fef.property[].value "attribute(year)" +rankprofile[].fef.property[].name "vespa.hidden.matchfeature" +rankprofile[].fef.property[].value "attribute(foo1)" +rankprofile[].fef.property[].name "vespa.globalphase.rerankcount" +rankprofile[].fef.property[].value "456" +rankprofile[].fef.property[].name "vespa.type.attribute.t1" +rankprofile[].fef.property[].value "tensor(m{},v[3])" +rankprofile[].normalizer[].name "normalize@5385018767@rrank" +rankprofile[].normalizer[].input "normBar" +rankprofile[].normalizer[].algo RRANK +rankprofile[].normalizer[].kparam 60.0 +rankprofile[].normalizer[].name "normalize@3221316369@rrank" +rankprofile[].normalizer[].input "nativeRank" +rankprofile[].normalizer[].algo RRANK +rankprofile[].normalizer[].kparam 60.0 diff --git a/config-model/src/test/derived/rankingexpression/rankexpression.sd b/config-model/src/test/derived/rankingexpression/rankexpression.sd index 16dff61b63a..15537f1f9d0 100644 --- a/config-model/src/test/derived/rankingexpression/rankexpression.sd +++ b/config-model/src/test/derived/rankingexpression/rankexpression.sd @@ -441,4 +441,32 @@ schema rankexpression { } } + rank-profile withnorm { + first-phase { + expression: attribute(foo1) + } + function normBar() { + expression: attribute(foo1) + attribute(year) + } + global-phase { + expression: normalize_linear(nativeRank) + reciprocal_rank(normBar(), 42.0) + rerank-count: 123 + } + match-features: nativeRank + } + + rank-profile withfusion { + first-phase { + expression: attribute(foo1) + } + function normBar() { + expression: attribute(foo1) + attribute(year) + } + global-phase { + expression: reciprocal_rank_fusion(normBar, nativeRank) + rerank-count: 456 + } + match-features: nativeRank + } + } diff --git a/config-model/src/test/derived/reference_fields/ad.sd b/config-model/src/test/derived/reference_fields/ad.sd index 390f8f6a154..097a6ed5bc9 100644 --- a/config-model/src/test/derived/reference_fields/ad.sd +++ b/config-model/src/test/derived/reference_fields/ad.sd @@ -12,6 +12,6 @@ schema ad { } } document-summary explicit_summary { - summary yet_another_ref type reference<campaign> {} + summary yet_another_ref {} } } diff --git a/config-model/src/test/derived/reference_from_several/bar.sd b/config-model/src/test/derived/reference_from_several/bar.sd index 12cf8e63378..47cd9b117aa 100644 --- a/config-model/src/test/derived/reference_from_several/bar.sd +++ b/config-model/src/test/derived/reference_from_several/bar.sd @@ -10,7 +10,7 @@ schema bar { } import field bpref.x as barsximp {} document-summary other { - summary bartitle type string {} - summary barsximp type int {} + summary bartitle {} + summary barsximp {} } } diff --git a/config-model/src/test/derived/reference_from_several/foo.sd b/config-model/src/test/derived/reference_from_several/foo.sd index 5ef42ee6f0d..ead26bcbba3 100644 --- a/config-model/src/test/derived/reference_from_several/foo.sd +++ b/config-model/src/test/derived/reference_from_several/foo.sd @@ -10,7 +10,7 @@ schema foo { } import field myref.x as myx {} document-summary small { - summary myx type int {} - summary foo type string {} + summary myx {} + summary foo {} } } diff --git a/config-model/src/test/derived/schemainheritance/child.sd b/config-model/src/test/derived/schemainheritance/child.sd index 77daf2ba34f..ea3ff9b85da 100644 --- a/config-model/src/test/derived/schemainheritance/child.sd +++ b/config-model/src/test/derived/schemainheritance/child.sd @@ -36,7 +36,7 @@ schema child inherits parent { } document-summary child_summary inherits parent_summary { - summary cf1 type string {} + summary cf1 {} } import field importedschema_ref.importedfield2 as child_imported {} diff --git a/config-model/src/test/derived/schemainheritance/parent.sd b/config-model/src/test/derived/schemainheritance/parent.sd index 03392b428ed..ab2b703ba57 100644 --- a/config-model/src/test/derived/schemainheritance/parent.sd +++ b/config-model/src/test/derived/schemainheritance/parent.sd @@ -32,7 +32,7 @@ schema parent { file: small_constants_and_functions.onnx } document-summary parent_summary { - summary pf1 type string { + summary pf1 { } } import field importedschema_ref.importedfield1 as parent_imported { diff --git a/config-model/src/test/derived/schemainheritance/schema-info.cfg b/config-model/src/test/derived/schemainheritance/schema-info.cfg new file mode 100644 index 00000000000..9fe71780c7a --- /dev/null +++ b/config-model/src/test/derived/schemainheritance/schema-info.cfg @@ -0,0 +1,127 @@ +schema[].name "child" +schema[].field[].name "parent_field" +schema[].field[].type "string" +schema[].field[].attribute true +schema[].field[].index true +schema[].field[].name "child_field" +schema[].field[].type "string" +schema[].field[].attribute true +schema[].field[].index true +schema[].field[].name "pf1" +schema[].field[].type "string" +schema[].field[].attribute false +schema[].field[].index false +schema[].field[].name "importedschema_ref" +schema[].field[].type "reference<importedschema>" +schema[].field[].attribute true +schema[].field[].index false +schema[].field[].name "parent_field" +schema[].field[].type "string" +schema[].field[].attribute true +schema[].field[].index true +schema[].field[].name "cf1" +schema[].field[].type "string" +schema[].field[].attribute false +schema[].field[].index false +schema[].field[].name "child_field" +schema[].field[].type "string" +schema[].field[].attribute true +schema[].field[].index true +schema[].field[].name "parent_imported" +schema[].field[].type "string" +schema[].field[].attribute true +schema[].field[].index false +schema[].field[].name "importedfield1" +schema[].field[].type "string" +schema[].field[].attribute true +schema[].field[].index false +schema[].field[].name "child_imported" +schema[].field[].type "string" +schema[].field[].attribute true +schema[].field[].index false +schema[].field[].name "importedfield2" +schema[].field[].type "string" +schema[].field[].attribute true +schema[].field[].index false +schema[].fieldset[].name "[document]" +schema[].fieldset[].field[] "cf1" +schema[].fieldset[].field[] "importedschema_ref" +schema[].fieldset[].field[] "pf1" +schema[].fieldset[].name "[search]" +schema[].fieldset[].field[] "child_field" +schema[].fieldset[].field[] "parent_field" +schema[].fieldset[].name "parent_set" +schema[].fieldset[].field[] "pf1" +schema[].fieldset[].name "child_set" +schema[].fieldset[].field[] "cf1" +schema[].fieldset[].field[] "pf1" +schema[].summaryclass[].name "default" +schema[].summaryclass[].fields[].name "parent_field" +schema[].summaryclass[].fields[].type "longstring" +schema[].summaryclass[].fields[].dynamic false +schema[].summaryclass[].fields[].name "child_field" +schema[].summaryclass[].fields[].type "longstring" +schema[].summaryclass[].fields[].dynamic false +schema[].summaryclass[].fields[].name "pf1" +schema[].summaryclass[].fields[].type "longstring" +schema[].summaryclass[].fields[].dynamic false +schema[].summaryclass[].fields[].name "cf1" +schema[].summaryclass[].fields[].type "longstring" +schema[].summaryclass[].fields[].dynamic false +schema[].summaryclass[].fields[].name "rankfeatures" +schema[].summaryclass[].fields[].type "featuredata" +schema[].summaryclass[].fields[].dynamic false +schema[].summaryclass[].fields[].name "summaryfeatures" +schema[].summaryclass[].fields[].type "featuredata" +schema[].summaryclass[].fields[].dynamic false +schema[].summaryclass[].fields[].name "documentid" +schema[].summaryclass[].fields[].type "longstring" +schema[].summaryclass[].fields[].dynamic false +schema[].summaryclass[].name "parent_summary" +schema[].summaryclass[].fields[].name "pf1" +schema[].summaryclass[].fields[].type "longstring" +schema[].summaryclass[].fields[].dynamic false +schema[].summaryclass[].fields[].name "rankfeatures" +schema[].summaryclass[].fields[].type "featuredata" +schema[].summaryclass[].fields[].dynamic false +schema[].summaryclass[].fields[].name "summaryfeatures" +schema[].summaryclass[].fields[].type "featuredata" +schema[].summaryclass[].fields[].dynamic false +schema[].summaryclass[].name "attributeprefetch" +schema[].summaryclass[].fields[].name "parent_field" +schema[].summaryclass[].fields[].type "longstring" +schema[].summaryclass[].fields[].dynamic false +schema[].summaryclass[].fields[].name "child_field" +schema[].summaryclass[].fields[].type "longstring" +schema[].summaryclass[].fields[].dynamic false +schema[].summaryclass[].fields[].name "rankfeatures" +schema[].summaryclass[].fields[].type "featuredata" +schema[].summaryclass[].fields[].dynamic false +schema[].summaryclass[].fields[].name "summaryfeatures" +schema[].summaryclass[].fields[].type "featuredata" +schema[].summaryclass[].fields[].dynamic false +schema[].summaryclass[].name "child_summary" +schema[].summaryclass[].fields[].name "pf1" +schema[].summaryclass[].fields[].type "longstring" +schema[].summaryclass[].fields[].dynamic false +schema[].summaryclass[].fields[].name "rankfeatures" +schema[].summaryclass[].fields[].type "featuredata" +schema[].summaryclass[].fields[].dynamic false +schema[].summaryclass[].fields[].name "summaryfeatures" +schema[].summaryclass[].fields[].type "featuredata" +schema[].summaryclass[].fields[].dynamic false +schema[].summaryclass[].fields[].name "cf1" +schema[].summaryclass[].fields[].type "longstring" +schema[].summaryclass[].fields[].dynamic false +schema[].rankprofile[].name "default" +schema[].rankprofile[].hasSummaryFeatures false +schema[].rankprofile[].hasRankFeatures false +schema[].rankprofile[].name "unranked" +schema[].rankprofile[].hasSummaryFeatures false +schema[].rankprofile[].hasRankFeatures false +schema[].rankprofile[].name "child_profile" +schema[].rankprofile[].hasSummaryFeatures false +schema[].rankprofile[].hasRankFeatures false +schema[].rankprofile[].name "parent_profile" +schema[].rankprofile[].hasSummaryFeatures false +schema[].rankprofile[].hasRankFeatures false diff --git a/config-model/src/test/derived/streamingstruct/streamingstruct.sd b/config-model/src/test/derived/streamingstruct/streamingstruct.sd index 70c0a55b833..8606ec0a7f7 100644 --- a/config-model/src/test/derived/streamingstruct/streamingstruct.sd +++ b/config-model/src/test/derived/streamingstruct/streamingstruct.sd @@ -174,11 +174,11 @@ schema streamingstruct { } document-summary summ { - summary snippet type string { + summary snippet { dynamic source: a.f1, b.f2 } - summary snippet2 type string { + summary snippet2 { source: a.f1, b.f1, b.f2 } } diff --git a/config-model/src/test/derived/streamingstructdefault/streamingstructdefault.sd b/config-model/src/test/derived/streamingstructdefault/streamingstructdefault.sd index 7727875b17d..837678ed174 100644 --- a/config-model/src/test/derived/streamingstructdefault/streamingstructdefault.sd +++ b/config-model/src/test/derived/streamingstructdefault/streamingstructdefault.sd @@ -14,7 +14,7 @@ schema streamingstructdefault { } } document-summary default { - summary sum1 type string { + summary sum1 { source: f1, f2.s1 dynamic } diff --git a/config-model/src/test/derived/twostreamingstructs/streamingstruct.sd b/config-model/src/test/derived/twostreamingstructs/streamingstruct.sd index d0084703bca..8e70fdfc8d1 100644 --- a/config-model/src/test/derived/twostreamingstructs/streamingstruct.sd +++ b/config-model/src/test/derived/twostreamingstructs/streamingstruct.sd @@ -176,11 +176,11 @@ schema streamingstruct { } document-summary summ { - summary snippet type string { + summary snippet { dynamic source: a.f1, b.f2 } - summary snippet2 type string { + summary snippet2 { source: a.f1, b.f1, b.f2 } } diff --git a/config-model/src/test/examples/documentidinsummary.sd b/config-model/src/test/examples/documentidinsummary.sd index b85506d4eef..23bca092cad 100644 --- a/config-model/src/test/examples/documentidinsummary.sd +++ b/config-model/src/test/examples/documentidinsummary.sd @@ -4,6 +4,6 @@ search documentidinsummary { field a type string { } } document-summary withid { - summary w type string { source: documentid } + summary w { source: documentid } } } diff --git a/config-model/src/test/examples/invalidimplicitsummarysource.sd b/config-model/src/test/examples/invalidimplicitsummarysource.sd index 2a7310d848a..309d5f6e9a6 100644 --- a/config-model/src/test/examples/invalidimplicitsummarysource.sd +++ b/config-model/src/test/examples/invalidimplicitsummarysource.sd @@ -6,6 +6,6 @@ search invalidsummarysource { } } document-summary baz { - summary cox type string { } + summary cox { } } } diff --git a/config-model/src/test/examples/invalidselfreferringsummary.sd b/config-model/src/test/examples/invalidselfreferringsummary.sd index 3ee4d2c457e..6a2f278ad1d 100644 --- a/config-model/src/test/examples/invalidselfreferringsummary.sd +++ b/config-model/src/test/examples/invalidselfreferringsummary.sd @@ -4,6 +4,6 @@ search invalidselfreferringsummary { field a type string { } } document-summary withid { - summary w type string { source: w } + summary w { source: w } } } diff --git a/config-model/src/test/examples/invalidsummarysource.sd b/config-model/src/test/examples/invalidsummarysource.sd index 86d6918ac7b..abb8ac0c4ae 100644 --- a/config-model/src/test/examples/invalidsummarysource.sd +++ b/config-model/src/test/examples/invalidsummarysource.sd @@ -6,6 +6,6 @@ search invalidsummarysource { } } document-summary baz { - summary cox type string { source: nonexistingfield } + summary cox { source: nonexistingfield } } } diff --git a/config-model/src/test/examples/multiplesummaries.sd b/config-model/src/test/examples/multiplesummaries.sd index 7e298b4e7a3..a7e3a78fe6d 100644 --- a/config-model/src/test/examples/multiplesummaries.sd +++ b/config-model/src/test/examples/multiplesummaries.sd @@ -19,13 +19,13 @@ search multiplesummaries { document-summary other { - summary field1 type weightedset<string> { + summary field1 { } - summary field2 type tag { + summary field2 { } - summary field3 type array<int> { + summary field3 { } } diff --git a/config-model/src/test/examples/nextgen/implicitstructtypes.sd b/config-model/src/test/examples/nextgen/implicitstructtypes.sd index 82fd1bb0121..182805a0ae7 100644 --- a/config-model/src/test/examples/nextgen/implicitstructtypes.sd +++ b/config-model/src/test/examples/nextgen/implicitstructtypes.sd @@ -10,8 +10,8 @@ search implicitstructtypes { } } document-summary docsum { - summary docsum_str type string { - source: doc_str_sum + summary docsum_str { + source: doc_str } } } diff --git a/config-model/src/test/examples/nextgen/simple.sd b/config-model/src/test/examples/nextgen/simple.sd index 6d371328453..897371f2991 100644 --- a/config-model/src/test/examples/nextgen/simple.sd +++ b/config-model/src/test/examples/nextgen/simple.sd @@ -7,8 +7,8 @@ search simple { } } document-summary explicit_summary { - summary summary_field type string { - source: doc_field_summary + summary summary_field { + source: doc_field } } field extern_field type string { diff --git a/config-model/src/test/examples/nextgen/summaryfield.sd b/config-model/src/test/examples/nextgen/summaryfield.sd index 06cc980ea73..906d5441e55 100644 --- a/config-model/src/test/examples/nextgen/summaryfield.sd +++ b/config-model/src/test/examples/nextgen/summaryfield.sd @@ -10,13 +10,13 @@ search summaryfield { } } document-summary baz { - summary cox type string { - source: bar + summary cox { + source: foo } - summary alltags type array<string> { + summary alltags { source: mytags } - summary sometags type array<string> { + summary sometags { source: mytags matched-elements-only } diff --git a/config-model/src/test/examples/outsidesummary.sd b/config-model/src/test/examples/outsidesummary.sd index 5fadc1948f0..a95cbb0c628 100644 --- a/config-model/src/test/examples/outsidesummary.sd +++ b/config-model/src/test/examples/outsidesummary.sd @@ -3,17 +3,17 @@ search outsidesummary { document-summary other { - summary sa type string { + summary sa { dynamic source: a } - summary sa2 type string { + summary sa2 { full source: a } - summary a type string { + summary a { } } diff --git a/config-model/src/test/examples/summaryfieldcollision.sd b/config-model/src/test/examples/summaryfieldcollision.sd index 6a8cb2eeb31..2235abce422 100644 --- a/config-model/src/test/examples/summaryfieldcollision.sd +++ b/config-model/src/test/examples/summaryfieldcollision.sd @@ -13,13 +13,13 @@ search summaryfieldcollision { } document-summary sum1 { - summary f type string { + summary f { source: title } } document-summary sum2 { - summary f type string { + summary f { source: description } } diff --git a/config-model/src/test/java/com/yahoo/schema/NoNormalizersTestCase.java b/config-model/src/test/java/com/yahoo/schema/NoNormalizersTestCase.java new file mode 100644 index 00000000000..6e2efadfa2c --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/NoNormalizersTestCase.java @@ -0,0 +1,172 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema; + +import com.yahoo.search.query.profile.QueryProfileRegistry; +import com.yahoo.schema.parser.ParseException; +import com.yahoo.yolean.Exceptions; +import ai.vespa.rankingexpression.importer.configmodelview.ImportedMlModels; + +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.*; + +/** + * Tests rank profiles with normalizers in bad places + * + * @author arnej + */ +public class NoNormalizersTestCase extends AbstractSchemaTestCase { + + void compileSchema(String schema) throws ParseException { + RankProfileRegistry registry = new RankProfileRegistry(); + var qp = new QueryProfileRegistry(); + ApplicationBuilder builder = new ApplicationBuilder(registry, qp); + builder.addSchema(schema); + builder.build(true); + for (RankProfile rp : registry.all()) { + rp.compile(qp, new ImportedMlModels()); + } + } + + @Test + void requireThatNormalizerInFirstPhaseIsChecked() throws ParseException { + try { + compileSchema(""" + search test { + document test { } + rank-profile p1 { + first-phase { + expression: normalize_linear(nativeRank) + } + } + } + """); + fail(); + } catch (IllegalArgumentException e) { + assertEquals("Rank profile 'p1' is invalid: " + + "Cannot use normalize_linear(nativeRank) from first-phase expression, only valid in global-phase expression", + Exceptions.toMessageString(e)); + } + } + + @Test + void requireThatNormalizerInSecondPhaseIsChecked() throws ParseException { + try { + compileSchema(""" + search test { + document test { + field title type string { + indexing: index + } + } + rank-profile p2 { + function foobar() { + expression: 42 + reciprocal_rank(whatever, 1.0) + } + function whatever() { + expression: fieldMatch(title) + } + first-phase { + expression: nativeRank + } + second-phase { + expression: foobar + } + } + } + """); + fail(); + } catch (IllegalArgumentException e) { + assertEquals("Rank profile 'p2' is invalid: " + + "Cannot use reciprocal_rank(whatever,1.0) from second-phase expression, only valid in global-phase expression", + Exceptions.toMessageString(e)); + } + } + + @Test + void requireThatNormalizerInMatchFeatureIsChecked() throws ParseException { + try { + compileSchema(""" + search test { + document test { } + rank-profile p3 { + function foobar() { + expression: normalize_linear(nativeRank) + } + first-phase { + expression: nativeRank + } + match-features { + nativeRank + foobar + } + } + } + """); + fail(); + } catch (IllegalArgumentException e) { + assertEquals("Rank profile 'p3' is invalid: " + + "Cannot use normalize_linear(nativeRank) from match-feature foobar, only valid in global-phase expression", + Exceptions.toMessageString(e)); + } + } + + @Test + void requireThatNormalizerInSummaryFeatureIsChecked() throws ParseException { + try { + compileSchema(""" + search test { + document test { } + rank-profile p4 { + function foobar() { + expression: normalize_linear(nativeRank) + } + first-phase { + expression: nativeRank + } + summary-features { + nativeRank + foobar + } + } + } + """); + fail(); + } catch (IllegalArgumentException e) { + assertEquals("Rank profile 'p4' is invalid: " + + "Cannot use normalize_linear(nativeRank) from summary-feature foobar, only valid in global-phase expression", + Exceptions.toMessageString(e)); + } + } + + @Test + void requireThatNormalizerInNormalizerIsChecked() throws ParseException { + try { + compileSchema(""" + search test { + document test { + field title type string { + indexing: index + } + } + rank-profile p5 { + function foobar() { + expression: reciprocal_rank(nativeRank) + } + first-phase { + expression: nativeRank + } + global-phase { + expression: normalize_linear(fieldMatch(title)) + normalize_linear(foobar) + } + } + } + """); + fail(); + } catch (IllegalArgumentException e) { + assertEquals("Rank profile 'p5' is invalid: " + + "Cannot use reciprocal_rank(nativeRank) from normalizer input foobar, only valid in global-phase expression", + Exceptions.toMessageString(e)); + } + } +} diff --git a/config-model/src/test/java/com/yahoo/schema/SchemaTestCase.java b/config-model/src/test/java/com/yahoo/schema/SchemaTestCase.java index 798252c9a34..c959634019d 100644 --- a/config-model/src/test/java/com/yahoo/schema/SchemaTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/SchemaTestCase.java @@ -143,10 +143,10 @@ public class SchemaTestCase { " file: models/my_model.onnx" + " }" + " document-summary parent_summary1 {" + - " summary pf1 type string {}" + + " summary pf1 {}" + " }" + " document-summary parent_summary2 {" + - " summary pf2 type string {}" + + " summary pf2 {}" + " }" + " import field parentschema_ref.name as parent_imported {}" + " raw-as-base64-in-summary" + @@ -177,7 +177,7 @@ public class SchemaTestCase { " file: models/my_model.onnx" + " }" + " document-summary child1_summary inherits parent_summary1 {" + - " summary c1f1 type string {}" + + " summary c1f1 {}" + " }" + " import field parentschema_ref.name as child1_imported {}" + "}"); @@ -208,7 +208,7 @@ public class SchemaTestCase { " file: models/my_model.onnx" + " }" + " document-summary child2_summary inherits parent_summary1, parent_summary2 {" + - " summary c2f1 type string {}" + + " summary c2f1 {}" + " }" + " import field parentschema_ref.name as child2_imported {}" + "}"); @@ -340,7 +340,7 @@ public class SchemaTestCase { " file: models/my_model.onnx" + " }" + " document-summary parent_summary {" + - " summary pf1 type string {}" + + " summary pf1 {}" + " }" + " import field parentschema_ref.name as parent_imported {}" + " raw-as-base64-in-summary" + diff --git a/config-model/src/test/java/com/yahoo/schema/SummaryTestCase.java b/config-model/src/test/java/com/yahoo/schema/SummaryTestCase.java index c9fa3ce145a..e1dcfe70e91 100644 --- a/config-model/src/test/java/com/yahoo/schema/SummaryTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/SummaryTestCase.java @@ -6,6 +6,7 @@ import com.yahoo.vespa.documentmodel.DocumentSummary; import com.yahoo.vespa.model.test.utils.DeployLoggerStub; import com.yahoo.vespa.objects.FieldBase; import com.yahoo.yolean.Exceptions; +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import static com.yahoo.config.model.test.TestUtil.joinLines; @@ -48,8 +49,8 @@ public class SummaryTestCase { String sd = joinLines( "schema disksummary {", " document-summary foobar {", - " summary foo1 type string { source: inmemory }", - " summary foo2 type string { source: ondisk }", + " summary foo1 { source: inmemory }", + " summary foo2 { source: ondisk }", " }", " document disksummary {", " field inmemory type string {", @@ -84,8 +85,8 @@ public class SummaryTestCase { " }", " }", " document-summary foobar {", - " summary foo1 type string { source: inmemory }", - " summary foo2 type string { source: ondisk }", + " summary foo1 { source: inmemory }", + " summary foo2 { source: ondisk }", " from-disk", " }", "}"); @@ -114,7 +115,7 @@ public class SummaryTestCase { " }", " }", " document-summary filtered {", - " summary elem_array_filtered type array<elem> {", + " summary elem_array_filtered {", " source: elem_array", " matched-elements-only", " }", @@ -141,17 +142,17 @@ public class SummaryTestCase { " }", " }", " document-summary title {", - " summary title type string {", + " summary title {", " source: title", " }", " }", " document-summary title_artist inherits title {", - " summary artist type string {", + " summary artist {", " source: artist", " }", " }", " document-summary everything inherits title_artist {", - " summary album type string {", + " summary album {", " source: album", " }", " }", @@ -201,12 +202,12 @@ public class SummaryTestCase { " }", " }", " document-summary title {", - " summary title type string {", + " summary title {", " source: title", " }", " }", " document-summary title2 inherits title {", - " summary title type string {", + " summary title {", " source: title_short", " }", " }", @@ -259,12 +260,12 @@ public class SummaryTestCase { } } document-summary parent1 { - summary s1 type string { + summary s1 { source: field1 } } document-summary parent2 { - summary field1 type int { + summary field1 { source: field2 } } @@ -297,12 +298,12 @@ public class SummaryTestCase { } } document-summary parent1 { - summary s1 type string { + summary s1 { source: field1 } } document-summary parent2 { - summary field1 type string { + summary field1 { source: field1 } } @@ -326,7 +327,7 @@ public class SummaryTestCase { " }" + " }" + " document-summary parent_summary {" + - " summary pf1 type string {}" + + " summary pf1 {}" + " }" + "}"); String child = joinLines( @@ -337,12 +338,100 @@ public class SummaryTestCase { " }" + " }" + " document-summary child_summary inherits parent_summary {" + - " summary cf1 type string {}" + + " summary cf1 {}" + " }" + "}"); DeployLoggerStub logger = new DeployLoggerStub(); ApplicationBuilder.createFromStrings(logger, parent, child); assertTrue(logger.entries.isEmpty()); + + } + private void testSummaryTypeInField(boolean explicit) throws ParseException { + String sd = joinLines("schema test {", + " document test {", + " field foo type string {", + " indexing: summary", + " summary bar " + (explicit ? "type string ": "") + "{ }", + " }", + " }", + "}"); + DeployLoggerStub logger = new DeployLoggerStub(); + ApplicationBuilder.createFromStrings(logger, sd); + if (explicit) { + assertEquals(1, logger.entries.size()); + assertEquals(Level.FINE, logger.entries.get(0).level); + assertEquals("For test, field 'foo', summary 'bar':" + + " Specifying the type is deprecated, ignored and will be an error in Vespa 9." + + " Remove the type specification to silence this warning.", logger.entries.get(0).message); + } else { + assertTrue(logger.entries.isEmpty()); + } + } + + @Test + void testSummaryInFieldWithoutTypeEmitsNoWarning() throws ParseException { + testSummaryTypeInField(false); + } + + @Test + void testSummaryInFieldWithTypeEmitsWarning() throws ParseException { + testSummaryTypeInField(true); + } + + private void testSummaryField(boolean explicit) throws ParseException { + String sd = joinLines("schema test {", + " document test {", + " field foo type string { indexing: summary }", + " }", + " document-summary bar {", + " summary foo " + (explicit ? "type string" : "") + "{ }", + " from-disk", + " }", + "}"); + DeployLoggerStub logger = new DeployLoggerStub(); + ApplicationBuilder.createFromStrings(logger, sd); + if (explicit) { + assertEquals(1, logger.entries.size()); + assertEquals(Level.FINE, logger.entries.get(0).level); + assertEquals("For test, document-summary 'bar', summary field 'foo':" + + " Specifying the type is deprecated, ignored and will be an error in Vespa 9." + + " Remove the type specification to silence this warning.", logger.entries.get(0).message); + } else { + assertTrue(logger.entries.isEmpty()); + } + } + + @Test + void testSummaryFieldWithoutTypeEmitsNoWarning() throws ParseException { + testSummaryField(false); + } + + @Test + void testSummaryFieldWithTypeEmitsWarning() throws ParseException { + testSummaryField(true); + } + + @Test + void testSummarySourceLoop() throws ParseException { + String sd = joinLines("schema test {", + " document test {", + " field foo type string { indexing: summary }", + " }", + " document-summary bar {", + " summary foo { source: foo2 }", + " summary foo2 { source: foo3 }", + " summary foo3 { source: foo2 }", + " from-disk", + " }", + "}"); + DeployLoggerStub logger = new DeployLoggerStub(); + try { + ApplicationBuilder.createFromStrings(logger, sd); + fail("expected exception"); + } catch (IllegalArgumentException e) { + assertEquals("For schema 'test' summary class 'bar' summary field 'foo'" + + ": Source loop detected for summary field 'foo2'", e.getMessage()); + } } private static class TestValue { diff --git a/config-model/src/test/java/com/yahoo/schema/derived/SummaryTestCase.java b/config-model/src/test/java/com/yahoo/schema/derived/SummaryTestCase.java index 1f18a5ed49b..5019ed0dd60 100644 --- a/config-model/src/test/java/com/yahoo/schema/derived/SummaryTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/derived/SummaryTestCase.java @@ -131,7 +131,7 @@ public class SummaryTestCase extends AbstractSchemaTestCase { " }", " }", " document-summary my_summary {", - " summary other_campaign_ref type reference<campaign> {}", + " summary other_campaign_ref {}", " }", "}")); builder.build(true); @@ -146,11 +146,11 @@ public class SummaryTestCase extends AbstractSchemaTestCase { " field foo type string { indexing: summary }", " }", " document-summary bar {", - " summary foo type string {}", + " summary foo {}", " omit-summary-features", " }", " document-summary baz {", - " summary foo type string {}", + " summary foo {}", " }", "}"); var search = ApplicationBuilder.createFromString(sd).getSchema(); @@ -221,12 +221,26 @@ public class SummaryTestCase extends AbstractSchemaTestCase { void documentid_summary_field_has_corresponding_summary_transform() throws ParseException { var schema = buildSchema("field foo type string { indexing: summary }", joinLines("document-summary bar {", - " summary documentid type string {}", + " summary documentid {}", "}")); assertOverride(schema, "documentid", SummaryTransform.DOCUMENT_ID.getName(), "", "bar"); } @Test + void tokens_override() throws ParseException { + var schema = buildSchema("field foo type string { indexing: summary }", + joinLines("document-summary bar {", + " summary baz {", + " source: foo ", + " tokens", + " }", + " from-disk", + "}")); + assertOverride(schema, "baz", SummaryTransform.TOKENS.getName(), "foo", "bar"); + assert(!schema.getSummary("default").getSummaryFields().containsKey("baz")); + } + + @Test void documentid_summary_transform_requires_disk_access() { assertFalse(SummaryTransform.DOCUMENT_ID.isInMemory()); } diff --git a/config-model/src/test/java/com/yahoo/schema/processing/AddAttributeTransformToSummaryOfImportedFieldsTest.java b/config-model/src/test/java/com/yahoo/schema/processing/AddDataTypeAndTransformToSummaryOfImportedFieldsTest.java index 3b4a6c0a67b..f09a95b89a0 100644 --- a/config-model/src/test/java/com/yahoo/schema/processing/AddAttributeTransformToSummaryOfImportedFieldsTest.java +++ b/config-model/src/test/java/com/yahoo/schema/processing/AddDataTypeAndTransformToSummaryOfImportedFieldsTest.java @@ -26,7 +26,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; /** * @author bjorncs */ -public class AddAttributeTransformToSummaryOfImportedFieldsTest { +public class AddDataTypeAndTransformToSummaryOfImportedFieldsTest { private static final String IMPORTED_FIELD_NAME = "imported_myfield"; private static final String DOCUMENT_NAME = "mydoc"; @@ -38,7 +38,7 @@ public class AddAttributeTransformToSummaryOfImportedFieldsTest { schema.setImportedFields(createSingleImportedField(IMPORTED_FIELD_NAME)); schema.addSummary(createDocumentSummary(IMPORTED_FIELD_NAME, schema)); - AddAttributeTransformToSummaryOfImportedFields processor = new AddAttributeTransformToSummaryOfImportedFields( + AddDataTypeAndTransformToSummaryOfImportedFields processor = new AddDataTypeAndTransformToSummaryOfImportedFields( schema, null, null, null); processor.process(true, false); SummaryField summaryField = schema.getSummaries().get(SUMMARY_NAME).getSummaryField(IMPORTED_FIELD_NAME); diff --git a/config-model/src/test/java/com/yahoo/schema/processing/MatchedElementsOnlyResolverTestCase.java b/config-model/src/test/java/com/yahoo/schema/processing/MatchedElementsOnlyResolverTestCase.java index e8f8ba4193f..91cd2418eef 100644 --- a/config-model/src/test/java/com/yahoo/schema/processing/MatchedElementsOnlyResolverTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/processing/MatchedElementsOnlyResolverTestCase.java @@ -72,7 +72,7 @@ public class MatchedElementsOnlyResolverTestCase { @Test void explicit_complex_summary_field_can_use_filter_transform_with_reference_to_source_field() throws ParseException { String documentSummary = joinLines("document-summary my_summary {", - " summary my_filter_field type map<string, string> {", + " summary my_filter_field {", " source: my_field", " matched-elements-only", " }", @@ -123,7 +123,7 @@ public class MatchedElementsOnlyResolverTestCase { @Test void explicit_summary_field_can_use_filter_transform_with_reference_to_attribute_source_field() throws ParseException { String documentSummary = joinLines("document-summary my_summary {", - " summary my_filter_field type array<string> {", + " summary my_filter_field {", " source: my_field", " matched-elements-only", " }", diff --git a/config-model/src/test/java/com/yahoo/schema/processing/SummaryConsistencyTestCase.java b/config-model/src/test/java/com/yahoo/schema/processing/SummaryConsistencyTestCase.java index d2938371c5b..9eca2106c5e 100644 --- a/config-model/src/test/java/com/yahoo/schema/processing/SummaryConsistencyTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/processing/SummaryConsistencyTestCase.java @@ -32,7 +32,7 @@ public class SummaryConsistencyTestCase { " }", " }", " document-summary unfiltered {", - " summary elem_array_unfiltered type array<elem> {", + " summary elem_array_unfiltered {", " source: elem_array", " }", " }", diff --git a/config-model/src/test/java/com/yahoo/schema/processing/SummaryDiskAccessValidatorTestCase.java b/config-model/src/test/java/com/yahoo/schema/processing/SummaryDiskAccessValidatorTestCase.java index ab376e539ec..bac29b52949 100644 --- a/config-model/src/test/java/com/yahoo/schema/processing/SummaryDiskAccessValidatorTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/processing/SummaryDiskAccessValidatorTestCase.java @@ -25,7 +25,7 @@ public class SummaryDiskAccessValidatorTestCase { " }", " }", " document-summary my_sum {", - " summary str_map type map<string, string> { source: str_map }", + " summary str_map { source: str_map }", " }", "}"); @@ -58,7 +58,7 @@ public class SummaryDiskAccessValidatorTestCase { " }", " import field ref.str_map as ref_str_map {}", " document-summary my_sum {", - " summary ref_str_map type map<string, string> { source: ref_str_map }", + " summary ref_str_map { source: ref_str_map }", " }", "}"); var logger = new TestableDeployLogger(); diff --git a/config-model/src/test/java/com/yahoo/schema/processing/TokensTransformValidatorTest.java b/config-model/src/test/java/com/yahoo/schema/processing/TokensTransformValidatorTest.java new file mode 100644 index 00000000000..6ca62321617 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/processing/TokensTransformValidatorTest.java @@ -0,0 +1,59 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.processing; + +import com.yahoo.schema.ApplicationBuilder; +import com.yahoo.schema.Schema; +import com.yahoo.schema.parser.ParseException; +import com.yahoo.vespa.documentmodel.SummaryTransform; +import org.junit.jupiter.api.Test; + +import static com.yahoo.config.model.test.TestUtil.joinLines; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.fail; + +public class TokensTransformValidatorTest { + private void buildSchema(String fieldType) throws ParseException { + String sd = joinLines( + "search test {", + " document test {", + " field f type " + fieldType + " {", + " indexing: summary", + " summary: tokens", + " }", + " }", + "}" + ); + Schema schema = ApplicationBuilder.createFromString(sd).getSchema(); + } + + void buildSchemaShouldFail(String fieldType, String expFail) throws ParseException { + try { + buildSchema(fieldType); + fail("expected IllegalArgumentException with message '" + expFail + "'"); + } catch (IllegalArgumentException e) { + assertEquals(expFail, e.getMessage()); + } + } + + @Test + void testTokensTransformWithPlainString() throws ParseException { + buildSchema("string"); + } + + @Test + void testTokensTransformWithArrayOfString() throws ParseException { + buildSchema("array<string>"); + } + + @Test + void testTokensTransformWithWeightedSetOfString() throws ParseException { + buildSchema("weightedset<string>"); + } + + @Test + void testTokensTransformWithWeightedSetOfInteger() throws ParseException { + buildSchemaShouldFail("weightedset<int>", "For schema 'test', document-summary 'default'" + + ", summary field 'f', source field 'f', source field type 'WeightedSet<int>'" + + ": transform 'tokens' is only allowed for fields of type string, array<string> or weightedset<string>"); + } +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/JvmHeapSizeValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/JvmHeapSizeValidatorTest.java index d6da03f5b94..8531aff3b1a 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/JvmHeapSizeValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/JvmHeapSizeValidatorTest.java @@ -2,7 +2,6 @@ package com.yahoo.vespa.model.application.validation; -import com.yahoo.config.ModelReference; import com.yahoo.config.application.api.ApplicationFile; import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.application.api.DeployLogger; @@ -14,7 +13,9 @@ import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.deploy.TestProperties; import com.yahoo.config.model.provision.InMemoryProvisioner; import com.yahoo.config.model.test.MockApplicationPackage; +import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.NodeResources; +import com.yahoo.text.Text; import com.yahoo.vespa.model.VespaModel; import org.junit.jupiter.api.Test; import org.xml.sax.SAXException; @@ -97,7 +98,7 @@ class JvmHeapSizeValidatorTest { private static DeployState createDeployState(double nodeGb, long modelCostBytes) { String servicesXml = - """ + Text.format(""" <?xml version="1.0" encoding="utf-8" ?> <services version='1.0'> <container version='1.0'> @@ -109,7 +110,7 @@ class JvmHeapSizeValidatorTest { <tokenizer-model path="app/tokenizer.json"/> </component> </container> - </services>""".formatted(nodeGb); + </services>""", nodeGb); return createDeployState(servicesXml, nodeGb, modelCostBytes); } @@ -119,12 +120,10 @@ class JvmHeapSizeValidatorTest { ModelCostDummy(long modelCost) { this.modelCost = modelCost; } - @Override public Calculator newCalculator(ApplicationPackage appPkg, DeployLogger logger) { return this; } + @Override public Calculator newCalculator(ApplicationPackage appPkg, ApplicationId applicationId) { return this; } @Override public long aggregatedModelCostInBytes() { return totalCost.get(); } @Override public void registerModel(ApplicationFile path) {} - @SuppressWarnings("removal") @Override public void registerModel(ModelReference ref) {} - @Override public void registerModel(URI uri) { assertEquals("https://my/url/model.onnx", uri.toString()); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/DomSchemaTuningBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/DomSchemaTuningBuilderTest.java index 4d2900b9e94..963fce0c666 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/DomSchemaTuningBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/DomSchemaTuningBuilderTest.java @@ -27,10 +27,6 @@ public class DomSchemaTuningBuilderTest extends DomBuilderTest { "</tuning>"); } - private Tuning newTuning(String xml) { - return createTuning(parse(xml)); - } - private Tuning createTuning(Element xml) { DomSearchTuningBuilder b = new DomSearchTuningBuilder(); return b.build(root.getDeployState(), root, xml); @@ -113,14 +109,11 @@ public class DomSchemaTuningBuilderTest extends DomBuilderTest { "<amortize-count>13</amortize-count>", "</resizing>")); assertEquals(128, t.searchNode.resizing.initialDocumentCount.intValue()); - assertEquals(13, t.searchNode.resizing.amortizeCount.intValue()); } @Test void requireThatWeCanParseIndexTag() { Tuning t = createTuning(parseXml("<index>", "<io>", - "<write>directio</write>", - "<read>normal</read>", "<search>mmap</search>", "</io>", "<warmup>" + @@ -128,14 +121,12 @@ public class DomSchemaTuningBuilderTest extends DomBuilderTest { "<unpack>true</unpack>", "</warmup>", "</index>")); - assertEquals(Tuning.SearchNode.IoType.DIRECTIO, t.searchNode.index.io.write); - assertEquals(Tuning.SearchNode.IoType.NORMAL, t.searchNode.index.io.read); assertEquals(Tuning.SearchNode.IoType.MMAP, t.searchNode.index.io.search); assertEquals(178, t.searchNode.index.warmup.time, DELTA); assertTrue(t.searchNode.index.warmup.unpack); ProtonConfig cfg = getProtonCfg(t); assertEquals(cfg.indexing().write().io(), ProtonConfig.Indexing.Write.Io.DIRECTIO); - assertEquals(cfg.indexing().read().io(), ProtonConfig.Indexing.Read.Io.NORMAL); + assertEquals(cfg.indexing().read().io(), ProtonConfig.Indexing.Read.Io.DIRECTIO); assertEquals(cfg.index().warmup().time(), 178, DELTA); assertTrue(cfg.index().warmup().unpack()); } @@ -172,9 +163,8 @@ public class DomSchemaTuningBuilderTest extends DomBuilderTest { @Test void requireThatWeCanParseAttributeTag() { Tuning t = createTuning(parseXml("<attribute>", "<io>", - "<write>directio</write>", + "<write>normal</write>", "</io>", "</attribute>")); - assertEquals(Tuning.SearchNode.IoType.DIRECTIO, t.searchNode.attribute.io.write); ProtonConfig cfg = getProtonCfg(t); assertEquals(cfg.attribute().write().io(), ProtonConfig.Attribute.Write.Io.DIRECTIO); } @@ -209,7 +199,6 @@ public class DomSchemaTuningBuilderTest extends DomBuilderTest { "</logstore>", "</store>", "</summary>")); - assertEquals(Tuning.SearchNode.IoType.DIRECTIO, t.searchNode.summary.io.write); assertEquals(Tuning.SearchNode.IoType.DIRECTIO, t.searchNode.summary.io.read); assertEquals(128, t.searchNode.summary.store.cache.maxSize.longValue()); assertEquals(30.7, t.searchNode.summary.store.cache.maxSizePercent, DELTA); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/AccessLogTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/AccessLogTest.java index f2e4ec052cb..a38a29893e0 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/AccessLogTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/AccessLogTest.java @@ -16,6 +16,7 @@ import com.yahoo.container.logging.ConnectionLogConfig; import com.yahoo.container.logging.FileConnectionLog; import com.yahoo.container.logging.JSONAccessLog; import com.yahoo.container.logging.VespaAccessLog; +import com.yahoo.jdisc.http.ServerConfig; import com.yahoo.vespa.model.container.ApplicationContainerCluster; import com.yahoo.vespa.model.container.component.Component; import org.junit.jupiter.api.Test; @@ -129,6 +130,7 @@ public class AccessLogTest extends ContainerModelBuilderTestBase { assertEquals("default", config.cluster()); assertEquals(-1, config.queueSize()); assertEquals(256 * 1024, config.bufferSize()); + assertTrue(root.getConfig(ServerConfig.class, "default/container.0/DefaultHttpServer").connectionLog().enabled()); } @Test @@ -141,6 +143,7 @@ public class AccessLogTest extends ContainerModelBuilderTestBase { createModel(root, clusterElem); Component<?, ?> fileConnectionLogComponent = getComponent("default", FileConnectionLog.class.getName()); assertNull(fileConnectionLogComponent); + assertFalse(root.getConfig(ServerConfig.class, "default/container.0/DefaultHttpServer").connectionLog().enabled()); } @Test diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/CloudDataPlaneFilterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/CloudDataPlaneFilterTest.java index 937052df122..49ed1972afe 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/CloudDataPlaneFilterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/CloudDataPlaneFilterTest.java @@ -1,6 +1,8 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.container.xml; +import com.yahoo.config.model.api.ApplicationClusterEndpoint; +import com.yahoo.config.model.api.ContainerEndpoint; import com.yahoo.config.model.api.EndpointCertificateSecrets; import com.yahoo.config.model.builder.xml.test.DomBuilderTest; import com.yahoo.config.model.deploy.DeployState; @@ -38,6 +40,7 @@ import java.time.Instant; import java.time.temporal.ChronoUnit; import java.util.List; import java.util.Optional; +import java.util.Set; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; @@ -91,6 +94,7 @@ public class CloudDataPlaneFilterTest extends ContainerModelBuilderTestBase { var caCerts = X509CertificateUtils.certificateListFromPem(connectorConfig.ssl().caCertificate()); assertEquals(1, caCerts.size()); assertEquals(List.of(certificate), caCerts); + assertEquals(List.of("foo.bar"), connectorConfig.serverName().known()); var srvCfg = root.getConfig(ServerConfig.class, "container/http"); assertEquals("cloud-data-plane-insecure", srvCfg.defaultFilters().get(0).filterId()); assertEquals(8080, srvCfg.defaultFilters().get(0).localPort()); @@ -191,6 +195,7 @@ public class CloudDataPlaneFilterTest extends ContainerModelBuilderTestBase { .setEndpointCertificateSecrets(Optional.of(new EndpointCertificateSecrets("CERT", "KEY"))) .setHostedVespa(true)) .zone(new Zone(SystemName.PublicCd, Environment.dev, RegionName.defaultName())) + .endpoints(Set.of(new ContainerEndpoint("foo", ApplicationClusterEndpoint.Scope.zone, List.of("foo.bar")))) .build(); return createModel(root, state, null, clusterElem); } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/CloudTokenDataPlaneFilterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/CloudTokenDataPlaneFilterTest.java index 1642e0ff8f2..c89ea421b39 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/CloudTokenDataPlaneFilterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/CloudTokenDataPlaneFilterTest.java @@ -14,9 +14,12 @@ import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.Zone; +import com.yahoo.jdisc.http.ConnectorConfig; import com.yahoo.jdisc.http.filter.security.cloud.config.CloudTokenDataPlaneFilterConfig; import com.yahoo.processing.response.Data; +import com.yahoo.vespa.model.container.ApplicationContainer; import com.yahoo.vespa.model.container.ContainerModel; +import com.yahoo.vespa.model.container.http.ConnectorFactory; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; @@ -127,6 +130,21 @@ public class CloudTokenDataPlaneFilterTest extends ContainerModelBuilderTestBase assertFalse(root.getConfigIds().stream().anyMatch(id -> id.contains("DataplaneProxyConfigurator"))); } + @Test + void configuresCorrectConnectors() throws IOException { + var certFile = securityFolder.resolve("foo.pem"); + var clusterElem = DomBuilderTest.parse(servicesXmlTemplate.formatted(applicationFolder.toPath().relativize(certFile).toString())); + createCertificate(certFile); + buildModel(Set.of(tokenEndpoint, mtlsEndpoint), defaultTokens, clusterElem); + + ConnectorConfig connectorConfig8443 = connectorConfig(8443); + assertEquals(List.of("mtls"),connectorConfig8443.serverName().known()); + + ConnectorConfig connectorConfig8444 = connectorConfig(8444); + assertEquals(List.of("token"),connectorConfig8444.serverName().known()); + + } + private static CloudTokenDataPlaneFilterConfig.Clients.Tokens tokenConfig( String id, Collection<String> fingerprints, Collection<String> accessCheckHashes, Collection<String> expirations) { return new CloudTokenDataPlaneFilterConfig.Clients.Tokens.Builder() @@ -150,4 +168,15 @@ public class CloudTokenDataPlaneFilterTest extends ContainerModelBuilderTestBase .build(); return createModel(root, state, null, clusterElem); } + + private ConnectorConfig connectorConfig(int port) { + ApplicationContainer container = (ApplicationContainer) root.getProducer("container/container.0"); + List<ConnectorFactory> connectorFactories = container.getHttp().getHttpServer().get().getConnectorFactories(); + ConnectorFactory tlsPort = connectorFactories.stream().filter(connectorFactory -> connectorFactory.getListenPort() == port).findFirst().orElseThrow(); + + ConnectorConfig.Builder builder = new ConnectorConfig.Builder(); + tlsPort.getConfig(builder); + + return new ConnectorConfig(builder); + } } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/HandlerBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/HandlerBuilderTest.java index 8a7ca27eec5..fdeea85c5a3 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/HandlerBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/HandlerBuilderTest.java @@ -1,11 +1,11 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.container.xml; +import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.model.ConfigModelContext; import com.yahoo.config.model.builder.xml.test.DomBuilderTest; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.deploy.TestProperties; -import com.yahoo.config.provision.ApplicationId; import com.yahoo.container.ComponentsConfig; import com.yahoo.container.jdisc.JdiscBindingsConfig; import com.yahoo.container.usability.BindingsOverviewHandler; @@ -15,8 +15,10 @@ import com.yahoo.vespa.model.container.component.Handler; import org.junit.jupiter.api.Test; import org.w3c.dom.Element; +import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.logging.Level; import static com.yahoo.vespa.model.container.ContainerCluster.ROOT_HANDLER_BINDING; import static com.yahoo.vespa.model.container.ContainerCluster.STATE_HANDLER_BINDING_1; @@ -25,7 +27,11 @@ import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.hasItem; -import static org.junit.jupiter.api.Assertions.*; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; /** * Tests for container model building with custom handlers. @@ -63,6 +69,31 @@ public class HandlerBuilderTest extends ContainerModelBuilderTestBase { } @Test + void warn_on_bindings_shared_by_multiple_handlers() { + class TestDeployLogger implements DeployLogger { + List<String> logs = new ArrayList<>(); + @Override public void log(Level level, String message) { logs.add(message); } + } + var clusterElem = DomBuilderTest.parse( + "<container id='default' version='1.0'>", + " <handler id='myHandler1'>", + " <binding>http://*/myhandler</binding>", + " <binding>https://*/myhandler</binding>", + " </handler>", + " <handler id='myHandler2'>", + " <binding>http://*/myhandler</binding>", + " <binding>https://*/myhandler</binding>", + " </handler>", + "</container>"); + var logger = new TestDeployLogger(); + createModel(root, logger, clusterElem); + assertEquals( + List.of("Binding 'http://*/myhandler' was already in use by handler 'myHandler1', but will now be taken over by handler: myHandler2", + "Binding 'https://*/myhandler' was already in use by handler 'myHandler1', but will now be taken over by handler: myHandler2"), + logger.logs); + } + + @Test void default_root_handler_binding_can_be_stolen_by_user_configured_handler() { Element clusterElem = DomBuilderTest.parse( "<container id='default' version='1.0'>" + diff --git a/config-model/src/test/java/com/yahoo/vespa/model/filedistribution/UserConfiguredFilesTest.java b/config-model/src/test/java/com/yahoo/vespa/model/filedistribution/UserConfiguredFilesTest.java index 523b0e74be1..3dd845ec56f 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/filedistribution/UserConfiguredFilesTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/filedistribution/UserConfiguredFilesTest.java @@ -5,10 +5,13 @@ import com.yahoo.config.FileNode; import com.yahoo.config.FileReference; import com.yahoo.config.ModelReference; import com.yahoo.config.UrlReference; +import com.yahoo.config.application.api.ApplicationPackage; +import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.application.api.FileRegistry; import com.yahoo.config.model.application.provider.BaseDeployLogger; import com.yahoo.config.model.deploy.TestProperties; import com.yahoo.config.model.producer.UserConfigRepo; +import com.yahoo.config.model.test.MockApplicationPackage; import com.yahoo.config.model.test.MockRoot; import com.yahoo.vespa.config.ConfigDefinition; import com.yahoo.vespa.config.ConfigDefinitionKey; @@ -25,6 +28,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.logging.Level; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; @@ -69,12 +73,20 @@ public class UserConfiguredFilesTest { public String toString() { return export().toString(); } } - private UserConfiguredFiles userConfiguredFiles() { + return userConfiguredFiles(new MockApplicationPackage.Builder().build()); + } + + private UserConfiguredFiles userConfiguredFiles(ApplicationPackage applicationPackage) { + return userConfiguredFiles(applicationPackage, new BaseDeployLogger()); + } + + private UserConfiguredFiles userConfiguredFiles(ApplicationPackage applicationPackage, DeployLogger deployLogger) { return new UserConfiguredFiles(fileRegistry, - new BaseDeployLogger(), + deployLogger, new TestProperties(), - new ApplicationContainerCluster.UserConfiguredUrls()); + new ApplicationContainerCluster.UserConfiguredUrls(), + applicationPackage); } @BeforeEach @@ -289,18 +301,42 @@ public class UserConfiguredFilesTest { } @Test - void require_that_using_empty_dir_gives_sane_error_message(@TempDir Path tempDir) { - String relativeTempDir = tempDir.toString().substring(tempDir.toString().lastIndexOf("target")); + void require_that_using_empty_dir_fails(@TempDir Path tempDir) { + String relativeTempDir = tempDir.toString().substring(tempDir.toString().lastIndexOf("target") + 7); + ApplicationPackage applicationPackage = + new MockApplicationPackage.Builder() + .withRoot(tempDir.toFile().getParentFile()) + .withFiles(Map.of(com.yahoo.path.Path.fromString(tempDir.toFile().getAbsolutePath()), "")) + .build(); + + var logger = new TestDeployLogger(); + def.addPathDef("pathVal"); + builder.setField("pathVal", relativeTempDir); + fileRegistry.pathToRef.put(relativeTempDir, new FileReference("bazshash")); + userConfiguredFiles(applicationPackage, logger).register(producer); + assertEquals("Directory '" + relativeTempDir + "' is empty", logger.log); + } + + @Test + void require_that_using_non_existing_dir_fails() { + String relativeTempDir = "non-existing"; try { def.addPathDef("pathVal"); builder.setField("pathVal", relativeTempDir); - fileRegistry.pathToRef.put(relativeTempDir, new FileReference("bazshash")); userConfiguredFiles().register(producer); fail("Should have thrown exception"); } catch (IllegalArgumentException e) { - assertEquals("Invalid config in services.xml for 'mynamespace.myname': Directory '" + relativeTempDir + "' is empty", + assertEquals("Invalid config in services.xml for 'mynamespace.myname': No such file or directory '" + relativeTempDir + "'", e.getMessage()); } } + private static class TestDeployLogger implements DeployLogger { + public String log = ""; + @Override + public void log(Level level, String message) { + log += message; + } + } + } diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/ProvisionLock.java b/config-provisioning/src/main/java/com/yahoo/config/provision/ApplicationMutex.java index bf683eddc4c..f147bde65f7 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/ProvisionLock.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/ApplicationMutex.java @@ -10,12 +10,12 @@ import java.util.Objects; * * @author mpolden */ -public class ProvisionLock implements AutoCloseable { +public class ApplicationMutex implements Mutex { private final ApplicationId application; private final Mutex lock; - public ProvisionLock(ApplicationId application, Mutex lock) { + public ApplicationMutex(ApplicationId application, Mutex lock) { this.application = Objects.requireNonNull(application); this.lock = Objects.requireNonNull(lock); } diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/ApplicationTransaction.java b/config-provisioning/src/main/java/com/yahoo/config/provision/ApplicationTransaction.java index bc755e02af7..e7c4ed65c46 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/ApplicationTransaction.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/ApplicationTransaction.java @@ -13,10 +13,10 @@ import java.util.Objects; */ public class ApplicationTransaction implements Closeable { - private final ProvisionLock lock; + private final ApplicationMutex lock; private final NestedTransaction transaction; - public ApplicationTransaction(ProvisionLock lock, NestedTransaction transaction) { + public ApplicationTransaction(ApplicationMutex lock, NestedTransaction transaction) { this.lock = Objects.requireNonNull(lock); this.transaction = Objects.requireNonNull(transaction); } diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterSpec.java b/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterSpec.java index ed0f9aac884..bb9bf8db4f3 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterSpec.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterSpec.java @@ -2,8 +2,6 @@ package com.yahoo.config.provision; import com.yahoo.component.Version; -import com.yahoo.config.provision.ZoneEndpoint.AccessType; -import com.yahoo.config.provision.ZoneEndpoint.AllowedUrn; import java.util.Objects; import java.util.Optional; @@ -96,12 +94,6 @@ public final class ClusterSpec { return new ClusterSpec(type, id, groupId, vespaVersion, exclusive, combinedId, dockerImageRepo, zoneEndpoint, stateful); } - // TODO: Remove after July 2023 - @Deprecated - public ClusterSpec exclusive(boolean exclusive) { - return new ClusterSpec(type, id, groupId, vespaVersion, exclusive, combinedId, dockerImageRepo, zoneEndpoint, stateful); - } - /** Creates a ClusterSpec when requesting a cluster */ public static Builder request(Type type, Id id) { return new Builder(type, id); @@ -121,6 +113,7 @@ public final class ClusterSpec { private Optional<DockerImage> dockerImageRepo = Optional.empty(); private Version vespaVersion; private boolean exclusive = false; + private boolean provisionForApplication = false; private Optional<Id> combinedId = Optional.empty(); private ZoneEndpoint zoneEndpoint = ZoneEndpoint.defaultEndpoint; private boolean stateful; @@ -155,6 +148,11 @@ public final class ClusterSpec { return this; } + public Builder provisionForApplication(boolean provisionForApplication) { + this.provisionForApplication = provisionForApplication; + return this; + } + public Builder combinedId(Optional<Id> combinedId) { this.combinedId = combinedId; return this; diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/NodeResources.java b/config-provisioning/src/main/java/com/yahoo/config/provision/NodeResources.java index c1296c6708e..a0c48200ff6 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/NodeResources.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/NodeResources.java @@ -278,7 +278,7 @@ public class NodeResources { return new NodeResources(vcpu, memoryGb, diskGb, bandwidthGbps, diskSpeed, storageType, architecture, gpuResources); } - public NodeResources withUnspecifiedNumbersFrom(NodeResources fullySpecified) { + public NodeResources withUnspecifiedFieldsFrom(NodeResources fullySpecified) { var resources = this; if (resources.vcpuIsUnspecified()) resources = resources.withVcpu(fullySpecified.vcpu()); @@ -288,6 +288,13 @@ public class NodeResources { resources = resources.withDiskGb(fullySpecified.diskGb()); if (resources.bandwidthGbpsIsUnspecified()) resources = resources.withBandwidthGbps(fullySpecified.bandwidthGbps()); + if (resources.diskSpeed() == DiskSpeed.any) + resources = resources.with(fullySpecified.diskSpeed()); + if (resources.storageType() == StorageType.any) + resources = resources.with(fullySpecified.storageType()); + if (resources.architecture() == Architecture.any) + resources = resources.with(fullySpecified.architecture()); + assert fullySpecified.gpuResources() == GpuResources.zero : "Not handled"; return resources; } diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/NodeType.java b/config-provisioning/src/main/java/com/yahoo/config/provision/NodeType.java index 266ede04800..0eac568ec45 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/NodeType.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/NodeType.java @@ -88,6 +88,14 @@ public enum NodeType { return childNodeTypes.contains(type); } + /** Returns the parent host type. */ + public NodeType parentNodeType() { + for (var type : values()) { + if (type.childNodeTypes.contains(this)) return type; + } + throw new IllegalStateException(this + " has no parent"); + } + /** Returns the host type of this */ public NodeType hostType() { if (isHost()) return this; @@ -97,7 +105,7 @@ public enum NodeType { return nodeType; } } - throw new IllegalArgumentException("No host of " + this + " exists"); + throw new IllegalStateException("No host of " + this + " exists"); } } diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/Provisioner.java b/config-provisioning/src/main/java/com/yahoo/config/provision/Provisioner.java index 38ab69d364c..4d1433cf5e9 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/Provisioner.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/Provisioner.java @@ -37,6 +37,6 @@ public interface Provisioner { void restart(ApplicationId application, HostFilter filter); /** Returns a provision lock for the given application */ - ProvisionLock lock(ApplicationId application); + ApplicationMutex lock(ApplicationId application); } diff --git a/config-provisioning/src/test/java/com/yahoo/config/provision/NodeResourcesTest.java b/config-provisioning/src/test/java/com/yahoo/config/provision/NodeResourcesTest.java index 27b659fe1c6..65ec070d744 100644 --- a/config-provisioning/src/test/java/com/yahoo/config/provision/NodeResourcesTest.java +++ b/config-provisioning/src/test/java/com/yahoo/config/provision/NodeResourcesTest.java @@ -1,22 +1,27 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.config.provision; +import com.yahoo.config.provision.NodeResources.Architecture; +import com.yahoo.config.provision.NodeResources.DiskSpeed; +import com.yahoo.config.provision.NodeResources.GpuResources; +import com.yahoo.config.provision.NodeResources.StorageType; import org.junit.jupiter.api.Test; import java.util.function.Supplier; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assertions.fail; /** * @author bratseth */ -public class NodeResourcesTest { +class NodeResourcesTest { @Test - public void testCost() { + void testCost() { assertEquals(5.408, new NodeResources(32, 128, 1200, 1).cost(), 0.0001); } @@ -81,7 +86,40 @@ public class NodeResourcesTest { } @Test - public void testJoiningResources() { + void testSpecifyFully() { + NodeResources empty = new NodeResources(0, 0, 0, 0).with(DiskSpeed.any); + + assertEquals(0, empty.withUnspecifiedFieldsFrom(empty).vcpu()); + assertEquals(3, empty.withUnspecifiedFieldsFrom(empty.withVcpu(3)).vcpu()); + assertEquals(2, empty.withVcpu(2).withUnspecifiedFieldsFrom(empty.withVcpu(3)).vcpu()); + + assertEquals(0, empty.withUnspecifiedFieldsFrom(empty).memoryGb()); + assertEquals(3, empty.withUnspecifiedFieldsFrom(empty.withMemoryGb(3)).memoryGb()); + assertEquals(2, empty.withMemoryGb(2).withUnspecifiedFieldsFrom(empty.withMemoryGb(3)).memoryGb()); + + assertEquals(0, empty.withUnspecifiedFieldsFrom(empty).diskGb()); + assertEquals(3, empty.withUnspecifiedFieldsFrom(empty.withDiskGb(3)).diskGb()); + assertEquals(2, empty.withDiskGb(2).withUnspecifiedFieldsFrom(empty.withDiskGb(3)).diskGb()); + + assertEquals(0, empty.withUnspecifiedFieldsFrom(empty).bandwidthGbps()); + assertEquals(3, empty.withUnspecifiedFieldsFrom(empty.withBandwidthGbps(3)).bandwidthGbps()); + assertEquals(2, empty.withBandwidthGbps(2).withUnspecifiedFieldsFrom(empty.withBandwidthGbps(3)).bandwidthGbps()); + + assertEquals(Architecture.any, empty.withUnspecifiedFieldsFrom(empty).architecture()); + assertEquals(Architecture.arm64, empty.withUnspecifiedFieldsFrom(empty.with(Architecture.arm64)).architecture()); + assertEquals(Architecture.x86_64, empty.with(Architecture.x86_64).withUnspecifiedFieldsFrom(empty.with(Architecture.arm64)).architecture()); + + assertEquals(DiskSpeed.any, empty.withUnspecifiedFieldsFrom(empty).diskSpeed()); + assertEquals(DiskSpeed.fast, empty.withUnspecifiedFieldsFrom(empty.with(DiskSpeed.fast)).diskSpeed()); + assertEquals(DiskSpeed.slow, empty.with(DiskSpeed.slow).withUnspecifiedFieldsFrom(empty.with(DiskSpeed.fast)).diskSpeed()); + + assertEquals(StorageType.any, empty.withUnspecifiedFieldsFrom(empty).storageType()); + assertEquals(StorageType.local, empty.withUnspecifiedFieldsFrom(empty.with(StorageType.local)).storageType()); + assertEquals(StorageType.remote, empty.with(StorageType.remote).withUnspecifiedFieldsFrom(empty.with(StorageType.local)).storageType()); + } + + @Test + void testJoiningResources() { var resources = new NodeResources(1, 2, 3, 1, NodeResources.DiskSpeed.fast, NodeResources.StorageType.local, diff --git a/config/src/vespa/config/frt/frtsource.cpp b/config/src/vespa/config/frt/frtsource.cpp index 42472750114..b160984e0e8 100644 --- a/config/src/vespa/config/frt/frtsource.cpp +++ b/config/src/vespa/config/frt/frtsource.cpp @@ -39,7 +39,7 @@ FRTSource::FRTSource(std::shared_ptr<ConnectionFactory> connectionFactory, const _lock(), _inflight(), _task(std::make_unique<GetConfigTask>(_connectionFactory->getScheduler(), this)), - _closed(false) + _state(State::OPEN) { LOG(spam, "New source!"); } @@ -68,6 +68,9 @@ FRTSource::getConfig() FRT_RPCRequest * req = request->getRequest(); { std::lock_guard guard(_lock); + if (_state != State::OPEN) { + return; + } _inflight[req] = std::move(request); } connection->invoke(req, clientTimeout, this); @@ -126,11 +129,19 @@ FRTSource::close() { RequestMap inflight; { - std::lock_guard guard(_lock); - if (_closed) + std::unique_lock guard(_lock); + if (_state != State::OPEN) { + while (_state != State::CLOSED) { + _cond.wait(guard); // Wait for first close to finish + } return; - LOG(spam, "Killing task"); - _task->Kill(); + } + _state = State::CLOSING; + } + LOG(spam, "Killing task"); + _task->Kill(); + { + std::lock_guard guard(_lock); inflight = _inflight; } LOG(spam, "Aborting"); @@ -144,14 +155,17 @@ FRTSource::close() _cond.wait(guard); } LOG(spam, "closed"); + _state = State::CLOSED; + _cond.notify_all(); } void FRTSource::scheduleNextGetConfig() { std::lock_guard guard(_lock); - if (_closed) + if (_state != State::OPEN) { return; + } double sec = vespalib::to_s(_agent->getWaitTime()); LOG(debug, "Scheduling task in %f seconds", sec); _task->Schedule(sec); diff --git a/config/src/vespa/config/frt/frtsource.h b/config/src/vespa/config/frt/frtsource.h index 7488b4b57c8..92aa6a7370d 100644 --- a/config/src/vespa/config/frt/frtsource.h +++ b/config/src/vespa/config/frt/frtsource.h @@ -37,11 +37,11 @@ private: const FRTConfigRequestFactory & _requestFactory; std::unique_ptr<ConfigAgent> _agent; const ConfigKey _key; - std::mutex _lock; // Protects _inflight, _task and _closed + std::mutex _lock; // Protects _inflight, _task and _state std::condition_variable _cond; RequestMap _inflight; std::unique_ptr<FNET_Task> _task; - bool _closed; + enum class State : uint8_t { OPEN, CLOSING, CLOSED } _state; }; } // namespace config diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java index 2680b4babb1..e1629a6c2c3 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java @@ -660,11 +660,14 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye log.log(Level.FINE, () -> "Remove unused file references last modified before " + instant); List<String> fileReferencesToDelete = sortedUnusedFileReferences(fileDirectory.getRoot(), fileReferencesInUse, instant); - if (fileReferencesToDelete.size() > 0) { - log.log(Level.FINE, () -> "Will delete file references not in use: " + fileReferencesToDelete); - fileReferencesToDelete.forEach(fileReference -> fileDirectory.delete(new FileReference(fileReference), this::isFileReferenceInUse)); + // Do max 20 at a time + var toDelete = fileReferencesToDelete.subList(0, Math.min(fileReferencesToDelete.size(), 20)); + if (toDelete.size() > 0) { + log.log(Level.FINE, () -> "Will delete file references not in use: " + toDelete); + toDelete.forEach(fileReference -> fileDirectory.delete(new FileReference(fileReference), this::isFileReferenceInUse)); + log.log(Level.FINE, () -> "Deleted " + toDelete.size() + " file references not in use"); } - return fileReferencesToDelete; + return toDelete; } private boolean isFileReferenceInUse(FileReference fileReference) { @@ -684,7 +687,7 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye private List<String> sortedUnusedFileReferences(File fileReferencesPath, Set<String> fileReferencesInUse, Instant instant) { Set<String> fileReferencesOnDisk = getFileReferencesOnDisk(fileReferencesPath); - log.log(Level.FINE, () -> "File references on disk (in " + fileReferencesPath + "): " + fileReferencesOnDisk); + log.log(Level.FINEST, () -> "File references on disk (in " + fileReferencesPath + "): " + fileReferencesOnDisk); return fileReferencesOnDisk .stream() .filter(fileReference -> ! fileReferencesInUse.contains(fileReference)) diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java index 95471fcdea0..c7877c23323 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java @@ -11,7 +11,7 @@ import com.yahoo.config.provision.ApplicationLockException; import com.yahoo.config.provision.ApplicationTransaction; import com.yahoo.config.provision.HostFilter; import com.yahoo.config.provision.HostSpec; -import com.yahoo.config.provision.ProvisionLock; +import com.yahoo.config.provision.ApplicationMutex; import com.yahoo.config.provision.Provisioner; import com.yahoo.config.provision.TransientException; import com.yahoo.transaction.NestedTransaction; @@ -313,7 +313,7 @@ public class Deployment implements com.yahoo.config.provision.Deployment { () -> "Timeout exceeded while waiting for application resources of '" + session.getApplicationId() + "'" + Optional.ofNullable(lastException.get()).map(e -> ". Last exception: " + e.getMessage()).orElse("")); - try (ProvisionLock lock = provisioner.get().lock(session.getApplicationId())) { + try (ApplicationMutex lock = provisioner.get().lock(session.getApplicationId())) { // Call to activate to make sure that everything is ready, but do not commit the transaction ApplicationTransaction transaction = new ApplicationTransaction(lock, new NestedTransaction()); provisioner.get().activate(preparedHosts, context, transaction); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java index a79e74f5744..5ad8eee90e8 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java @@ -309,7 +309,7 @@ public class ModelContextImpl implements ModelContext { private static <V> V flagValue(FlagSource source, ApplicationId appId, Version vespaVersion, UnboundFlag<? extends V, ?, ?> flag) { return flag.bindTo(source) .with(FetchVector.Dimension.INSTANCE_ID, appId.serializedForm()) - .with(FetchVector.Dimension.APPLICATION_ID, appId.toSerializedFormWithoutInstance()) + .with(FetchVector.Dimension.APPLICATION, appId.toSerializedFormWithoutInstance()) .with(FetchVector.Dimension.VESPA_VERSION, vespaVersion.toFullString()) .with(FetchVector.Dimension.TENANT_ID, appId.tenant().value()) .boxedValue(); @@ -322,7 +322,7 @@ public class ModelContextImpl implements ModelContext { UnboundFlag<? extends V, ?, ?> flag) { return flag.bindTo(source) .with(FetchVector.Dimension.INSTANCE_ID, appId.serializedForm()) - .with(FetchVector.Dimension.APPLICATION_ID, appId.toSerializedFormWithoutInstance()) + .with(FetchVector.Dimension.APPLICATION, appId.toSerializedFormWithoutInstance()) .with(FetchVector.Dimension.CLUSTER_TYPE, clusterType.name()) .with(FetchVector.Dimension.VESPA_VERSION, vespaVersion.toFullString()) .boxedValue(); @@ -335,7 +335,7 @@ public class ModelContextImpl implements ModelContext { UnboundFlag<? extends V, ?, ?> flag) { return flag.bindTo(source) .with(FetchVector.Dimension.INSTANCE_ID, appId.serializedForm()) - .with(FetchVector.Dimension.APPLICATION_ID, appId.toSerializedFormWithoutInstance()) + .with(FetchVector.Dimension.APPLICATION, appId.toSerializedFormWithoutInstance()) .with(FetchVector.Dimension.CLUSTER_ID, clusterId.value()) .with(FetchVector.Dimension.VESPA_VERSION, vespaVersion.toFullString()) .boxedValue(); @@ -413,7 +413,7 @@ public class ModelContextImpl implements ModelContext { this.secretStore = secretStore; this.jvmGCOptionsFlag = PermanentFlags.JVM_GC_OPTIONS.bindTo(flagSource) .with(FetchVector.Dimension.INSTANCE_ID, applicationId.serializedForm()) - .with(FetchVector.Dimension.APPLICATION_ID, applicationId.toSerializedFormWithoutInstance()); + .with(FetchVector.Dimension.APPLICATION, applicationId.toSerializedFormWithoutInstance()); this.allowDisableMtls = flagValue(flagSource, applicationId, PermanentFlags.ALLOW_DISABLE_MTLS); this.operatorCertificates = operatorCertificates; this.tlsCiphersOverride = flagValue(flagSource, applicationId, PermanentFlags.TLS_CIPHERS_OVERRIDE); @@ -525,7 +525,7 @@ public class ModelContextImpl implements ModelContext { private static <V> V flagValue(FlagSource source, ApplicationId appId, UnboundFlag<? extends V, ?, ?> flag) { return flag.bindTo(source) .with(FetchVector.Dimension.INSTANCE_ID, appId.serializedForm()) - .with(FetchVector.Dimension.APPLICATION_ID, appId.toSerializedFormWithoutInstance()) + .with(FetchVector.Dimension.APPLICATION, appId.toSerializedFormWithoutInstance()) .boxedValue(); } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationApiHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationApiHandler.java index ade63e8c90c..29f2125ac3c 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationApiHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationApiHandler.java @@ -93,12 +93,13 @@ public class ApplicationApiHandler extends SessionHandler { PartItem appPackagePart = parts.get(MULTIPART_APPLICATION_PACKAGE); compressedStream = createFromCompressedStream(appPackagePart.data(), appPackagePart.contentType(), maxApplicationPackageSize); } catch (IOException e) { - // Multipart exception happens when controller abandons the request due to other exceptions while deploying. - log.log(e instanceof MultiPartFormParser.MultiPartException ? FINE : WARNING, - "Unable to parse multipart in deploy from tenant '" + tenantName.value() + "': " + Exceptions.toMessageString(e)); - var message = "Deploy request from '" + tenantName.value() + "' contains invalid data: " + e.getMessage(); - log.log(INFO, message + ", parts: " + parts, e); + if (e instanceof MultiPartFormParser.MultiPartException) + log.log(INFO, "Unable to parse multipart in deploy from tenant '" + tenantName.value() + "': " + + Exceptions.toMessageString(e) + ". This is usually caused by controller abandoning request " + + "while streaming data to config server"); + else + log.log(INFO, message + ", parts: " + parts, e); throw new BadRequestException(message); } } else { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ApplicationPackageMaintainer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ApplicationPackageMaintainer.java index 82732a00dc4..dcc5d7caa0d 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ApplicationPackageMaintainer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ApplicationPackageMaintainer.java @@ -16,7 +16,7 @@ import com.yahoo.vespa.defaults.Defaults; import com.yahoo.vespa.filedistribution.FileDistributionConnectionPool; import com.yahoo.vespa.filedistribution.FileDownloader; import com.yahoo.vespa.filedistribution.FileReferenceDownload; -import com.yahoo.vespa.flags.FlagSource; + import java.io.File; import java.time.Duration; import java.util.List; @@ -24,6 +24,7 @@ import java.util.Optional; import java.util.logging.Logger; import static com.yahoo.vespa.config.server.filedistribution.FileDistributionUtil.fileReferenceExistsOnDisk; +import static com.yahoo.vespa.config.server.filedistribution.FileDistributionUtil.getOtherConfigServersInCluster; /** * Verifies that all active sessions has an application package on local disk. @@ -37,20 +38,14 @@ public class ApplicationPackageMaintainer extends ConfigServerMaintainer { private static final Logger log = Logger.getLogger(ApplicationPackageMaintainer.class.getName()); - private final ApplicationRepository applicationRepository; private final File downloadDirectory; private final Supervisor supervisor = new Supervisor(new Transport("filedistribution-pool")).setDropEmptyBuffers(true); private final FileDownloader fileDownloader; - ApplicationPackageMaintainer(ApplicationRepository applicationRepository, - Curator curator, - Duration interval, - FlagSource flagSource, - List<String> otherConfigServersInCluster) { - super(applicationRepository, curator, flagSource, applicationRepository.clock(), interval, false); - this.applicationRepository = applicationRepository; + ApplicationPackageMaintainer(ApplicationRepository applicationRepository, Curator curator, Duration interval) { + super(applicationRepository, curator, applicationRepository.flagSource(), applicationRepository.clock(), interval, false); this.downloadDirectory = new File(Defaults.getDefaults().underVespaHome(applicationRepository.configserverConfig().fileReferencesDir())); - this.fileDownloader = createFileDownloader(otherConfigServersInCluster, downloadDirectory, supervisor); + this.fileDownloader = createFileDownloader(applicationRepository, downloadDirectory, supervisor); } @Override @@ -91,9 +86,10 @@ public class ApplicationPackageMaintainer extends ConfigServerMaintainer { return asSuccessFactorDeviation(attempts, failures); } - private static FileDownloader createFileDownloader(List<String> otherConfigServersInCluster, + private static FileDownloader createFileDownloader(ApplicationRepository applicationRepository, File downloadDirectory, Supervisor supervisor) { + List<String> otherConfigServersInCluster = getOtherConfigServersInCluster(applicationRepository.configserverConfig()); ConfigSourceSet configSourceSet = new ConfigSourceSet(otherConfigServersInCluster); ConnectionPool connectionPool = new FileDistributionConnectionPool(configSourceSet, supervisor); return new FileDownloader(connectionPool, supervisor, downloadDirectory, Duration.ofSeconds(300)); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ConfigServerMaintenance.java b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ConfigServerMaintenance.java index a3e774feec4..dbd30f72c24 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ConfigServerMaintenance.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ConfigServerMaintenance.java @@ -6,7 +6,7 @@ import com.yahoo.vespa.config.server.ApplicationRepository; import com.yahoo.vespa.config.server.application.ConfigConvergenceChecker; import com.yahoo.vespa.config.server.filedistribution.FileDirectory; import com.yahoo.vespa.curator.Curator; -import com.yahoo.vespa.flags.FlagSource; + import java.time.Clock; import java.time.Duration; import java.util.List; @@ -26,7 +26,6 @@ public class ConfigServerMaintenance { private final List<Maintainer> maintainers = new CopyOnWriteArrayList<>(); private final ApplicationRepository applicationRepository; private final Curator curator; - private final FlagSource flagSource; private final ConfigConvergenceChecker convergenceChecker; private final FileDirectory fileDirectory; private final Duration interval; @@ -34,28 +33,21 @@ public class ConfigServerMaintenance { public ConfigServerMaintenance(ApplicationRepository applicationRepository, FileDirectory fileDirectory) { this.applicationRepository = applicationRepository; this.curator = applicationRepository.tenantRepository().getCurator(); - this.flagSource = applicationRepository.flagSource(); this.convergenceChecker = applicationRepository.configConvergenceChecker(); this.fileDirectory = fileDirectory; this.interval = Duration.ofMinutes(applicationRepository.configserverConfig().maintainerIntervalMinutes()); } public void startBeforeBootstrap() { - List<String> otherConfigServersInCluster = getOtherConfigServersInCluster(applicationRepository.configserverConfig()); - if ( ! otherConfigServersInCluster.isEmpty()) - maintainers.add(new ApplicationPackageMaintainer(applicationRepository, curator, Duration.ofSeconds(30), - flagSource, otherConfigServersInCluster)); - maintainers.add(new TenantsMaintainer(applicationRepository, curator, flagSource, interval, Clock.systemUTC())); + if (moreThanOneConfigServer()) + maintainers.add(new ApplicationPackageMaintainer(applicationRepository, curator, Duration.ofSeconds(15))); + maintainers.add(new TenantsMaintainer(applicationRepository, curator, interval, Clock.systemUTC())); } public void startAfterBootstrap() { - maintainers.add(new FileDistributionMaintainer(applicationRepository, - curator, - interval, - flagSource, - fileDirectory)); - maintainers.add(new SessionsMaintainer(applicationRepository, curator, Duration.ofSeconds(30), flagSource)); - maintainers.add(new ReindexingMaintainer(applicationRepository, curator, flagSource, + maintainers.add(new FileDistributionMaintainer(applicationRepository, curator, interval, fileDirectory)); + maintainers.add(new SessionsMaintainer(applicationRepository, curator, Duration.ofSeconds(30))); + maintainers.add(new ReindexingMaintainer(applicationRepository, curator, Duration.ofMinutes(3), convergenceChecker, Clock.systemUTC())); } @@ -66,4 +58,8 @@ public class ConfigServerMaintenance { public List<Maintainer> maintainers() { return List.copyOf(maintainers); } + private boolean moreThanOneConfigServer() { + return ! getOtherConfigServersInCluster(applicationRepository.configserverConfig()).isEmpty(); + } + } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/FileDistributionMaintainer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/FileDistributionMaintainer.java index 91721bbf409..4a0221fdc2c 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/FileDistributionMaintainer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/FileDistributionMaintainer.java @@ -5,7 +5,7 @@ import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.vespa.config.server.ApplicationRepository; import com.yahoo.vespa.config.server.filedistribution.FileDirectory; import com.yahoo.vespa.curator.Curator; -import com.yahoo.vespa.flags.FlagSource; + import java.time.Duration; /** @@ -24,9 +24,8 @@ public class FileDistributionMaintainer extends ConfigServerMaintainer { FileDistributionMaintainer(ApplicationRepository applicationRepository, Curator curator, Duration interval, - FlagSource flagSource, FileDirectory fileDirectory) { - super(applicationRepository, curator, flagSource, applicationRepository.clock(), interval, false); + super(applicationRepository, curator, applicationRepository.flagSource(), applicationRepository.clock(), interval, false); ConfigserverConfig configserverConfig = applicationRepository.configserverConfig(); this.maxUnusedFileReferenceAge = Duration.ofMinutes(configserverConfig.keepUnusedFileReferencesMinutes()); this.fileDirectory = fileDirectory; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ReindexingMaintainer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ReindexingMaintainer.java index decf658f6ee..8171d63ae37 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ReindexingMaintainer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ReindexingMaintainer.java @@ -10,7 +10,6 @@ import com.yahoo.vespa.config.server.application.ApplicationReindexing.Cluster; import com.yahoo.vespa.config.server.application.ConfigConvergenceChecker; import com.yahoo.vespa.config.server.tenant.Tenant; import com.yahoo.vespa.curator.Curator; -import com.yahoo.vespa.flags.FlagSource; import com.yahoo.yolean.Exceptions; import java.time.Clock; @@ -46,9 +45,9 @@ public class ReindexingMaintainer extends ConfigServerMaintainer { private final ConfigConvergenceChecker convergence; private final Clock clock; - public ReindexingMaintainer(ApplicationRepository applicationRepository, Curator curator, FlagSource flagSource, + public ReindexingMaintainer(ApplicationRepository applicationRepository, Curator curator, Duration interval, ConfigConvergenceChecker convergence, Clock clock) { - super(applicationRepository, curator, flagSource, clock, interval, true); + super(applicationRepository, curator, applicationRepository.flagSource(), clock, interval, true); this.convergence = convergence; this.clock = clock; } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/SessionsMaintainer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/SessionsMaintainer.java index 4c27913251a..844b667fd85 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/SessionsMaintainer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/SessionsMaintainer.java @@ -3,7 +3,6 @@ package com.yahoo.vespa.config.server.maintenance; import com.yahoo.vespa.config.server.ApplicationRepository; import com.yahoo.vespa.curator.Curator; -import com.yahoo.vespa.flags.FlagSource; import java.time.Duration; import java.util.logging.Level; @@ -17,8 +16,8 @@ import java.util.logging.Level; */ public class SessionsMaintainer extends ConfigServerMaintainer { - SessionsMaintainer(ApplicationRepository applicationRepository, Curator curator, Duration interval, FlagSource flagSource) { - super(applicationRepository, curator, flagSource, applicationRepository.clock(), interval, true); + SessionsMaintainer(ApplicationRepository applicationRepository, Curator curator, Duration interval) { + super(applicationRepository, curator, applicationRepository.flagSource(), applicationRepository.clock(), interval, true); } @Override diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/TenantsMaintainer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/TenantsMaintainer.java index 4ce0546fa2d..3fcdc8878d2 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/TenantsMaintainer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/TenantsMaintainer.java @@ -23,9 +23,8 @@ public class TenantsMaintainer extends ConfigServerMaintainer { private final Duration ttlForUnusedTenant; private final Clock clock; - TenantsMaintainer(ApplicationRepository applicationRepository, Curator curator, FlagSource flagSource, - Duration interval, Clock clock) { - super(applicationRepository, curator, flagSource, applicationRepository.clock(), interval, true); + TenantsMaintainer(ApplicationRepository applicationRepository, Curator curator, Duration interval, Clock clock) { + super(applicationRepository, curator, applicationRepository.flagSource(), applicationRepository.clock(), interval, true); this.ttlForUnusedTenant = defaultTtlForUnusedTenant; this.clock = clock; } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/MockProvisioner.java b/configserver/src/test/java/com/yahoo/vespa/config/server/MockProvisioner.java index 256545313cd..d93ee19085a 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/MockProvisioner.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/MockProvisioner.java @@ -9,7 +9,7 @@ import com.yahoo.config.provision.Capacity; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.HostFilter; import com.yahoo.config.provision.HostSpec; -import com.yahoo.config.provision.ProvisionLock; +import com.yahoo.config.provision.ApplicationMutex; import com.yahoo.config.provision.ProvisionLogger; import com.yahoo.config.provision.Provisioner; import com.yahoo.config.provision.exception.LoadBalancerServiceException; @@ -59,8 +59,8 @@ public class MockProvisioner implements Provisioner { } @Override - public ProvisionLock lock(ApplicationId application) { - return new ProvisionLock(application, () -> {}); + public ApplicationMutex lock(ApplicationId application) { + return new ApplicationMutex(application, () -> {}); } } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/maintenance/TenantsMaintainerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/maintenance/TenantsMaintainerTest.java index 5fd23a95eed..42c22977f79 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/maintenance/TenantsMaintainerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/maintenance/TenantsMaintainerTest.java @@ -9,7 +9,6 @@ import com.yahoo.test.ManualClock; import com.yahoo.vespa.config.server.ApplicationRepository; import com.yahoo.vespa.config.server.session.PrepareParams; import com.yahoo.vespa.config.server.tenant.TenantRepository; -import com.yahoo.vespa.flags.InMemoryFlagSource; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; @@ -46,7 +45,7 @@ public class TenantsMaintainerTest { assertNotNull(tenantRepository.getTenant(shouldNotBeDeleted)); clock.advance(TenantsMaintainer.defaultTtlForUnusedTenant.plus(Duration.ofDays(1))); - new TenantsMaintainer(applicationRepository, tester.curator(), new InMemoryFlagSource(), Duration.ofDays(1), clock).run(); + new TenantsMaintainer(applicationRepository, tester.curator(), Duration.ofDays(1), clock).run(); // One tenant should now have been deleted assertNull(tenantRepository.getTenant(shouldBeDeleted)); diff --git a/container-core/abi-spec.json b/container-core/abi-spec.json index 2b5e4386e94..f29c4c2a8f7 100644 --- a/container-core/abi-spec.json +++ b/container-core/abi-spec.json @@ -1261,10 +1261,13 @@ "public com.yahoo.jdisc.http.ConnectorConfig$ServerName$Builder fallback(java.lang.String)", "public com.yahoo.jdisc.http.ConnectorConfig$ServerName$Builder allowed(java.lang.String)", "public com.yahoo.jdisc.http.ConnectorConfig$ServerName$Builder allowed(java.util.Collection)", + "public com.yahoo.jdisc.http.ConnectorConfig$ServerName$Builder known(java.lang.String)", + "public com.yahoo.jdisc.http.ConnectorConfig$ServerName$Builder known(java.util.Collection)", "public com.yahoo.jdisc.http.ConnectorConfig$ServerName build()" ], "fields" : [ - "public java.util.List allowed" + "public java.util.List allowed", + "public java.util.List known" ] }, "com.yahoo.jdisc.http.ConnectorConfig$ServerName" : { @@ -1278,7 +1281,9 @@ "public void <init>(com.yahoo.jdisc.http.ConnectorConfig$ServerName$Builder)", "public java.lang.String fallback()", "public java.util.List allowed()", - "public java.lang.String allowed(int)" + "public java.lang.String allowed(int)", + "public java.util.List known()", + "public java.lang.String known(int)" ], "fields" : [ ] }, diff --git a/container-core/src/main/java/com/yahoo/container/handler/Coverage.java b/container-core/src/main/java/com/yahoo/container/handler/Coverage.java index 69c6c9681bc..bbb47d14571 100644 --- a/container-core/src/main/java/com/yahoo/container/handler/Coverage.java +++ b/container-core/src/main/java/com/yahoo/container/handler/Coverage.java @@ -110,7 +110,7 @@ public class Coverage { return switch (fullReason) { case EXPLICITLY_FULL: yield true; case EXPLICITLY_INCOMPLETE: yield false; - case DOCUMENT_COUNT: yield docs == active; + case DOCUMENT_COUNT: yield (docs == active) && !((active == 0) && isDegradedByTimeout()) ; }; } @@ -163,6 +163,9 @@ public class Coverage { if (docs < total) { return (int) Math.round(docs * 100.0d / total); } + if ((total == 0) && isDegradedByTimeout()) { + return 0; + } return getFullResultSets() * 100 / getResultSets(); } diff --git a/container-core/src/main/java/com/yahoo/container/jdisc/state/StateHandler.java b/container-core/src/main/java/com/yahoo/container/jdisc/state/StateHandler.java index da730dc9acc..045b13e5d63 100644 --- a/container-core/src/main/java/com/yahoo/container/jdisc/state/StateHandler.java +++ b/container-core/src/main/java/com/yahoo/container/jdisc/state/StateHandler.java @@ -131,12 +131,12 @@ public class StateHandler extends AbstractRequestHandler implements CapabilityRe try { String suffix = resolvePath(requestUri); return switch (suffix) { - case "" -> ByteBuffer.wrap(apiLinks(requestUri)); - case CONFIG_GENERATION_PATH -> ByteBuffer.wrap(toPrettyString(config)); - case HISTOGRAMS_PATH -> ByteBuffer.wrap(buildHistogramsOutput()); - case HEALTH_PATH, METRICS_PATH -> ByteBuffer.wrap(buildMetricOutput(suffix)); - case VERSION_PATH -> ByteBuffer.wrap(buildVersionOutput()); - default -> ByteBuffer.wrap(buildMetricOutput(suffix)); // XXX should possibly do something else here + case "" -> ByteBuffer.wrap(apiLinks(requestUri)); + case CONFIG_GENERATION_PATH -> ByteBuffer.wrap(toPrettyString(config)); + case HISTOGRAMS_PATH -> ByteBuffer.wrap(buildHistogramsOutput()); + case HEALTH_PATH, METRICS_PATH -> ByteBuffer.wrap(buildMetricOutput(suffix)); + case VERSION_PATH -> ByteBuffer.wrap(buildVersionOutput()); + default -> ByteBuffer.wrap(buildMetricOutput(suffix)); // XXX should possibly do something else here }; } catch (JsonProcessingException e) { throw new RuntimeException("Bad JSON construction", e); diff --git a/container-core/src/main/java/com/yahoo/container/logging/Coverage.java b/container-core/src/main/java/com/yahoo/container/logging/Coverage.java index ee85711852d..397ff35b54c 100644 --- a/container-core/src/main/java/com/yahoo/container/logging/Coverage.java +++ b/container-core/src/main/java/com/yahoo/container/logging/Coverage.java @@ -59,6 +59,9 @@ public class Coverage { if (docs < total) { return (int) Math.round(docs * 100.0d / total); } + if ((total == 0) && isDegradedByTimeout()) { + return 0; + } return 100; } diff --git a/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/JDiscServerConnector.java b/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/JDiscServerConnector.java index e699b9f200c..983adec034d 100644 --- a/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/JDiscServerConnector.java +++ b/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/JDiscServerConnector.java @@ -11,8 +11,8 @@ import org.eclipse.jetty.server.Server; import org.eclipse.jetty.server.ServerConnector; import java.util.HashMap; +import java.util.List; import java.util.Map; -import java.util.Optional; /** * @author bjorncs @@ -26,6 +26,7 @@ class JDiscServerConnector extends ServerConnector { private final Metric metric; private final String connectorName; private final int listenPort; + private final List<String> knownServerNames; JDiscServerConnector(ConnectorConfig config, Metric metric, Server server, JettyConnectionLogger connectionLogger, ConnectionMetricAggregator connectionMetricAggregator, ConnectionFactory... factories) { @@ -35,6 +36,7 @@ class JDiscServerConnector extends ServerConnector { this.connectorName = config.name(); this.listenPort = config.listenPort(); this.metricCtx = metric.createContext(createConnectorDimensions(listenPort, connectorName, 0)); + this.knownServerNames = List.copyOf(config.serverName().known()); this.statistics = new ConnectionStatistics(); setAcceptedTcpNoDelay(config.tcpNoDelay()); @@ -69,7 +71,7 @@ class JDiscServerConnector extends ServerConnector { dimensions.put(MetricDefinitions.SCHEME_DIMENSION, scheme); dimensions.put(MetricDefinitions.CLIENT_AUTHENTICATED_DIMENSION, Boolean.toString(clientAuthenticated)); dimensions.put(MetricDefinitions.PROTOCOL_DIMENSION, request.getProtocol()); - String serverName = Optional.ofNullable(request.getServerName()).orElse("unknown"); + String serverName = knownServerNames.stream().filter(name -> name.equalsIgnoreCase(request.getServerName())).findFirst().orElse("unknown"); dimensions.put(MetricDefinitions.REQUEST_SERVER_NAME_DIMENSION, serverName); dimensions.putAll(extraDimensions); return metric.createContext(dimensions); diff --git a/container-core/src/main/java/com/yahoo/restapi/SlimeJsonResponse.java b/container-core/src/main/java/com/yahoo/restapi/SlimeJsonResponse.java index 252fc99a273..d6720a8797e 100644 --- a/container-core/src/main/java/com/yahoo/restapi/SlimeJsonResponse.java +++ b/container-core/src/main/java/com/yahoo/restapi/SlimeJsonResponse.java @@ -16,24 +16,27 @@ import java.io.OutputStream; public class SlimeJsonResponse extends HttpResponse { protected final Slime slime; + private final boolean compact; public SlimeJsonResponse() { this(new Slime()); } - public SlimeJsonResponse(Slime slime) { - super(200); - this.slime = slime; - } + public SlimeJsonResponse(Slime slime) { this(200, slime, true); } + + public SlimeJsonResponse(Slime slime, boolean compact) { this(200, slime, compact); } + + public SlimeJsonResponse(int statusCode, Slime slime) { this(statusCode, slime, true); } - public SlimeJsonResponse(int statusCode, Slime slime) { + public SlimeJsonResponse(int statusCode, Slime slime, boolean compact) { super(statusCode); this.slime = slime; + this.compact = compact; } @Override public void render(OutputStream stream) throws IOException { - new JsonFormat(true).encode(stream, slime); + new JsonFormat(compact).encode(stream, slime); } @Override diff --git a/container-core/src/main/resources/configdefinitions/jdisc.http.jdisc.http.connector.def b/container-core/src/main/resources/configdefinitions/jdisc.http.jdisc.http.connector.def index c1c0944d7eb..b4a513e0de8 100644 --- a/container-core/src/main/resources/configdefinitions/jdisc.http.jdisc.http.connector.def +++ b/container-core/src/main/resources/configdefinitions/jdisc.http.jdisc.http.connector.def @@ -139,6 +139,9 @@ serverName.fallback string default="" # The list of accepted server names. Empty list to accept any. Elements follows format of 'serverName.default'. serverName.allowed[] string +# The list of known server names. Used for e.g matching metric dimensions. +serverName.known[] string + # HTTP request headers that contain remote address accessLog.remoteAddressHeaders[] string diff --git a/container-core/src/test/java/com/yahoo/container/logging/JSONLogTestCase.java b/container-core/src/test/java/com/yahoo/container/logging/JSONLogTestCase.java index 8887bc720ed..7132a0a5beb 100644 --- a/container-core/src/test/java/com/yahoo/container/logging/JSONLogTestCase.java +++ b/container-core/src/test/java/com/yahoo/container/logging/JSONLogTestCase.java @@ -291,6 +291,8 @@ public class JSONLogTestCase { newRequestLogEntry("test", new Coverage(100, 200, 200, 2)).build()); verifyCoverage("\"coverage\":{\"coverage\":50,\"documents\":100,\"degraded\":{\"adaptive-timeout\":true}}", newRequestLogEntry("test", new Coverage(100, 200, 200, 4)).build()); + verifyCoverage("\"coverage\":{\"coverage\":0,\"documents\":0,\"degraded\":{\"timeout\":true}}", + newRequestLogEntry("test", new Coverage(0, 0, 0, 2)).build()); } private String formatEntry(RequestLogEntry entry) { diff --git a/container-disc/src/main/java/com/yahoo/container/jdisc/DataplaneProxyService.java b/container-disc/src/main/java/com/yahoo/container/jdisc/DataplaneProxyService.java index 4c5ac951384..d94244b0e47 100644 --- a/container-disc/src/main/java/com/yahoo/container/jdisc/DataplaneProxyService.java +++ b/container-disc/src/main/java/com/yahoo/container/jdisc/DataplaneProxyService.java @@ -109,6 +109,10 @@ public class DataplaneProxyService extends AbstractComponent { config.tokenPort(), config.tokenEndpoints(), root)); + if (configChanged) { + logger.log(Level.INFO, "Configuring data plane proxy service. Token endpoints: [%s]" + .formatted(String.join(", ", config.tokenEndpoints()))); + } if (configChanged && state == NginxState.RUNNING) { changeState(NginxState.RELOAD_REQUIRED); } diff --git a/container-search/abi-spec.json b/container-search/abi-spec.json index ffae12867fd..31b4dd2c920 100644 --- a/container-search/abi-spec.json +++ b/container-search/abi-spec.json @@ -5434,6 +5434,7 @@ "public void setListFeatures(boolean)", "public boolean getListFeatures()", "public com.yahoo.search.query.ranking.MatchPhase getMatchPhase()", + "public com.yahoo.search.query.ranking.GlobalPhase getGlobalPhase()", "public com.yahoo.search.query.ranking.Matching getMatching()", "public com.yahoo.search.query.ranking.SoftTimeout getSoftTimeout()", "public com.yahoo.search.query.Sorting getSorting()", @@ -5461,6 +5462,7 @@ "public static final java.lang.String KEEPRANKCOUNT", "public static final java.lang.String RANKSCOREDROPLIMIT", "public static final java.lang.String MATCH_PHASE", + "public static final java.lang.String GLOBAL_PHASE", "public static final java.lang.String DIVERSITY", "public static final java.lang.String SOFTTIMEOUT", "public static final java.lang.String MATCHING", @@ -6934,6 +6936,26 @@ "public static final java.lang.String STRATEGY" ] }, + "com.yahoo.search.query.ranking.GlobalPhase" : { + "superClass" : "java.lang.Object", + "interfaces" : [ + "java.lang.Cloneable" + ], + "attributes" : [ + "public" + ], + "methods" : [ + "public void <init>()", + "public static com.yahoo.search.query.profile.types.QueryProfileType getArgumentType()", + "public void setRerankCount(int)", + "public java.lang.Integer getRerankCount()", + "public int hashCode()", + "public boolean equals(java.lang.Object)", + "public com.yahoo.search.query.ranking.GlobalPhase clone()", + "public bridge synthetic java.lang.Object clone()" + ], + "fields" : [ ] + }, "com.yahoo.search.query.ranking.MatchPhase" : { "superClass" : "java.lang.Object", "interfaces" : [ @@ -7054,6 +7076,7 @@ "public void put(java.lang.String, java.lang.String)", "public void put(java.lang.String, java.lang.Object)", "public java.util.List get(java.lang.String)", + "public java.util.Optional getAsTensor(java.lang.String)", "public void remove(java.lang.String)", "public boolean isEmpty()", "public java.util.Map asMap()", diff --git a/container-search/pom.xml b/container-search/pom.xml index 1e40539a79e..5e7c60d49c3 100644 --- a/container-search/pom.xml +++ b/container-search/pom.xml @@ -75,6 +75,18 @@ <version>${project.version}</version> <scope>provided</scope> </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>model-integration</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>container-onnxruntime</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> <dependency> <groupId>xerces</groupId> diff --git a/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java b/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java index f0222eea618..12f8cdf9852 100644 --- a/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java @@ -244,13 +244,24 @@ public class ClusterSearcher extends Searcher { throw new IllegalStateException("perSchemaSearch must always be called with 1 schema, got: " + restrict.size()); } String schema = restrict.iterator().next(); - boolean useGlobalPhase = globalPhaseRanker != null; + int rerankCount = globalPhaseRanker != null ? globalPhaseRanker.getRerankCount(query, schema) : 0; + boolean useGlobalPhase = rerankCount > 0; + final int wantOffset = query.getOffset(); + final int wantHits = query.getHits(); if (useGlobalPhase) { var error = globalPhaseRanker.validateNoSorting(query, schema).orElse(null); if (error != null) return new Result(query, error); + int useHits = Math.max(wantOffset + wantHits, rerankCount); + query.setOffset(0); + query.setHits(useHits); } Result result = searcher.search(query, execution); - if (useGlobalPhase) globalPhaseRanker.rerankHits(query, result, schema); + if (useGlobalPhase) { + globalPhaseRanker.rerankHits(query, result, schema); + result.hits().trim(wantOffset, wantHits); + query.setOffset(wantOffset); + query.setHits(wantHits); + } return result; } diff --git a/container-search/src/main/java/com/yahoo/prelude/query/BoolItem.java b/container-search/src/main/java/com/yahoo/prelude/query/BoolItem.java index f6170048158..b6b84c4b276 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/BoolItem.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/BoolItem.java @@ -7,6 +7,8 @@ import java.nio.ByteBuffer; /** * A true/false term suitable for searching bool indexes. + * + * @author bratseth */ public class BoolItem extends TermItem { @@ -58,11 +60,11 @@ public class BoolItem extends TermItem { } private boolean toBoolean(String stringValue) { - switch (stringValue.toLowerCase()) { - case "true" : return true; - case "false" : return false; - default: throw new IllegalInputException("Expected 'true' or 'false', got '" + stringValue + "'"); - } + return switch (stringValue.toLowerCase()) { + case "true" -> true; + case "false" -> false; + default -> throw new IllegalInputException("Expected 'true' or 'false', got '" + stringValue + "'"); + }; } /** Returns the same as stringValue */ diff --git a/container-search/src/main/java/com/yahoo/search/Result.java b/container-search/src/main/java/com/yahoo/search/Result.java index 4962466c752..b1a0107c6d8 100644 --- a/container-search/src/main/java/com/yahoo/search/Result.java +++ b/container-search/src/main/java/com/yahoo/search/Result.java @@ -70,8 +70,8 @@ public final class Result extends com.yahoo.processing.Response implements Clone */ public Result(Query query, HitGroup hits) { super(query); - if (query==null) throw new NullPointerException("The query reference in a result cannot be null"); - this.hits=hits; + if (query == null) throw new NullPointerException("The query reference in a result cannot be null"); + this.hits = hits; hits.setQuery(query); if (query.getRanking().getSorting() != null) { setHitOrderer(new HitSortOrderer(query.getRanking().getSorting())); diff --git a/container-search/src/main/java/com/yahoo/search/query/Ranking.java b/container-search/src/main/java/com/yahoo/search/query/Ranking.java index b07b440ac62..3c2a8a83c40 100644 --- a/container-search/src/main/java/com/yahoo/search/query/Ranking.java +++ b/container-search/src/main/java/com/yahoo/search/query/Ranking.java @@ -9,6 +9,7 @@ import com.yahoo.search.query.profile.types.FieldDescription; import com.yahoo.search.query.profile.types.QueryProfileFieldType; import com.yahoo.search.query.profile.types.QueryProfileType; import com.yahoo.search.query.ranking.Diversity; +import com.yahoo.search.query.ranking.GlobalPhase; import com.yahoo.search.query.ranking.MatchPhase; import com.yahoo.search.query.ranking.Matching; import com.yahoo.search.query.ranking.RankFeatures; @@ -44,6 +45,7 @@ public class Ranking implements Cloneable { public static final String KEEPRANKCOUNT = "keepRankCount"; public static final String RANKSCOREDROPLIMIT = "rankScoreDropLimit"; public static final String MATCH_PHASE = "matchPhase"; + public static final String GLOBAL_PHASE = "globalPhase"; public static final String DIVERSITY = "diversity"; public static final String SOFTTIMEOUT = "softtimeout"; public static final String MATCHING = "matching"; @@ -65,6 +67,7 @@ public class Ranking implements Cloneable { argumentType.addField(new FieldDescription(RERANKCOUNT, "integer")); argumentType.addField(new FieldDescription(KEEPRANKCOUNT, "integer")); argumentType.addField(new FieldDescription(RANKSCOREDROPLIMIT, "double")); + argumentType.addField(new FieldDescription(GLOBAL_PHASE, new QueryProfileFieldType(GlobalPhase.getArgumentType()))); argumentType.addField(new FieldDescription(MATCH_PHASE, new QueryProfileFieldType(MatchPhase.getArgumentType()), "matchPhase")); argumentType.addField(new FieldDescription(DIVERSITY, new QueryProfileFieldType(Diversity.getArgumentType()))); argumentType.addField(new FieldDescription(SOFTTIMEOUT, new QueryProfileFieldType(SoftTimeout.getArgumentType()))); @@ -104,6 +107,8 @@ public class Ranking implements Cloneable { private MatchPhase matchPhase = new MatchPhase(); + private GlobalPhase globalPhase = new GlobalPhase(); + private Matching matching = new Matching(); private SoftTimeout softTimeout = new SoftTimeout(); @@ -215,6 +220,9 @@ public class Ranking implements Cloneable { /** Returns the match phase rank settings of this. This is never null. */ public MatchPhase getMatchPhase() { return matchPhase; } + /** Returns the global-phase rank settings of this. This is never null. */ + public GlobalPhase getGlobalPhase() { return globalPhase; } + /** Returns the matching settings of this. This is never null. */ public Matching getMatching() { return matching; } @@ -279,6 +287,7 @@ public class Ranking implements Cloneable { clone.rankProperties = this.rankProperties.clone(); clone.rankFeatures = this.rankFeatures.cloneFor(clone); clone.matchPhase = this.matchPhase.clone(); + clone.globalPhase = this.globalPhase.clone(); clone.matching = this.matching.clone(); clone.softTimeout = this.softTimeout.clone(); return clone; @@ -305,12 +314,13 @@ public class Ranking implements Cloneable { if ( ! QueryHelper.equals(this.sorting, other.sorting)) return false; if ( ! QueryHelper.equals(this.location, other.location)) return false; if ( ! QueryHelper.equals(this.profile, other.profile)) return false; + if ( ! QueryHelper.equals(this.globalPhase, other.globalPhase)) return false; return true; } @Override public int hashCode() { - return Objects.hash(rankFeatures, rankProperties, matchPhase, softTimeout, matching, sorting, location, profile); + return Objects.hash(rankFeatures, rankProperties, matchPhase, globalPhase, softTimeout, matching, sorting, location, profile); } } diff --git a/container-search/src/main/java/com/yahoo/search/query/SelectParser.java b/container-search/src/main/java/com/yahoo/search/query/SelectParser.java index 4c4eef38d39..93df7fdfb18 100644 --- a/container-search/src/main/java/com/yahoo/search/query/SelectParser.java +++ b/container-search/src/main/java/com/yahoo/search/query/SelectParser.java @@ -992,7 +992,7 @@ public class SelectParser implements Parser { if (origin != null) { out.setOrigin(origin); } - if (annotations != null){ + if (annotations != null) { Boolean usePositionData = getBoolAnnotation(USE_POSITION_DATA, annotations, null); if (usePositionData != null) { out.setPositionData(usePositionData); diff --git a/container-search/src/main/java/com/yahoo/search/query/properties/QueryProperties.java b/container-search/src/main/java/com/yahoo/search/query/properties/QueryProperties.java index f476ee1afc5..505759d8967 100644 --- a/container-search/src/main/java/com/yahoo/search/query/properties/QueryProperties.java +++ b/container-search/src/main/java/com/yahoo/search/query/properties/QueryProperties.java @@ -62,7 +62,8 @@ public class QueryProperties extends Properties { map.put(CompoundName.fromComponents(Ranking.RANKING, Ranking.MATCHING, last.toLowerCase()), accessor); } - private static final Map<CompoundName, GetterSetter> properyAccessors = createPropertySetterMap(); + private static final Map<CompoundName, GetterSetter> propertyAccessors = createPropertySetterMap(); + private static Map<CompoundName, GetterSetter> createPropertySetterMap() { Map<CompoundName, GetterSetter> map = new HashMap<>(); map.put(CompoundName.fromComponents(Model.MODEL, Model.QUERY_STRING), GetterSetter.of(query -> query.getModel().getQueryString(), (query, value) -> query.getModel().setQueryString(asString(value, "")))); @@ -84,7 +85,6 @@ public class QueryProperties extends Properties { map.put(CompoundName.fromComponents(Ranking.RANKING, Ranking.KEEPRANKCOUNT), GetterSetter.of(query -> query.getRanking().getKeepRankCount(), (query, value) -> query.getRanking().setKeepRankCount(asInteger(value, null)))); map.put(CompoundName.fromComponents(Ranking.RANKING, Ranking.RANKSCOREDROPLIMIT), GetterSetter.of(query -> query.getRanking().getRankScoreDropLimit(), (query, value) -> query.getRanking().setRankScoreDropLimit(asDouble(value, null)))); map.put(CompoundName.fromComponents(Ranking.RANKING, Ranking.LIST_FEATURES), GetterSetter.of(query -> query.getRanking().getListFeatures(), (query, value) -> query.getRanking().setListFeatures(asBoolean(value,false)))); - addDualCasedRM(map, Matching.TERMWISELIMIT, GetterSetter.of(query -> query.getRanking().getMatching().getTermwiseLimit(), (query, value) -> query.getRanking().getMatching().setTermwiselimit(asDouble(value, 1.0)))); addDualCasedRM(map, Matching.NUMTHREADSPERSEARCH, GetterSetter.of(query -> query.getRanking().getMatching().getNumThreadsPerSearch(), (query, value) -> query.getRanking().getMatching().setNumThreadsPerSearch(asInteger(value, 1)))); addDualCasedRM(map, Matching.NUMSEARCHPARTITIIONS, GetterSetter.of(query -> query.getRanking().getMatching().getNumSearchPartitions(), (query, value) -> query.getRanking().getMatching().setNumSearchPartitions(asInteger(value, 1)))); @@ -101,6 +101,9 @@ public class QueryProperties extends Properties { map.put(CompoundName.fromComponents(Ranking.RANKING, Ranking.MATCH_PHASE, Ranking.DIVERSITY, Diversity.MINGROUPS), GetterSetter.of(query -> query.getRanking().getMatchPhase().getDiversity().getMinGroups(), (query, value) -> query.getRanking().getMatchPhase().getDiversity().setMinGroups(asLong(value, null)))); map.put(CompoundName.fromComponents(Ranking.RANKING, Ranking.MATCH_PHASE, Ranking.DIVERSITY, Diversity.CUTOFF, Diversity.FACTOR), GetterSetter.of(query -> query.getRanking().getMatchPhase().getDiversity().getCutoffFactor(), (query, value) -> query.getRanking().getMatchPhase().getDiversity().setCutoffFactor(asDouble(value, 10.0)))); map.put(CompoundName.fromComponents(Ranking.RANKING, Ranking.MATCH_PHASE, Ranking.DIVERSITY, Diversity.CUTOFF, Diversity.STRATEGY), GetterSetter.of(query -> query.getRanking().getMatchPhase().getDiversity().getCutoffStrategy(), (query, value) -> query.getRanking().getMatchPhase().getDiversity().setCutoffStrategy(asString(value, "loose")))); + map.put(CompoundName.fromComponents(Ranking.RANKING, Ranking.GLOBAL_PHASE, Ranking.RERANKCOUNT), + GetterSetter.of(query -> query.getRanking().getGlobalPhase().getRerankCount(), + (query, value) -> query.getRanking().getGlobalPhase().setRerankCount(asInteger(value, null)))); map.put(CompoundName.fromComponents(Ranking.RANKING, Ranking.SOFTTIMEOUT, SoftTimeout.ENABLE), GetterSetter.of(query -> query.getRanking().getSoftTimeout().getEnable(), (query, value) -> query.getRanking().getSoftTimeout().setEnable(asBoolean(value, true)))); map.put(CompoundName.fromComponents(Ranking.RANKING, Ranking.SOFTTIMEOUT, SoftTimeout.FACTOR), GetterSetter.of(query -> query.getRanking().getSoftTimeout().getFactor(), (query, value) -> query.getRanking().getSoftTimeout().setFactor(asDouble(value, null)))); map.put(CompoundName.fromComponents(Ranking.RANKING, Ranking.SOFTTIMEOUT, SoftTimeout.TAILCOST), GetterSetter.of(query -> query.getRanking().getSoftTimeout().getTailcost(), (query, value) -> query.getRanking().getSoftTimeout().setTailcost(asDouble(value, null)))); @@ -148,7 +151,7 @@ public class QueryProperties extends Properties { public Object get(CompoundName key, Map<String, String> context, com.yahoo.processing.request.Properties substitution) { - GetterSetter propertyAccessor = properyAccessors.get(key); + GetterSetter propertyAccessor = propertyAccessors.get(key); if (propertyAccessor != null && propertyAccessor.getter != null) return propertyAccessor.getter.get(query); if (key.first().equals(Ranking.RANKING)) { @@ -164,7 +167,7 @@ public class QueryProperties extends Properties { } private void setInternal(CompoundName key, Object value, Map<String,String> context) { - GetterSetter propertyAccessor = properyAccessors.get(key); + GetterSetter propertyAccessor = propertyAccessors.get(key); if (propertyAccessor != null && propertyAccessor.setter != null) { propertyAccessor.setter.set(query, value); return; diff --git a/container-search/src/main/java/com/yahoo/search/query/properties/RankProfileInputProperties.java b/container-search/src/main/java/com/yahoo/search/query/properties/RankProfileInputProperties.java index 4e340d47016..c9f935e5f52 100644 --- a/container-search/src/main/java/com/yahoo/search/query/properties/RankProfileInputProperties.java +++ b/container-search/src/main/java/com/yahoo/search/query/properties/RankProfileInputProperties.java @@ -38,17 +38,17 @@ public class RankProfileInputProperties extends Properties { @Override public void set(CompoundName name, Object value, Map<String, String> context) { if (RankFeatures.isFeatureName(name.toString())) { - TensorType expectedType = typeOf(name); - if (expectedType != null) { - try { + try { + TensorType expectedType = typeOf(name); + if (expectedType != null) { value = tensorConverter.convertTo(expectedType, name.last(), value, query.getModel().getLanguage()); } - catch (IllegalArgumentException e) { - throw new IllegalInputException("Could not set '" + name + "' to '" + value + "'", e); - } + } + catch (IllegalArgumentException e) { + throw new IllegalInputException("Could not set '" + name + "' to '" + value + "'", e); } } super.set(name, value, context); diff --git a/container-search/src/main/java/com/yahoo/search/query/ranking/GlobalPhase.java b/container-search/src/main/java/com/yahoo/search/query/ranking/GlobalPhase.java new file mode 100644 index 00000000000..7d28ae85ad1 --- /dev/null +++ b/container-search/src/main/java/com/yahoo/search/query/ranking/GlobalPhase.java @@ -0,0 +1,68 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.query.ranking; + +import com.yahoo.search.query.Ranking; +import com.yahoo.search.query.profile.types.FieldDescription; +import com.yahoo.search.query.profile.types.FieldType; +import com.yahoo.search.query.profile.types.QueryProfileType; + +import java.util.Objects; + +/** + * The global-phase ranking settings of this query. + * + * @author arnej + */ +public class GlobalPhase implements Cloneable { + + /** The type representing the property arguments consumed by this */ + private static final QueryProfileType argumentType; + + static { + argumentType = new QueryProfileType(Ranking.GLOBAL_PHASE); + argumentType.setStrict(true); + argumentType.setBuiltin(true); + argumentType.addField(new FieldDescription(Ranking.RERANKCOUNT, FieldType.integerType)); + argumentType.freeze(); + } + public static QueryProfileType getArgumentType() { return argumentType; } + + private Integer rerankCount = null; + + /** + * Sets the number of hits for which the global-phase function will be evaluated. + * When set, this overrides the setting in the rank profile. + */ + public void setRerankCount(int rerankCount) { this.rerankCount = rerankCount; } + + /** Returns the rerank-count that will be used, or null if not set */ + public Integer getRerankCount() { return rerankCount; } + + @Override + public int hashCode() { + return Objects.hashCode(this.rerankCount); + } + + @Override + public boolean equals(Object o) { + if (o == this) return true; + if (o instanceof GlobalPhase other) { + if ( ! Objects.equals(this.rerankCount, other.rerankCount)) return false; + return true; + } + return false; + } + + @Override + public GlobalPhase clone() { + try { + GlobalPhase clone = (GlobalPhase)super.clone(); + clone.rerankCount = this.rerankCount; + return clone; + } + catch (CloneNotSupportedException e) { + throw new RuntimeException("Won't happen", e); + } + } + +} diff --git a/container-search/src/main/java/com/yahoo/search/query/ranking/RankProperties.java b/container-search/src/main/java/com/yahoo/search/query/ranking/RankProperties.java index 544f26a7d89..4ac5375807b 100644 --- a/container-search/src/main/java/com/yahoo/search/query/ranking/RankProperties.java +++ b/container-search/src/main/java/com/yahoo/search/query/ranking/RankProperties.java @@ -3,6 +3,7 @@ package com.yahoo.search.query.ranking; import com.yahoo.fs4.GetDocSumsPacket; import com.yahoo.fs4.MapEncoder; +import com.yahoo.tensor.Tensor; import com.yahoo.text.JSON; import java.nio.ByteBuffer; @@ -11,6 +12,7 @@ import java.util.Collections; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Optional; /** * Contains the properties of a query. @@ -61,6 +63,26 @@ public class RankProperties implements Cloneable { return Collections.unmodifiableList(stringValues); } + /** + * Returns a tensor (as moved from RankFeatures by prepare step) if present + * + * @throws IllegalArgumentException if the value is there but wrong type + */ + public Optional<Tensor> getAsTensor(String name) { + List<Object> values = properties.get(name); + if (values == null || values.isEmpty()) return Optional.empty(); + if (values.size() != 1) { + throw new IllegalArgumentException("unexpected multiple [" + values.size() + "] values for property '" + name + "'"); + } + Object feature = values.get(0); + if (feature == null) return Optional.empty(); + if (feature instanceof Tensor t) return Optional.of(t); + if (feature instanceof Double d) return Optional.of(Tensor.from(d)); + throw new IllegalArgumentException("Expected '" + name + "' to be a tensor or double, but it is '" + feature + + "', this usually means that '" + name + "' is not defined in the schema. " + + "See https://docs.vespa.ai/en/tensor-user-guide.html#querying-with-tensors"); + } + /** Removes all properties for a given name */ public void remove(String name) { properties.remove(name); diff --git a/container-search/src/main/java/com/yahoo/search/ranking/GlobalPhaseRanker.java b/container-search/src/main/java/com/yahoo/search/ranking/GlobalPhaseRanker.java index 6e30a81eebc..91acc883803 100644 --- a/container-search/src/main/java/com/yahoo/search/ranking/GlobalPhaseRanker.java +++ b/container-search/src/main/java/com/yahoo/search/ranking/GlobalPhaseRanker.java @@ -13,10 +13,7 @@ import com.yahoo.tensor.Tensor; import com.yahoo.data.access.helpers.MatchFeatureData; import com.yahoo.data.access.helpers.MatchFeatureFilter; -import java.util.ArrayList; -import java.util.Collection; -import java.util.List; -import java.util.Optional; +import java.util.*; import java.util.function.Supplier; import java.util.logging.Logger; @@ -31,6 +28,11 @@ public class GlobalPhaseRanker { logger.fine(() -> "Using factory: " + factory); } + public int getRerankCount(Query query, String schema) { + var setup = globalPhaseSetupFor(query, schema).orElse(null); + return resolveRerankCount(setup, query); + } + public Optional<ErrorMessage> validateNoSorting(Query query, String schema) { var setup = globalPhaseSetupFor(query, schema).orElse(null); if (setup == null) return Optional.empty(); @@ -45,16 +47,14 @@ public class GlobalPhaseRanker { return Optional.empty(); } - public void rerankHits(Query query, Result result, String schema) { - var setup = globalPhaseSetupFor(query, schema).orElse(null); - if (setup == null) return; + static void rerankHitsImpl(GlobalPhaseSetup setup, Query query, Result result) { var mainSpec = setup.globalPhaseEvalSpec; - var mainSrc = withQueryPrep(mainSpec.evalSource(), mainSpec.fromQuery(), query); - int rerankCount = setup.rerankCount; + var mainSrc = withQueryPrep(mainSpec.evalSource(), mainSpec.fromQuery(), setup.defaultValues, query); + int rerankCount = resolveRerankCount(setup, query); var normalizers = new ArrayList<NormalizerContext>(); for (var nSetup : setup.normalizers) { var normSpec = nSetup.inputEvalSpec(); - var normEvalSrc = withQueryPrep(normSpec.evalSource(), normSpec.fromQuery(), query); + var normEvalSrc = withQueryPrep(normSpec.evalSource(), normSpec.fromQuery(), setup.defaultValues, query); normalizers.add(new NormalizerContext(nSetup.name(), nSetup.supplier().get(), normEvalSrc, normSpec.fromMF())); } var rescorer = new HitRescorer(mainSrc, mainSpec.fromMF(), normalizers); @@ -63,8 +63,15 @@ public class GlobalPhaseRanker { hideImplicitMatchFeatures(result, setup.matchFeaturesToHide); } - static Supplier<Evaluator> withQueryPrep(Supplier<Evaluator> evalSource, List<String> queryFeatures, Query query) { - var prepared = PreparedInput.findFromQuery(query, queryFeatures); + public void rerankHits(Query query, Result result, String schema) { + var setup = globalPhaseSetupFor(query, schema); + if (setup.isPresent()) { + rerankHitsImpl(setup.get(), query, result); + } + } + + static Supplier<Evaluator> withQueryPrep(Supplier<Evaluator> evalSource, List<String> queryFeatures, Map<String, Tensor> defaultValues, Query query) { + var prepared = PreparedInput.findFromQuery(query, queryFeatures, defaultValues); Supplier<Evaluator> supplier = () -> { var evaluator = evalSource.get(); for (var entry : prepared) { @@ -75,7 +82,7 @@ public class GlobalPhaseRanker { return supplier; } - private void hideImplicitMatchFeatures(Result result, Collection<String> namesToHide) { + private static void hideImplicitMatchFeatures(Result result, Collection<String> namesToHide) { if (namesToHide.size() == 0) return; var filter = new MatchFeatureFilter(namesToHide); for (var iterator = result.hits().deepIterator(); iterator.hasNext();) { @@ -89,7 +96,7 @@ public class GlobalPhaseRanker { if (newValue.fieldCount() == 0) { hit.removeField("matchfeatures"); } else { - hit.setField("matchfeatures", newValue); + hit.setField("matchfeatures", new FeatureData(newValue)); } } } @@ -101,4 +108,15 @@ public class GlobalPhaseRanker { .flatMap(evaluator -> evaluator.getGlobalPhaseSetup(query.getRanking().getProfile())); } + private static int resolveRerankCount(GlobalPhaseSetup setup, Query query) { + if (setup == null) { + // there is no global-phase at all (ignore override) + return 0; + } + Integer override = query.getRanking().getGlobalPhase().getRerankCount(); + if (override != null) { + return override; + } + return setup.rerankCount; + } } diff --git a/container-search/src/main/java/com/yahoo/search/ranking/GlobalPhaseSetup.java b/container-search/src/main/java/com/yahoo/search/ranking/GlobalPhaseSetup.java index 31a676e4c8e..7340e9e2a5e 100644 --- a/container-search/src/main/java/com/yahoo/search/ranking/GlobalPhaseSetup.java +++ b/container-search/src/main/java/com/yahoo/search/ranking/GlobalPhaseSetup.java @@ -3,15 +3,11 @@ package com.yahoo.search.ranking; import ai.vespa.models.evaluation.FunctionEvaluator; +import com.yahoo.tensor.Tensor; +import com.yahoo.tensor.TensorType; import com.yahoo.vespa.config.search.RankProfilesConfig; -import java.util.ArrayList; -import java.util.Collection; -import java.util.HashSet; -import java.util.List; -import java.util.Set; -import java.util.Map; -import java.util.HashMap; +import java.util.*; import java.util.function.Supplier; class GlobalPhaseSetup { @@ -20,16 +16,84 @@ class GlobalPhaseSetup { final int rerankCount; final Collection<String> matchFeaturesToHide; final List<NormalizerSetup> normalizers; + final Map<String, Tensor> defaultValues; GlobalPhaseSetup(FunEvalSpec globalPhaseEvalSpec, final int rerankCount, Collection<String> matchFeaturesToHide, - List<NormalizerSetup> normalizers) + List<NormalizerSetup> normalizers, + Map<String, Tensor> defaultValues) { this.globalPhaseEvalSpec = globalPhaseEvalSpec; this.rerankCount = rerankCount; this.matchFeaturesToHide = matchFeaturesToHide; this.normalizers = normalizers; + this.defaultValues = defaultValues; + } + + static class DefaultQueryFeatureExtractor { + final String baseName; + final String qfName; + TensorType type = null; + Tensor value = null; + DefaultQueryFeatureExtractor(String unwrappedQueryFeature) { + baseName = unwrappedQueryFeature; + qfName = "query(" + baseName + ")"; + } + List<String> lookingFor() { + return List.of(qfName, "vespa.type.query." + baseName); + } + void accept(String key, String propValue) { + if (key.equals(qfName)) { + this.value = Tensor.from(propValue); + } else { + this.type = TensorType.fromSpec(propValue); + } + } + Tensor extract() { + if (value != null) { + return value; + } + if (type != null) { + return Tensor.Builder.of(type).build(); + } + return Tensor.from(0.0); + } + } + + static private Map<String, Tensor> extraDefaultQueryFeatureValues(RankProfilesConfig.Rankprofile rp, + List<String> fromQuery, + List<NormalizerSetup> normalizers) + { + Map<String, DefaultQueryFeatureExtractor> extractors = new HashMap<>(); + for (String fn : fromQuery) { + extractors.put(fn, new DefaultQueryFeatureExtractor(fn)); + } + for (var n : normalizers) { + for (String fn : n.inputEvalSpec().fromQuery()) { + extractors.put(fn, new DefaultQueryFeatureExtractor(fn)); + } + } + Map<String, DefaultQueryFeatureExtractor> targets = new HashMap<>(); + for (var extractor : extractors.values()) { + for (String key : extractor.lookingFor()) { + var old = targets.put(key, extractor); + if (old != null) { + throw new IllegalStateException("Multiple targets for key: " + key); + } + } + } + for (var prop : rp.fef().property()) { + var extractor = targets.get(prop.name()); + if (extractor != null) { + extractor.accept(prop.name(), prop.value()); + } + } + Map<String, Tensor> defaultValues = new HashMap<>(); + for (var extractor : extractors.values()) { + defaultValues.put(extractor.qfName, extractor.extract()); + } + return defaultValues; } static GlobalPhaseSetup maybeMakeSetup(RankProfilesConfig.Rankprofile rp, RankProfilesEvaluator modelEvaluator) { @@ -98,7 +162,7 @@ class GlobalPhaseSetup { var normSupplier = SimpleEvaluator.wrap(normSource); normalizers.add(makeNormalizerSetup(cfg, matchFeatures, normSupplier, normInputs, rerankCount)); } - } else if (matchFeatures.contains(input)) { + } else if (matchFeatures.contains(input) || matchFeatures.contains(WrappedHit.alternate(input))) { fromMF.add(input); } else { throw new IllegalArgumentException("Bad config, missing global-phase input: " + input); @@ -106,7 +170,8 @@ class GlobalPhaseSetup { } Supplier<Evaluator> supplier = SimpleEvaluator.wrap(functionEvaluatorSource); var gfun = new FunEvalSpec(supplier, fromQuery, fromMF); - return new GlobalPhaseSetup(gfun, rerankCount, namesToHide, normalizers); + var defaultValues = extraDefaultQueryFeatureValues(rp, fromQuery, normalizers); + return new GlobalPhaseSetup(gfun, rerankCount, namesToHide, normalizers, defaultValues); } return null; } @@ -123,7 +188,7 @@ class GlobalPhaseSetup { String queryFeatureName = asQueryFeature(input); if (queryFeatureName != null) { fromQuery.add(queryFeatureName); - } else if (matchFeatures.contains(input)) { + } else if (matchFeatures.contains(input) || matchFeatures.contains(WrappedHit.alternate(input))) { fromMF.add(input); } else { throw new IllegalArgumentException("Bad config, missing normalizer input: " + input); diff --git a/container-search/src/main/java/com/yahoo/search/ranking/PreparedInput.java b/container-search/src/main/java/com/yahoo/search/ranking/PreparedInput.java index 5ab2d7160f9..914635fef59 100644 --- a/container-search/src/main/java/com/yahoo/search/ranking/PreparedInput.java +++ b/container-search/src/main/java/com/yahoo/search/ranking/PreparedInput.java @@ -13,35 +13,35 @@ import com.yahoo.tensor.Tensor; import com.yahoo.data.access.helpers.MatchFeatureData; import com.yahoo.data.access.helpers.MatchFeatureFilter; -import java.util.ArrayList; -import java.util.Collection; -import java.util.List; -import java.util.Optional; +import java.util.*; import java.util.function.Supplier; import java.util.logging.Logger; record PreparedInput(String name, Tensor value) { - static List<PreparedInput> findFromQuery(Query query, Collection<String> queryFeatures) { + static List<PreparedInput> findFromQuery(Query query, Collection<String> queryFeatures, Map<String, Tensor> defaultValues) { List<PreparedInput> result = new ArrayList<>(); var ranking = query.getRanking(); var rankFeatures = ranking.getFeatures(); - var rankProps = ranking.getProperties().asMap(); + var rankProps = ranking.getProperties(); for (String queryFeatureName : queryFeatures) { String needed = "query(" + queryFeatureName + ")"; - // searchers are recommended to place query features here: - var feature = rankFeatures.getTensor(queryFeatureName); - if (feature.isPresent()) { - result.add(new PreparedInput(needed, feature.get())); - } else { - // but other ways of setting query features end up in the properties: - var objList = rankProps.get(queryFeatureName); - if (objList != null && objList.size() == 1 && objList.get(0) instanceof Tensor t) { - result.add(new PreparedInput(needed, t)); - } else { - throw new IllegalArgumentException("missing query feature: " + queryFeatureName); + // after prepare() the query tensor ends up here: + var feature = rankProps.getAsTensor(queryFeatureName); + if (feature.isEmpty()) { + // searchers are recommended to place query features here: + feature = rankFeatures.getTensor(needed); + } + if (feature.isEmpty()) { + var t = defaultValues.get(needed); + if (t != null) { + feature = Optional.of(t); } } + if (feature.isEmpty()) { + throw new IllegalArgumentException("missing query feature: " + queryFeatureName); + } + result.add(new PreparedInput(needed, feature.get())); } return result; } diff --git a/container-search/src/main/java/com/yahoo/search/result/HitIterator.java b/container-search/src/main/java/com/yahoo/search/result/HitIterator.java index d701c15cb24..3d3eb6030f4 100644 --- a/container-search/src/main/java/com/yahoo/search/result/HitIterator.java +++ b/container-search/src/main/java/com/yahoo/search/result/HitIterator.java @@ -19,10 +19,10 @@ public class HitIterator implements Iterator<Hit> { private int index = -1; /** The list of hits to iterate over */ - private List<Hit> hits; + private final List<Hit> hits; /** The result the hits belong to */ - private HitGroup hitGroup; + private final HitGroup hitGroup; /** Whether the iterator is in a state where remove is OK */ private boolean canRemove = false; diff --git a/container-search/src/test/java/com/yahoo/search/query/RankProfileInputTest.java b/container-search/src/test/java/com/yahoo/search/query/RankProfileInputTest.java index a76f52fd811..90e21e5f3b0 100644 --- a/container-search/src/test/java/com/yahoo/search/query/RankProfileInputTest.java +++ b/container-search/src/test/java/com/yahoo/search/query/RankProfileInputTest.java @@ -56,7 +56,7 @@ public class RankProfileInputTest { fail("Expected exception"); } catch (IllegalArgumentException e) { - assertEquals("No profile named 'bOnly' exists in schemas [a]", Exceptions.toMessageString(e)); + assertEquals("Could not set 'ranking.features.query(myTensor1)' to '{{a:a1, b:b1}:1.0, {a:a2, b:b1}:2.0}}': No profile named 'bOnly' exists in schemas [a]", Exceptions.toMessageString(e)); } } @@ -68,9 +68,10 @@ public class RankProfileInputTest { fail("Expected exception"); } catch (IllegalArgumentException e) { - assertEquals("Conflicting input type declarations for 'query(myTensor1)': " + - "Declared as tensor(a{},b{}) in rank profile 'inconsistent' in schema 'a', " + - "and as tensor(x[10]) in rank profile 'inconsistent' in schema 'b'", + assertEquals("Could not set 'ranking.features.query(myTensor1)' to '{{a:a1, b:b1}:1.0, {a:a2, b:b1}:2.0}}': " + + "Conflicting input type declarations for 'query(myTensor1)': " + + "Declared as tensor(a{},b{}) in rank profile 'inconsistent' in schema 'a', " + + "and as tensor(x[10]) in rank profile 'inconsistent' in schema 'b'", Exceptions.toMessageString(e)); } } diff --git a/container-search/src/test/java/com/yahoo/search/query/ranking/RankPropertiesTestCase.java b/container-search/src/test/java/com/yahoo/search/query/ranking/RankPropertiesTestCase.java new file mode 100644 index 00000000000..81c657f6323 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/ranking/RankPropertiesTestCase.java @@ -0,0 +1,49 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.query.ranking; + +import com.yahoo.search.Query; +import com.yahoo.search.query.Ranking; +import com.yahoo.tensor.Tensor; +import com.yahoo.tensor.TensorType; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +/** + * @author arnej + */ +public class RankPropertiesTestCase { + + @Test + void requireThatGetAsTensorCanGetDoublesAndTensors() { + TensorType ttype = new TensorType.Builder().mapped("cat").build(); + Tensor mappedTensor = Tensor.from(ttype, "{ {cat:foo}:2.5, {cat:bar}:1.25 }"); + RankFeatures f = new RankFeatures(new Ranking(new Query())); + f.put("query(myDouble)", 42.75); + f.put("query(myTensor)", mappedTensor); + RankProperties p = new RankProperties(); + f.prepare(p); + var optT = p.getAsTensor("myDouble"); + assertEquals(true, optT.isPresent()); + assertEquals(TensorType.empty, optT.get().type()); + assertEquals(42.75, optT.get().asDouble()); + optT = p.getAsTensor("myTensor"); + assertEquals(true, optT.isPresent()); + assertEquals(mappedTensor, optT.get()); + } + + @Test + void requireThatGetAsTensorFailsOnStrings() { + RankFeatures f = new RankFeatures(new Ranking(new Query())); + // common mistake: + f.put("query(myTensor)", "{ {cat:foo}:2.5, {cat:bar}:1.25 }"); + RankProperties p = new RankProperties(); + f.prepare(p); + var ex = assertThrows(IllegalArgumentException.class, () -> p.getAsTensor("myTensor")); + assertEquals("Expected 'myTensor' to be a tensor or double, " + + "but it is '{ {cat:foo}:2.5, {cat:bar}:1.25 }', " + + "this usually means that 'myTensor' is not defined in the schema. " + + "See https://docs.vespa.ai/en/tensor-user-guide.html#querying-with-tensors", ex.getMessage()); + } +} diff --git a/container-search/src/test/java/com/yahoo/search/ranking/GlobalPhaseRerankHitsImplTest.java b/container-search/src/test/java/com/yahoo/search/ranking/GlobalPhaseRerankHitsImplTest.java new file mode 100644 index 00000000000..f55130c0c93 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/ranking/GlobalPhaseRerankHitsImplTest.java @@ -0,0 +1,248 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.ranking; + +import com.yahoo.data.access.Inspectable; +import com.yahoo.data.access.Type; +import com.yahoo.data.access.helpers.MatchFeatureData; +import com.yahoo.data.access.simple.Value; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.result.FeatureData; +import com.yahoo.search.result.Hit; +import com.yahoo.tensor.Tensor; +import org.junit.jupiter.api.Test; + +import java.util.*; +import java.util.function.Supplier; + +import static org.junit.jupiter.api.Assertions.*; + +public class GlobalPhaseRerankHitsImplTest { + static class EvalSum implements Evaluator { + double baseValue; + List<Tensor> values = new ArrayList<>(); + EvalSum(double baseValue) { this.baseValue = baseValue; } + @Override public Evaluator bind(String name, Tensor value) { + values.add(value); + return this; + } + @Override public double evaluateScore() { + double result = baseValue; + for (var value: values) { + result += value.asDouble(); + } + return result; + } + } + static FunEvalSpec makeConstSpec(double constValue) { + return new FunEvalSpec(() -> new EvalSum(constValue), Collections.emptyList(), Collections.emptyList()); + } + static FunEvalSpec makeSumSpec(List<String> fromQuery, List<String> fromMF) { + return new FunEvalSpec(() -> new EvalSum(0.0), fromQuery, fromMF); + } + static class ExpectingNormalizer extends Normalizer { + List<Double> expected; + ExpectingNormalizer(List<Double> expected) { + super(100); + this.expected = expected; + } + @Override void normalize() { + double rank = 1; + assertEquals(size, expected.size()); + for (int i = 0; i < size; i++) { + assertEquals(data[i], expected.get(i)); + data[i] = rank; + rank += 1; + } + } + @Override String normalizing() { return "expecting"; } + } + static NormalizerSetup makeNormalizer(String name, List<Double> expected, FunEvalSpec evalSpec) { + return new NormalizerSetup(name, () -> new ExpectingNormalizer(expected), evalSpec); + } + static class SetupBuilder { + FunEvalSpec mainSpec = makeConstSpec(0.0); + int rerankCount = 100; + List<String> hiddenMF = new ArrayList<>(); + List<NormalizerSetup> normalizers = new ArrayList<>(); + Map<String, Tensor> defaultValues = new HashMap<>(); + SetupBuilder eval(FunEvalSpec spec) { mainSpec = spec; return this; } + SetupBuilder rerank(int value) { rerankCount = value; return this; } + SetupBuilder hide(String mf) { hiddenMF.add(mf); return this; } + SetupBuilder addNormalizer(NormalizerSetup normalizer) { normalizers.add(normalizer); return this; } + SetupBuilder addDefault(String name, Tensor value) { defaultValues.put(name, value); return this; } + GlobalPhaseSetup build() { return new GlobalPhaseSetup(mainSpec, rerankCount, hiddenMF, normalizers, defaultValues); } + } + static SetupBuilder setup() { return new SetupBuilder(); } + static record NamedValue(String name, double value) {} + NamedValue value(String name, double value) { + return new NamedValue(name, value); + } + Query makeQuery(List<NamedValue> inQuery, boolean withPrepare) { + var query = new Query(); + for (var v: inQuery) { + query.getRanking().getFeatures().put(v.name, v.value); + } + if (withPrepare) { + query.getRanking().prepare(); + } + return query; + } + Query makeQuery(List<NamedValue> inQuery) { return makeQuery(inQuery, false); } + Query makeQueryWithPrepare(List<NamedValue> inQuery) { return makeQuery(inQuery, true); } + + static Hit makeHit(String id, double score, FeatureData mf) { + Hit hit = new Hit(id, score); + hit.setField("matchfeatures", mf); + return hit; + } + static Hit hit(String id, double score) { + return makeHit(id, score, FeatureData.empty()); + } + static class HitFactory { + MatchFeatureData mfData; + Map<String,Integer> map = new HashMap<>(); + HitFactory(List<String> mfNames) { + int i = 0; + for (var name: mfNames) { + map.put(name, i++); + } + mfData = new MatchFeatureData(mfNames); + } + Hit create(String id, double score, List<NamedValue> inMF) { + var mf = mfData.addHit(); + for (var v: inMF) { + var idx = map.get(v.name); + assertNotNull(idx); + mf.set(idx, v.value); + } + return makeHit(id, score, new FeatureData(mf)); + } + } + Result makeResult(Query query, List<Hit> hits) { + var result = new Result(query); + result.hits().addAll(hits); + return result; + } + static class Expect { + Map<String,Double> map = new HashMap<>(); + static Expect make(List<Hit> hits) { + var result = new Expect(); + for (var hit : hits) { + result.map.put(hit.getId().stringValue(), hit.getRelevance().getScore()); + } + return result; + } + void verifyScores(Result actual) { + double prev = Double.MAX_VALUE; + assertEquals(actual.hits().size(), map.size()); + for (var hit : actual.hits()) { + var name = hit.getId().stringValue(); + var score = map.get(name); + assertNotNull(score, name); + assertEquals(score.doubleValue(), hit.getRelevance().getScore(), name); + assertTrue(score <= prev); + prev = score; + } + } + } + void verifyHasMF(Result result, String name) { + for (var hit: result.hits()) { + if (hit.getField("matchfeatures") instanceof FeatureData mf) { + assertNotNull(mf.getTensor(name)); + } else { + fail("matchfeatures are missing"); + } + } + } + void verifyDoesNotHaveMF(Result result, String name) { + for (var hit: result.hits()) { + if (hit.getField("matchfeatures") instanceof FeatureData mf) { + assertNull(mf.getTensor(name)); + } else { + fail("matchfeatures are missing"); + } + } + } + void verifyDoesNotHaveMatchFeaturesField(Result result) { + for (var hit: result.hits()) { + assertNull(hit.getField("matchfeatures")); + } + } + @Test void partialRerankWithRescaling() { + var setup = setup().rerank(2).eval(makeConstSpec(3.0)).build(); + var query = makeQuery(Collections.emptyList()); + var result = makeResult(query, List.of(hit("a", 3), hit("b", 4), hit("c", 5), hit("d", 6))); + var expect = Expect.make(List.of(hit("a", 1), hit("b", 2), hit("c", 3), hit("d", 3))); + GlobalPhaseRanker.rerankHitsImpl(setup, query, result); + expect.verifyScores(result); + } + @Test void matchFeaturesCanBePartiallyHidden() { + var setup = setup().eval(makeSumSpec(Collections.emptyList(), List.of("public_value", "private_value"))).hide("private_value").build(); + var query = makeQuery(Collections.emptyList()); + var factory = new HitFactory(List.of("public_value", "private_value")); + var result = makeResult(query, List.of(factory.create("a", 1, List.of(value("public_value", 2), value("private_value", 3))), + factory.create("b", 2, List.of(value("public_value", 5), value("private_value", 7))))); + var expect = Expect.make(List.of(hit("a", 5), hit("b", 12))); + GlobalPhaseRanker.rerankHitsImpl(setup, query, result); + expect.verifyScores(result); + verifyHasMF(result, "public_value"); + verifyDoesNotHaveMF(result, "private_value"); + } + @Test void matchFeaturesCanBeRemoved() { + var setup = setup().eval(makeSumSpec(Collections.emptyList(), List.of("private_value"))).hide("private_value").build(); + var query = makeQuery(Collections.emptyList()); + var factory = new HitFactory(List.of("private_value")); + var result = makeResult(query, List.of(factory.create("a", 1, List.of(value("private_value", 3))), + factory.create("b", 2, List.of(value("private_value", 7))))); + var expect = Expect.make(List.of(hit("a", 3), hit("b", 7))); + GlobalPhaseRanker.rerankHitsImpl(setup, query, result); + expect.verifyScores(result); + verifyDoesNotHaveMatchFeaturesField(result); + } + @Test void queryFeaturesCanBeUsed() { + var setup = setup().eval(makeSumSpec(List.of("foo"), List.of("bar"))).build(); + var query = makeQuery(List.of(value("query(foo)", 7))); + var factory = new HitFactory(List.of("bar")); + var result = makeResult(query, List.of(factory.create("a", 1, List.of(value("bar", 2))), + factory.create("b", 2, List.of(value("bar", 5))))); + var expect = Expect.make(List.of(hit("a", 9), hit("b", 12))); + GlobalPhaseRanker.rerankHitsImpl(setup, query, result); + expect.verifyScores(result); + verifyHasMF(result, "bar"); + } + @Test void queryFeaturesCanBeUsedWhenPrepared() { + var setup = setup().eval(makeSumSpec(List.of("foo"), List.of("bar"))).build(); + var query = makeQueryWithPrepare(List.of(value("query(foo)", 7))); + var factory = new HitFactory(List.of("bar")); + var result = makeResult(query, List.of(factory.create("a", 1, List.of(value("bar", 2))), + factory.create("b", 2, List.of(value("bar", 5))))); + var expect = Expect.make(List.of(hit("a", 9), hit("b", 12))); + GlobalPhaseRanker.rerankHitsImpl(setup, query, result); + expect.verifyScores(result); + verifyHasMF(result, "bar"); + } + @Test void queryFeaturesCanBeDefaultValues() { + var setup = setup().eval(makeSumSpec(List.of("foo", "bar"), Collections.emptyList())) + .addDefault("query(bar)", Tensor.from(5.0)).build(); + var query = makeQuery(List.of(value("query(foo)", 7))); + var result = makeResult(query, List.of(hit("a", 1))); + var expect = Expect.make(List.of(hit("a", 12))); + GlobalPhaseRanker.rerankHitsImpl(setup, query, result); + expect.verifyScores(result); + } + @Test void withNormalizer() { + var setup = setup().eval(makeSumSpec(Collections.emptyList(), List.of("bar"))) + .addNormalizer(makeNormalizer("foo", List.of(115.0, 65.0, 55.0, 45.0, 15.0), makeSumSpec(List.of("x"), List.of("bar")))).build(); + var query = makeQuery(List.of(value("query(x)", 5))); + var factory = new HitFactory(List.of("bar")); + var result = makeResult(query, List.of(factory.create("a", 1, List.of(value("bar", 10))), + factory.create("b", 2, List.of(value("bar", 40))), + factory.create("c", 3, List.of(value("bar", 50))), + factory.create("d", 4, List.of(value("bar", 60))), + factory.create("e", 5, List.of(value("bar", 110))))); + var expect = Expect.make(List.of(hit("a", 15), hit("b", 44), hit("c", 53), hit("d", 62), hit("e", 111))); + GlobalPhaseRanker.rerankHitsImpl(setup, query, result); + expect.verifyScores(result); + } +} diff --git a/container-search/src/test/java/com/yahoo/search/ranking/GlobalPhaseSetupTest.java b/container-search/src/test/java/com/yahoo/search/ranking/GlobalPhaseSetupTest.java new file mode 100644 index 00000000000..082531a97dd --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/ranking/GlobalPhaseSetupTest.java @@ -0,0 +1,123 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.ranking; + +import com.yahoo.config.subscription.ConfigGetter; +import com.yahoo.filedistribution.fileacquirer.MockFileAcquirer; +import com.yahoo.tensor.Tensor; +import com.yahoo.vespa.config.search.RankProfilesConfig; +import com.yahoo.vespa.config.search.core.OnnxModelsConfig; +import com.yahoo.vespa.config.search.core.RankingConstantsConfig; +import com.yahoo.vespa.config.search.core.RankingExpressionsConfig; + +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.*; + +public class GlobalPhaseSetupTest { + private static final String CONFIG_DIR = "src/test/resources/config/"; + + @SuppressWarnings("deprecation") + RankProfilesConfig readConfig(String subDir) { + String cfgId = "file:" + CONFIG_DIR + subDir + "/rank-profiles.cfg"; + return ConfigGetter.getConfig(RankProfilesConfig.class, cfgId); + } + + @Test void mediumAdvancedSetup() { + RankProfilesConfig rpCfg = readConfig("medium"); + assertEquals(1, rpCfg.rankprofile().size()); + RankProfilesEvaluator rpEvaluator = createEvaluator(rpCfg); + var setup = GlobalPhaseSetup.maybeMakeSetup(rpCfg.rankprofile().get(0), rpEvaluator); + assertNotNull(setup); + assertEquals(42, setup.rerankCount); + assertEquals(0, setup.normalizers.size()); + assertEquals(9, setup.matchFeaturesToHide.size()); + assertEquals(1, setup.globalPhaseEvalSpec.fromQuery().size()); + assertEquals(9, setup.globalPhaseEvalSpec.fromMF().size()); + } + + @Test void queryFeaturesWithDefaults() { + RankProfilesConfig rpCfg = readConfig("qf_defaults"); + assertEquals(1, rpCfg.rankprofile().size()); + RankProfilesEvaluator rpEvaluator = createEvaluator(rpCfg); + var setup = GlobalPhaseSetup.maybeMakeSetup(rpCfg.rankprofile().get(0), rpEvaluator); + assertNotNull(setup); + assertEquals(0, setup.normalizers.size()); + assertEquals(0, setup.matchFeaturesToHide.size()); + assertEquals(5, setup.globalPhaseEvalSpec.fromQuery().size()); + assertEquals(2, setup.globalPhaseEvalSpec.fromMF().size()); + assertEquals(5, setup.defaultValues.size()); + assertEquals(Tensor.from(0.0), setup.defaultValues.get("query(w_no_def)")); + assertEquals(Tensor.from(1.0), setup.defaultValues.get("query(w_has_def)")); + assertEquals(Tensor.from("tensor(m{}):{}"), setup.defaultValues.get("query(m_no_def)")); + assertEquals(Tensor.from("tensor(v[3]):[0,0,0]"), setup.defaultValues.get("query(v_no_def)")); + assertEquals(Tensor.from("tensor(v[3]):[2,0.25,1.5]"), setup.defaultValues.get("query(v_has_def)")); + } + + @Test void withNormalizers() { + RankProfilesConfig rpCfg = readConfig("with_normalizers"); + assertEquals(1, rpCfg.rankprofile().size()); + RankProfilesEvaluator rpEvaluator = createEvaluator(rpCfg); + var setup = GlobalPhaseSetup.maybeMakeSetup(rpCfg.rankprofile().get(0), rpEvaluator); + assertNotNull(setup); + var nList = setup.normalizers; + assertEquals(7, nList.size()); + nList.sort((a,b) -> a.name().compareTo(b.name())); + + var n = nList.get(0); + assertEquals("normalize@2974853441@linear", n.name()); + assertEquals(0, n.inputEvalSpec().fromQuery().size()); + assertEquals(1, n.inputEvalSpec().fromMF().size()); + assertEquals("funmf", n.inputEvalSpec().fromMF().get(0)); + assertEquals("linear", n.supplier().get().normalizing()); + + n = nList.get(1); + assertEquals("normalize@3414032797@rrank", n.name()); + assertEquals(0, n.inputEvalSpec().fromQuery().size()); + assertEquals(1, n.inputEvalSpec().fromMF().size()); + assertEquals("attribute(year)", n.inputEvalSpec().fromMF().get(0)); + assertEquals("reciprocal-rank{k:60.0}", n.supplier().get().normalizing()); + + n = nList.get(2); + assertEquals("normalize@3551296680@linear", n.name()); + assertEquals(0, n.inputEvalSpec().fromQuery().size()); + assertEquals(1, n.inputEvalSpec().fromMF().size()); + assertEquals("nativeRank", n.inputEvalSpec().fromMF().get(0)); + assertEquals("linear", n.supplier().get().normalizing()); + + n = nList.get(3); + assertEquals("normalize@4280591309@rrank", n.name()); + assertEquals(0, n.inputEvalSpec().fromQuery().size()); + assertEquals(1, n.inputEvalSpec().fromMF().size()); + assertEquals("bm25(myabstract)", n.inputEvalSpec().fromMF().get(0)); + assertEquals("reciprocal-rank{k:42.0}", n.supplier().get().normalizing()); + + n = nList.get(4); + assertEquals("normalize@4370385022@linear", n.name()); + assertEquals(1, n.inputEvalSpec().fromQuery().size()); + assertEquals("myweight", n.inputEvalSpec().fromQuery().get(0)); + assertEquals(1, n.inputEvalSpec().fromMF().size()); + assertEquals("attribute(foo1)", n.inputEvalSpec().fromMF().get(0)); + assertEquals("linear", n.supplier().get().normalizing()); + + n = nList.get(5); + assertEquals("normalize@4640646880@linear", n.name()); + assertEquals(0, n.inputEvalSpec().fromQuery().size()); + assertEquals(1, n.inputEvalSpec().fromMF().size()); + assertEquals("attribute(foo1)", n.inputEvalSpec().fromMF().get(0)); + assertEquals("linear", n.supplier().get().normalizing()); + + n = nList.get(6); + assertEquals("normalize@6283155534@linear", n.name()); + assertEquals(0, n.inputEvalSpec().fromQuery().size()); + assertEquals(1, n.inputEvalSpec().fromMF().size()); + assertEquals("bm25(mytitle)", n.inputEvalSpec().fromMF().get(0)); + assertEquals("linear", n.supplier().get().normalizing()); + } + + private RankProfilesEvaluator createEvaluator(RankProfilesConfig config) { + RankingConstantsConfig constantsConfig = new RankingConstantsConfig.Builder().build(); + RankingExpressionsConfig expressionsConfig = new RankingExpressionsConfig.Builder().build(); + OnnxModelsConfig onnxModelsConfig = new OnnxModelsConfig.Builder().build(); + return new RankProfilesEvaluator(config, constantsConfig, expressionsConfig, onnxModelsConfig, MockFileAcquirer.returnFile(null)); + } +} diff --git a/container-search/src/test/java/com/yahoo/search/ranking/RangeAdjusterTest.java b/container-search/src/test/java/com/yahoo/search/ranking/RangeAdjusterTest.java new file mode 100644 index 00000000000..af495d829b6 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/ranking/RangeAdjusterTest.java @@ -0,0 +1,78 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.ranking; + +import com.google.common.collect.RangeMap; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.*; + +class RangeAdjusterTest { + static void rerank(RangeAdjuster adjuster, double before, double after) { + adjuster.withInitialScore(before); + adjuster.withFinalScore(after); + } + static double adjust(RangeAdjuster adjuster, double score) { + return score * adjuster.scale() + adjuster.bias(); + } + @Test void noScoresNoNeed() { + var adjuster = new RangeAdjuster(); + assertFalse(adjuster.rescaleNeeded()); + } + @Test void increasingScoresNoNeed() { + var adjuster = new RangeAdjuster(); + rerank(adjuster, 1.0, 2.0); + rerank(adjuster, 3.0, 4.0); + rerank(adjuster, 2.0, 3.0); + assertFalse(adjuster.rescaleNeeded()); + } + @Test void singleScoreAdjuster() { + var adjuster = new RangeAdjuster(); + rerank(adjuster, 10.0, 5.0); + assertEquals(adjuster.scale(), 1.0); + assertEquals(adjuster.bias(), -5.0); + assertEquals(adjust(adjuster, 10.0), 5.0); + assertEquals(adjust(adjuster, 7.0), 2.0); + } + @Test void movingAdjuster() { + var adjuster = new RangeAdjuster(); + rerank(adjuster, -10.0, -15.0); + rerank(adjuster, -11.0, -16.0); + rerank(adjuster, -12.0, -17.0); + assertEquals(adjuster.scale(), 1.0); + assertEquals(adjuster.bias(), -5.0); + assertEquals(adjust(adjuster, -10.0), -15.0); + assertEquals(adjust(adjuster, -15.0), -20.0); + } + @Test void compactingAdjuster() { + var adjuster = new RangeAdjuster(); + rerank(adjuster, 100.0, 10.0); + rerank(adjuster, 200.0, 20.0); + rerank(adjuster, 300.0, 30.0); + assertEquals(adjuster.scale(), 0.1); + assertEquals(adjuster.bias(), 0.0); + assertEquals(adjust(adjuster, 100.0), 10.0); + assertEquals(adjust(adjuster, 50.0), 5.0); + } + @Test void expandingAdjuster() { + var adjuster = new RangeAdjuster(); + rerank(adjuster, -10.0, -100.0); + rerank(adjuster, -20.0, -200.0); + rerank(adjuster, -30.0, -300.0); + assertEquals(adjuster.scale(), 10.0); + assertEquals(adjuster.bias(), 0.0); + assertEquals(adjust(adjuster, -10.0), -100.0); + assertEquals(adjust(adjuster, -40.0), -400.0); + } + // this test represents the normal re-scaling case. + @Test void compactingAndMovingAdjuster() { + var adjuster = new RangeAdjuster(); + rerank(adjuster, 1000.0, 1.0); + rerank(adjuster, 800.0, 0.8); + rerank(adjuster, 500.0, 0.5); + assertEquals(adjuster.scale(), 1.0/500.0); + assertEquals(adjuster.bias(), -0.5); + assertEquals(adjust(adjuster, 500.0), 0.5); + assertEquals(adjust(adjuster, 250.0), 0.0); + assertEquals(adjust(adjuster, 0.0), -0.5); + } +}
\ No newline at end of file diff --git a/container-search/src/test/java/com/yahoo/search/result/CoverageTestCase.java b/container-search/src/test/java/com/yahoo/search/result/CoverageTestCase.java index 2d1dac64302..cadbc6adf5c 100644 --- a/container-search/src/test/java/com/yahoo/search/result/CoverageTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/result/CoverageTestCase.java @@ -6,7 +6,9 @@ import com.yahoo.search.Result; import org.junit.jupiter.api.Test; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; /** * @author Steinar Knutsen @@ -103,4 +105,22 @@ public class CoverageTestCase { } } + private void verifyNoCoverage(Coverage zero) { + assertFalse(zero.isDegraded()); + assertEquals(100, zero.getResultPercentage()); + assertTrue(zero.getFull()); + zero.setDegradedReason(com.yahoo.container.handler.Coverage.DEGRADED_BY_TIMEOUT); + assertTrue(zero.isDegraded()); + assertEquals(0, zero.getResultPercentage()); + assertFalse(zero.getFull()); + } + @Test + void testCoverageWithNoResponseFromSearchNodesAndTimeout() { + verifyNoCoverage(new Coverage(0, 0, 0)); + } + @Test + void testCoverageWithResponseFromSearchNodesAndTimeout() { + verifyNoCoverage(new Coverage(0, 0, 1)); + } + } diff --git a/container-search/src/test/java/com/yahoo/search/test/QueryTestCase.java b/container-search/src/test/java/com/yahoo/search/test/QueryTestCase.java index 65df1206c47..6a310180eab 100644 --- a/container-search/src/test/java/com/yahoo/search/test/QueryTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/test/QueryTestCase.java @@ -623,6 +623,15 @@ public class QueryTestCase { } @Test + void globalphase_parameters_are_resolved() { + var q = new Query("?query=foo"); + assertNull(q.getRanking().getGlobalPhase().getRerankCount()); + q = new Query("?query=foo&" + + "ranking.globalPhase.rerankCount=42"); + assertEquals(42, q.getRanking().getGlobalPhase().getRerankCount()); + } + + @Test void testQueryPropertyResolveTracing() { QueryProfile testProfile = new QueryProfile("test"); testProfile.setOverridable("u", false, DimensionValues.empty); diff --git a/container-search/src/test/resources/config/medium/rank-profiles.cfg b/container-search/src/test/resources/config/medium/rank-profiles.cfg new file mode 100644 index 00000000000..5a609f70cef --- /dev/null +++ b/container-search/src/test/resources/config/medium/rank-profiles.cfg @@ -0,0 +1,55 @@ +rankprofile[0].name "withglobalphase" +rankprofile[0].fef.property[0].name "rankingExpression(myplus).rankingScript" +rankprofile[0].fef.property[0].value "attribute(foo1) + attribute(foo2)" +rankprofile[0].fef.property[1].name "rankingExpression(mymul).rankingScript" +rankprofile[0].fef.property[1].value "attribute(t1) * query(fromq)" +rankprofile[0].fef.property[2].name "rankingExpression(mymul).type" +rankprofile[0].fef.property[2].value "tensor(m{},v[3])" +rankprofile[0].fef.property[3].name "vespa.type.feature.attribute(t1)" +rankprofile[0].fef.property[3].value "tensor(m{},v[3])" +rankprofile[0].fef.property[4].name "vespa.rank.firstphase" +rankprofile[0].fef.property[4].value "attribute(foo1)" +rankprofile[0].fef.property[5].name "vespa.rank.globalphase" +rankprofile[0].fef.property[5].value "rankingExpression(globalphase)" +rankprofile[0].fef.property[6].name "rankingExpression(globalphase).rankingScript" +rankprofile[0].fef.property[6].value "rankingExpression(myplus) + reduce(rankingExpression(mymul), sum) + firstPhase + term(0).significance + fieldLength(artist) + fieldTermMatch(title,0).occurrences + termDistance(title,1,2).reverse + closeness(field,t1)" +rankprofile[0].fef.property[7].name "vespa.match.feature" +rankprofile[0].fef.property[7].value "fieldLength(artist)" +rankprofile[0].fef.property[8].name "vespa.match.feature" +rankprofile[0].fef.property[8].value "term(0).significance" +rankprofile[0].fef.property[9].name "vespa.match.feature" +rankprofile[0].fef.property[9].value "closeness(field,t1)" +rankprofile[0].fef.property[10].name "vespa.match.feature" +rankprofile[0].fef.property[10].value "termDistance(title,1,2).reverse" +rankprofile[0].fef.property[11].name "vespa.match.feature" +rankprofile[0].fef.property[11].value "firstPhase" +rankprofile[0].fef.property[12].name "vespa.match.feature" +rankprofile[0].fef.property[12].value "attribute(t1)" +rankprofile[0].fef.property[13].name "vespa.match.feature" +rankprofile[0].fef.property[13].value "attribute(foo1)" +rankprofile[0].fef.property[14].name "vespa.match.feature" +rankprofile[0].fef.property[14].value "fieldTermMatch(title,0).occurrences" +rankprofile[0].fef.property[15].name "vespa.match.feature" +rankprofile[0].fef.property[15].value "attribute(foo2)" +rankprofile[0].fef.property[16].name "vespa.hidden.matchfeature" +rankprofile[0].fef.property[16].value "fieldLength(artist)" +rankprofile[0].fef.property[17].name "vespa.hidden.matchfeature" +rankprofile[0].fef.property[17].value "term(0).significance" +rankprofile[0].fef.property[18].name "vespa.hidden.matchfeature" +rankprofile[0].fef.property[18].value "closeness(field,t1)" +rankprofile[0].fef.property[19].name "vespa.hidden.matchfeature" +rankprofile[0].fef.property[19].value "termDistance(title,1,2).reverse" +rankprofile[0].fef.property[20].name "vespa.hidden.matchfeature" +rankprofile[0].fef.property[20].value "firstPhase" +rankprofile[0].fef.property[21].name "vespa.hidden.matchfeature" +rankprofile[0].fef.property[21].value "attribute(t1)" +rankprofile[0].fef.property[22].name "vespa.hidden.matchfeature" +rankprofile[0].fef.property[22].value "attribute(foo1)" +rankprofile[0].fef.property[23].name "vespa.hidden.matchfeature" +rankprofile[0].fef.property[23].value "fieldTermMatch(title,0).occurrences" +rankprofile[0].fef.property[24].name "vespa.hidden.matchfeature" +rankprofile[0].fef.property[24].value "attribute(foo2)" +rankprofile[0].fef.property[25].name "vespa.globalphase.rerankcount" +rankprofile[0].fef.property[25].value "42" +rankprofile[0].fef.property[26].name "vespa.type.attribute.t1" +rankprofile[0].fef.property[26].value "tensor(m{},v[3])" diff --git a/container-search/src/test/resources/config/qf_defaults/rank-profiles.cfg b/container-search/src/test/resources/config/qf_defaults/rank-profiles.cfg new file mode 100644 index 00000000000..731064c4dd6 --- /dev/null +++ b/container-search/src/test/resources/config/qf_defaults/rank-profiles.cfg @@ -0,0 +1,23 @@ +rankprofile[0].name "gp_with_qf_defaults" +rankprofile[0].fef.property[0].name "vespa.rank.firstphase" +rankprofile[0].fef.property[0].value "attribute(foo1)" +rankprofile[0].fef.property[1].name "vespa.rank.globalphase" +rankprofile[0].fef.property[1].value "rankingExpression(globalphase)" +rankprofile[0].fef.property[2].name "rankingExpression(globalphase).rankingScript" +rankprofile[0].fef.property[2].value "reduce(query(m_no_def) * query(v_no_def) * query(v_has_def) * attribute(t1), sum) + attribute(bar3) * query(w_no_def) * query(w_has_def)" +rankprofile[0].fef.property[3].name "vespa.match.feature" +rankprofile[0].fef.property[3].value "attribute(t1)" +rankprofile[0].fef.property[4].name "vespa.match.feature" +rankprofile[0].fef.property[4].value "attribute(bar3)" +rankprofile[0].fef.property[5].name "vespa.type.attribute.t1" +rankprofile[0].fef.property[5].value "tensor(m{},v[3])" +rankprofile[0].fef.property[6].name "vespa.type.query.m_no_def" +rankprofile[0].fef.property[6].value "tensor(m{})" +rankprofile[0].fef.property[7].name "vespa.type.query.v_no_def" +rankprofile[0].fef.property[7].value "tensor(v[3])" +rankprofile[0].fef.property[8].name "query(w_has_def)" +rankprofile[0].fef.property[8].value "1.0" +rankprofile[0].fef.property[9].name "vespa.type.query.v_has_def" +rankprofile[0].fef.property[9].value "tensor(v[3])" +rankprofile[0].fef.property[10].name "query(v_has_def)" +rankprofile[0].fef.property[10].value "tensor(v[3]):{{v:0}:2.0, {v:1}:0.25, {v:2}:1.5}" diff --git a/container-search/src/test/resources/config/with_normalizers/rank-profiles.cfg b/container-search/src/test/resources/config/with_normalizers/rank-profiles.cfg new file mode 100644 index 00000000000..ea5df465cce --- /dev/null +++ b/container-search/src/test/resources/config/with_normalizers/rank-profiles.cfg @@ -0,0 +1,67 @@ +rankprofile[0].name "with_normalizers" +rankprofile[0].fef.property[0].name "rankingExpression(funmf).rankingScript" +rankprofile[0].fef.property[0].value "attribute(foo1) * attribute(year)" +rankprofile[0].fef.property[1].name "rankingExpression(simplefun).rankingScript" +rankprofile[0].fef.property[1].value "attribute(foo1) * query(myweight)" +rankprofile[0].fef.property[2].name "rankingExpression(bm25two).rankingScript" +rankprofile[0].fef.property[2].value "bm25(mytitle) + bm25(myabstract)" +rankprofile[0].fef.property[3].name "rankingExpression(notused).rankingScript" +rankprofile[0].fef.property[3].value "normalize@5969841192@linear" +rankprofile[0].fef.property[4].name "vespa.rank.firstphase" +rankprofile[0].fef.property[4].value "rankingExpression(firstphase)" +rankprofile[0].fef.property[5].name "rankingExpression(firstphase).rankingScript" +rankprofile[0].fef.property[5].value "attribute(foo1) + bm25(mytitle) + bm25(myabstract)" +rankprofile[0].fef.property[6].name "vespa.rank.globalphase" +rankprofile[0].fef.property[6].value "rankingExpression(globalphase)" +rankprofile[0].fef.property[7].name "rankingExpression(globalphase).rankingScript" +rankprofile[0].fef.property[7].value "normalize@3551296680@linear + normalize@4640646880@linear + normalize@4370385022@linear + normalize@2974853441@linear + normalize@6283155534@linear + normalize@3414032797@rrank + normalize@4280591309@rrank" +rankprofile[0].fef.property[8].name "vespa.match.feature" +rankprofile[0].fef.property[8].value "nativeRank" +rankprofile[0].fef.property[9].name "vespa.match.feature" +rankprofile[0].fef.property[9].value "attribute(foo1)" +rankprofile[0].fef.property[10].name "vespa.match.feature" +rankprofile[0].fef.property[10].value "attribute(year)" +rankprofile[0].fef.property[11].name "vespa.match.feature" +rankprofile[0].fef.property[11].value "bm25(mytitle)" +rankprofile[0].fef.property[12].name "vespa.match.feature" +rankprofile[0].fef.property[12].value "bm25(myabstract)" +rankprofile[0].fef.property[13].name "vespa.match.feature" +rankprofile[0].fef.property[13].value "rankingExpression(funmf)" +rankprofile[0].fef.property[14].name "vespa.feature.rename" +rankprofile[0].fef.property[14].value "rankingExpression(funmf)" +rankprofile[0].fef.property[15].name "vespa.feature.rename" +rankprofile[0].fef.property[15].value "funmf" +rankprofile[0].fef.property[16].name "vespa.type.attribute.t1" +rankprofile[0].fef.property[16].value "tensor(m{},v[3])" +rankprofile[0].normalizer[0].name "normalize@3551296680@linear" +rankprofile[0].normalizer[0].input "nativeRank" +rankprofile[0].normalizer[0].algo LINEAR +rankprofile[0].normalizer[0].kparam 0.0 +rankprofile[0].normalizer[1].name "normalize@4640646880@linear" +rankprofile[0].normalizer[1].input "attribute(foo1)" +rankprofile[0].normalizer[1].algo LINEAR +rankprofile[0].normalizer[1].kparam 0.0 +rankprofile[0].normalizer[2].name "normalize@4370385022@linear" +rankprofile[0].normalizer[2].input "simplefun" +rankprofile[0].normalizer[2].algo LINEAR +rankprofile[0].normalizer[2].kparam 0.0 +rankprofile[0].normalizer[3].name "normalize@2974853441@linear" +rankprofile[0].normalizer[3].input "funmf" +rankprofile[0].normalizer[3].algo LINEAR +rankprofile[0].normalizer[3].kparam 0.0 +rankprofile[0].normalizer[4].name "normalize@6283155534@linear" +rankprofile[0].normalizer[4].input "bm25(mytitle)" +rankprofile[0].normalizer[4].algo LINEAR +rankprofile[0].normalizer[4].kparam 0.0 +rankprofile[0].normalizer[5].name "normalize@3414032797@rrank" +rankprofile[0].normalizer[5].input "attribute(year)" +rankprofile[0].normalizer[5].algo RRANK +rankprofile[0].normalizer[5].kparam 60.0 +rankprofile[0].normalizer[6].name "normalize@4280591309@rrank" +rankprofile[0].normalizer[6].input "bm25(myabstract)" +rankprofile[0].normalizer[6].algo RRANK +rankprofile[0].normalizer[6].kparam 42.0 +rankprofile[0].normalizer[7].name "normalize@5969841192@linear" +rankprofile[0].normalizer[7].input "firstPhase" +rankprofile[0].normalizer[7].algo LINEAR +rankprofile[0].normalizer[7].kparam 0.0 diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/DeploymentData.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/DeploymentData.java index f406095d579..fd4a34118c5 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/DeploymentData.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/DeploymentData.java @@ -14,9 +14,12 @@ import com.yahoo.yolean.concurrent.Memoized; import java.io.InputStream; import java.security.cert.X509Certificate; +import java.time.Duration; import java.util.List; import java.util.Optional; import java.util.function.Supplier; +import java.util.logging.Level; +import java.util.logging.Logger; import static java.util.Objects.requireNonNull; @@ -28,6 +31,8 @@ import static java.util.Objects.requireNonNull; */ public class DeploymentData { + private static final Logger log = Logger.getLogger(DeploymentData.class.getName()); + private final ApplicationId instance; private final ZoneId zone; private final Supplier<InputStream> applicationPackage; @@ -56,14 +61,14 @@ public class DeploymentData { this.zone = requireNonNull(zone); this.applicationPackage = requireNonNull(applicationPackage); this.platform = requireNonNull(platform); - this.endpoints = new Memoized<>(requireNonNull(endpoints)); + this.endpoints = wrap(requireNonNull(endpoints), Duration.ofSeconds(30), "deployment endpoints for " + instance + " in " + zone); this.dockerImageRepo = requireNonNull(dockerImageRepo); this.athenzDomain = athenzDomain; - this.quota = new Memoized<>(requireNonNull(quota)); + this.quota = wrap(requireNonNull(quota), Duration.ofSeconds(10), "quota for " + instance); this.tenantSecretStores = List.copyOf(requireNonNull(tenantSecretStores)); this.operatorCertificates = List.copyOf(requireNonNull(operatorCertificates)); - this.cloudAccount = new Memoized<>(requireNonNull(cloudAccount)); - this.dataPlaneTokens = new Memoized<>(dataPlaneTokens); + this.cloudAccount = wrap(requireNonNull(cloudAccount), Duration.ofSeconds(5), "cloud account for " + instance + " in " + zone); + this.dataPlaneTokens = wrap(dataPlaneTokens, Duration.ofSeconds(5), "data plane tokens for " + instance + " in " + zone); this.dryRun = dryRun; } @@ -83,8 +88,8 @@ public class DeploymentData { return platform; } - public Supplier<DeploymentEndpoints> endpoints() { - return endpoints; + public DeploymentEndpoints endpoints() { + return endpoints.get(); } public Optional<DockerImage> dockerImageRepo() { @@ -119,4 +124,41 @@ public class DeploymentData { return dryRun; } + private static <T> Supplier<T> wrap(Supplier<T> delegate, Duration timeout, String description) { + return new TimingSupplier<>(new Memoized<>(delegate), timeout, description); + } + + public static class TimingSupplier<T> implements Supplier<T> { + + private final Supplier<T> delegate; + private final Duration timeout; + private final String description; + + public TimingSupplier(Supplier<T> delegate, Duration timeout, String description) { + this.delegate = delegate; + this.timeout = timeout; + this.description = description; + } + + @Override + public T get() { + long startNanos = System.nanoTime(); + Throwable thrown = null; + try { + return delegate.get(); + } + catch (Throwable t) { + thrown = t; + throw t; + } + finally { + long durationNanos = System.nanoTime() - startNanos; + Level level = durationNanos > timeout.toNanos() ? Level.WARNING : Level.FINE; + String thrownMessage = thrown == null ? "" : " with exception " + thrown; + log.log(level, () -> String.format("Getting %s took %.6f seconds%s", description, durationNanos / 1e9, thrownMessage)); + } + } + + } + } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/ConsoleUrls.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/ConsoleUrls.java new file mode 100644 index 00000000000..82cddb46d9a --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/ConsoleUrls.java @@ -0,0 +1,89 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.integration; + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.ApplicationName; +import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.Environment; +import com.yahoo.config.provision.TenantName; +import com.yahoo.config.provision.zone.ZoneId; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; + +import java.net.URI; +import java.net.URLEncoder; +import java.nio.charset.StandardCharsets; + +/** + * Generates URLs to various views in the Console. Prefer to create new methods and return + * String instead of URI to make it easier to track which views are linked from where. + * + * @author freva + */ +public class ConsoleUrls { + private final String root; + public ConsoleUrls(URI root) { + this.root = root.toString().replaceFirst("/$", ""); // Remove trailing slash + } + + public String root() { + return root; + } + + public String tenantOverview(TenantName tenantName) { + return "%s/tenant/%s".formatted(root, tenantName.value()); + } + + /** Returns URL to notification settings view for the given tenant */ + public String tenantNotifications(TenantName tenantName) { + return "%s/tenant/%s/account/notifications".formatted(root, tenantName.value()); + } + + public String tenantBilling(TenantName t) { return "%s/tenant/%s/account/billing".formatted(root, t.value()); } + + public String prodApplicationOverview(TenantName tenantName, ApplicationName applicationName) { + return "%s/tenant/%s/application/%s/prod/instance".formatted(root, tenantName.value(), applicationName.value()); + } + + public String instanceOverview(ApplicationId application, Environment environment) { + return "%s/tenant/%s/application/%s/%s/instance/%s".formatted(root, + application.tenant().value(), + application.application().value(), + environment.isManuallyDeployed() ? environment.value() : "prod", + application.instance().value()); + } + + public String clusterOverview(ApplicationId application, ZoneId zone, ClusterSpec.Id clusterId) { + return cluster(application, zone, clusterId, null); + } + + public String clusterReindexing(ApplicationId application, ZoneId zone, ClusterSpec.Id clusterId) { + return cluster(application, zone, clusterId, "reindexing"); + } + + public String deploymentRun(RunId id) { + return "%s/job/%s/run/%s".formatted( + instanceOverview(id.application(), id.type().environment()), id.type().jobName(), id.number()); + } + + /** Returns URL used to request support from the Vespa team. */ + public String support() { + return root + "/support"; + } + + /** Returns URL to verify an email address with the given verification code */ + public String verifyEmail(String verifyCode) { + return "%s/verify?%s".formatted(root, queryParam("code", verifyCode)); + } + + public String termsOfService() { return root + "/terms-of-service-trial.html"; } + + private String cluster(ApplicationId application, ZoneId zone, ClusterSpec.Id clusterId, String viewOrNull) { + return instanceOverview(application, zone.environment()) + '?' + + queryParam("%s.%s.%s".formatted(application.instance().value(), zone.environment().value(), zone.region().value()), + "clusters," + clusterId.value() + (viewOrNull == null ? "" : '=' + viewOrNull)); + } + + private static String queryParam(String key, String value) { + return URLEncoder.encode(key, StandardCharsets.UTF_8) + '=' + URLEncoder.encode(value, StandardCharsets.UTF_8); + } +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/MockPricingController.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/MockPricingController.java deleted file mode 100644 index f72f80155ed..00000000000 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/MockPricingController.java +++ /dev/null @@ -1,35 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.api.integration; - -import com.yahoo.config.provision.ClusterResources; -import com.yahoo.vespa.hosted.controller.api.integration.billing.Plan; -import com.yahoo.vespa.hosted.controller.api.integration.pricing.PriceInformation; -import com.yahoo.vespa.hosted.controller.api.integration.pricing.PricingController; -import com.yahoo.vespa.hosted.controller.api.integration.pricing.PricingInfo; - -import java.math.BigDecimal; -import java.util.List; - -import static com.yahoo.vespa.hosted.controller.api.integration.pricing.PricingInfo.SupportLevel.BASIC; - -public class MockPricingController implements PricingController { - - @Override - public PriceInformation price(List<ClusterResources> clusterResources, PricingInfo pricingInfo, Plan plan) { - BigDecimal listPrice = BigDecimal.valueOf(clusterResources.stream() - .mapToDouble(resources -> resources.nodes() * - (resources.nodeResources().vcpu() * 1000 + - resources.nodeResources().memoryGb() * 100 + - resources.nodeResources().diskGb() * 10)) - .sum()); - - BigDecimal supportLevelCost = pricingInfo.supportLevel() == BASIC ? new BigDecimal("-160.00") : new BigDecimal("800.00"); - BigDecimal listPriceWithSupport = listPrice.add(supportLevelCost); - BigDecimal enclaveDiscount = pricingInfo.enclave() ? new BigDecimal("-15.1234") : BigDecimal.ZERO; - BigDecimal volumeDiscount = new BigDecimal("-5.64315634"); - BigDecimal committedAmountDiscount = new BigDecimal("-1.23"); - BigDecimal totalAmount = listPrice.add(supportLevelCost).add(enclaveDiscount).add(volumeDiscount).add(committedAmountDiscount); - return new PriceInformation(listPriceWithSupport, volumeDiscount, committedAmountDiscount, enclaveDiscount, totalAmount); - } - -} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/ServiceRegistry.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/ServiceRegistry.java index 1c5f5f972cd..e39a8cf38b7 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/ServiceRegistry.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/ServiceRegistry.java @@ -30,11 +30,10 @@ import com.yahoo.vespa.hosted.controller.api.integration.organization.IssueHandl import com.yahoo.vespa.hosted.controller.api.integration.organization.Mailer; import com.yahoo.vespa.hosted.controller.api.integration.organization.OwnershipIssues; import com.yahoo.vespa.hosted.controller.api.integration.organization.SystemMonitor; -import com.yahoo.vespa.hosted.controller.api.integration.pricing.PricingController; import com.yahoo.vespa.hosted.controller.api.integration.resource.CostReportConsumer; import com.yahoo.vespa.hosted.controller.api.integration.resource.ResourceDatabaseClient; -import com.yahoo.vespa.hosted.controller.api.integration.secrets.GcpSecretStore; import com.yahoo.vespa.hosted.controller.api.integration.secrets.EndpointSecretManager; +import com.yahoo.vespa.hosted.controller.api.integration.secrets.GcpSecretStore; import com.yahoo.vespa.hosted.controller.api.integration.secrets.TenantSecretService; import com.yahoo.vespa.hosted.controller.api.integration.user.RoleMaintainer; import com.yahoo.vespa.hosted.controller.api.integration.vcmr.ChangeRequestClient; @@ -91,6 +90,8 @@ public interface ServiceRegistry { ZoneRegistry zoneRegistry(); + ConsoleUrls consoleUrls(); + ResourceTagger resourceTagger(); EnclaveAccessService enclaveAccessService(); @@ -127,6 +128,4 @@ public interface ServiceRegistry { BillingReporter billingReporter(); - PricingController pricingController(); - } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/AcceptedCountries.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/AcceptedCountries.java new file mode 100644 index 00000000000..c665b4fb7c2 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/AcceptedCountries.java @@ -0,0 +1,23 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package com.yahoo.vespa.hosted.controller.api.integration.billing; + +import java.util.List; + +/** + * @author bjorncs + */ +public record AcceptedCountries(List<Country> countries) { + + public AcceptedCountries { + countries = List.copyOf(countries); + } + + public record Country(String code, String displayName, List<TaxType> taxTypes) { + public Country { + taxTypes = List.copyOf(taxTypes); + } + } + + public record TaxType(String id, String description, String pattern, String example) {} +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/Bill.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/Bill.java index 1acb4964ea6..e7959d2057a 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/Bill.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/Bill.java @@ -8,16 +8,11 @@ import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.zone.ZoneId; import java.math.BigDecimal; -import java.time.Clock; import java.time.LocalDate; -import java.time.ZoneOffset; import java.time.ZonedDateTime; import java.util.List; -import java.util.Map; import java.util.Objects; import java.util.Optional; -import java.util.SortedMap; -import java.util.TreeMap; import java.util.UUID; import java.util.function.Function; @@ -69,7 +64,7 @@ public class Bill { return tenant; } - public String status() { + public BillStatus status() { return statusHistory.current(); } @@ -389,28 +384,4 @@ public class Bill { } } - public static class StatusHistory { - SortedMap<ZonedDateTime, String> history; - - public StatusHistory(SortedMap<ZonedDateTime, String> history) { - this.history = history; - } - - public static StatusHistory open(Clock clock) { - var now = clock.instant().atZone(ZoneOffset.UTC); - return new StatusHistory( - new TreeMap<>(Map.of(now, "OPEN")) - ); - } - - public String current() { - return history.get(history.lastKey()); - } - - public SortedMap<ZonedDateTime, String> getHistory() { - return history; - } - - } - } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/BillStatus.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/BillStatus.java new file mode 100644 index 00000000000..4f35b47219a --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/BillStatus.java @@ -0,0 +1,33 @@ +package com.yahoo.vespa.hosted.controller.api.integration.billing; + +/** + * @author gjoranv + */ +public enum BillStatus { + OPEN, // All bills start in this state. The bill can be modified and exported/synced to external systems. + FROZEN, // Syncing to external systems is switched off. No changes can be made. + CLOSED, // End state for a valid bill. + VOID; // End state, indicating that the bill is not valid. + + // Legacy states, used by historical bills + private static final String LEGACY_ISSUED = "ISSUED"; + private static final String LEGACY_EXPORTED = "EXPORTED"; + private static final String LEGACY_CANCELED = "CANCELED"; + + private final String value; + + BillStatus() { + this.value = name(); + } + + public String value() { + return value; + } + + public static BillStatus from(String status) { + if (LEGACY_ISSUED.equals(status) || LEGACY_EXPORTED.equals(status)) return OPEN; + if (LEGACY_CANCELED.equals(status)) return VOID; + return Enum.valueOf(BillStatus.class, status.toUpperCase()); + } + +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/BillingController.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/BillingController.java index 95b2ba9f8f8..8b48c72f88e 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/BillingController.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/BillingController.java @@ -2,7 +2,7 @@ package com.yahoo.vespa.hosted.controller.api.integration.billing; import com.yahoo.config.provision.TenantName; -import com.yahoo.vespa.hosted.controller.tenant.CloudTenant; +import com.yahoo.vespa.hosted.controller.tenant.TaxId; import java.math.BigDecimal; import java.time.LocalDate; @@ -90,7 +90,7 @@ public interface BillingController { boolean deleteInstrument(TenantName tenant, String userId, String instrumentId); /** Change the status of the given bill */ - void updateBillStatus(Bill.Id billId, String agent, String status); + void updateBillStatus(Bill.Id billId, String agent, BillStatus status); /** Add a line item to the given bill */ void addLineItem(TenantName tenant, String description, BigDecimal amount, Optional<Bill.Id> billId, String agent); @@ -130,7 +130,10 @@ public interface BillingController { default void updateCache(List<TenantName> tenants) {} - default String exportBill(Bill bill, String exportMethod, CloudTenant tenant) { - return "NOT_IMPLEMENTED"; - } + /** Get the list of countries that are accepted */ + AcceptedCountries getAcceptedCountries(); + + /** Validation of tax id */ + void validateTaxId(TaxId id) throws IllegalArgumentException; + } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/BillingDatabaseClient.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/BillingDatabaseClient.java index 3e24314ba5c..c5859cd7d2f 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/BillingDatabaseClient.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/BillingDatabaseClient.java @@ -69,7 +69,7 @@ public interface BillingDatabaseClient { * @param agent The agent that added the status * @param status The new status of the bill */ - void setStatus(Bill.Id billId, String agent, String status); + void setStatus(Bill.Id billId, String agent, BillStatus status); List<Bill.LineItem> getUnusedLineItems(TenantName tenantName); diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/BillingDatabaseClientMock.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/BillingDatabaseClientMock.java index 300c1658c29..a6bcc9bf0ed 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/BillingDatabaseClientMock.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/BillingDatabaseClientMock.java @@ -26,9 +26,10 @@ public class BillingDatabaseClientMock implements BillingDatabaseClient { private final Map<Bill.Id, List<Bill.LineItem>> lineItems = new HashMap<>(); private final Map<TenantName, List<Bill.LineItem>> uncommittedLineItems = new HashMap<>(); - private final Map<Bill.Id, Bill.StatusHistory> statuses = new HashMap<>(); + private final Map<Bill.Id, StatusHistory> statuses = new HashMap<>(); private final Map<Bill.Id, ZonedDateTime> startTimes = new HashMap<>(); private final Map<Bill.Id, ZonedDateTime> endTimes = new HashMap<>(); + private final Map<Bill.Id, String> exportedInvoiceIds = new HashMap<>(); private final ZonedDateTime startTime = LocalDate.of(2020, 4, 1).atStartOfDay(ZoneId.of("UTC")); private final ZonedDateTime endTime = LocalDate.of(2020, 5, 1).atStartOfDay(ZoneId.of("UTC")); @@ -53,7 +54,7 @@ public class BillingDatabaseClientMock implements BillingDatabaseClient { .findFirst(); } - public String getStatus(Bill.Id invoiceId) { + public BillStatus getStatus(Bill.Id invoiceId) { return statuses.get(invoiceId).current(); } @@ -61,7 +62,7 @@ public class BillingDatabaseClientMock implements BillingDatabaseClient { public Bill.Id createBill(TenantName tenant, ZonedDateTime startTime, ZonedDateTime endTime, String agent) { var invoiceId = Bill.Id.generate(); invoices.put(invoiceId, tenant); - statuses.computeIfAbsent(invoiceId, l -> Bill.StatusHistory.open(clock)); + statuses.computeIfAbsent(invoiceId, l -> StatusHistory.open(clock)); startTimes.put(invoiceId, startTime); endTimes.put(invoiceId, endTime); return invoiceId; @@ -71,10 +72,11 @@ public class BillingDatabaseClientMock implements BillingDatabaseClient { public Optional<Bill> readBill(Bill.Id billId) { var invoice = Optional.ofNullable(invoices.get(billId)); var lines = lineItems.getOrDefault(billId, List.of()); - var status = statuses.getOrDefault(billId, Bill.StatusHistory.open(clock)); + var status = statuses.getOrDefault(billId, StatusHistory.open(clock)); var start = startTimes.getOrDefault(billId, startTime); var end = endTimes.getOrDefault(billId, endTime); - return invoice.map(tenant -> new Bill(billId, tenant, status, lines, start, end)); + var exportedId = exportedInvoiceId(billId); + return invoice.map(tenant -> new Bill(billId, tenant, status, lines, start, end, exportedId)); } @Override @@ -88,8 +90,8 @@ public class BillingDatabaseClientMock implements BillingDatabaseClient { } @Override - public void setStatus(Bill.Id invoiceId, String agent, String status) { - statuses.computeIfAbsent(invoiceId, k -> Bill.StatusHistory.open(clock)) + public void setStatus(Bill.Id invoiceId, String agent, BillStatus status) { + statuses.computeIfAbsent(invoiceId, k -> StatusHistory.open(clock)) .getHistory() .put(ZonedDateTime.now(), status); } @@ -157,7 +159,7 @@ public class BillingDatabaseClientMock implements BillingDatabaseClient { var status = statuses.get(invoiceId); var start = startTimes.get(invoiceId); var end = endTimes.get(invoiceId); - return new Bill(invoiceId, tenant, status, items, start, end); + return new Bill(invoiceId, tenant, status, items, start, end, exportedInvoiceId(invoiceId)); }) .toList(); } @@ -171,7 +173,7 @@ public class BillingDatabaseClientMock implements BillingDatabaseClient { var status = statuses.get(invoiceId); var start = startTimes.get(invoiceId); var end = endTimes.get(invoiceId); - return new Bill(invoiceId, tenant, status, items, start, end); + return new Bill(invoiceId, tenant, status, items, start, end, exportedInvoiceId(invoiceId)); }) .toList(); } @@ -180,9 +182,14 @@ public class BillingDatabaseClientMock implements BillingDatabaseClient { public void maintain() {} @Override - public void setExportedInvoiceId(Bill.Id billId, String invoiceId) { } + public void setExportedInvoiceId(Bill.Id billId, String invoiceId) { + exportedInvoiceIds.put(billId, invoiceId); + } @Override public void setExportedInvoiceItemId(String lineItemId, String invoiceItemId) { } + private String exportedInvoiceId(Bill.Id billId) { + return exportedInvoiceIds.getOrDefault(billId, null); + } } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/BillingReporter.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/BillingReporter.java index 7339555e578..676c29cec5d 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/BillingReporter.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/BillingReporter.java @@ -6,4 +6,12 @@ import com.yahoo.vespa.hosted.controller.tenant.CloudTenant; public interface BillingReporter { BillingReference maintainTenant(CloudTenant tenant); + + InvoiceUpdate maintainInvoice(Bill bill); + + /** Export a bill to a payment service. Returns the invoice ID in the external system. */ + default String exportBill(Bill bill, String exportMethod, CloudTenant tenant) { + return "NOT_IMPLEMENTED"; + } + } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/BillingReporterMock.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/BillingReporterMock.java index 29c7fbbf410..689ecc356dc 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/BillingReporterMock.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/BillingReporterMock.java @@ -4,18 +4,42 @@ package com.yahoo.vespa.hosted.controller.api.integration.billing; import com.yahoo.vespa.hosted.controller.tenant.BillingReference; import com.yahoo.vespa.hosted.controller.tenant.CloudTenant; +import java.math.BigDecimal; import java.time.Clock; +import java.time.ZonedDateTime; +import java.util.Optional; import java.util.UUID; public class BillingReporterMock implements BillingReporter { private final Clock clock; + private final BillingDatabaseClient dbClient; - public BillingReporterMock(Clock clock) { + public BillingReporterMock(Clock clock, BillingDatabaseClient dbClient) { this.clock = clock; + this.dbClient = dbClient; } @Override public BillingReference maintainTenant(CloudTenant tenant) { return new BillingReference(UUID.randomUUID().toString(), clock.instant()); } + + @Override + public InvoiceUpdate maintainInvoice(Bill bill) { + dbClient.addLineItem(bill.tenant(), maintainedMarkerItem(), Optional.of(bill.id())); + return new InvoiceUpdate(1,0,0); + } + + @Override + public String exportBill(Bill bill, String exportMethod, CloudTenant tenant) { + // Replace bill with a copy with exportedId set + var exportedId = "EXT-ID-123"; + dbClient.setExportedInvoiceId(bill.id(), exportedId); + return exportedId; + } + + private static Bill.LineItem maintainedMarkerItem() { + return new Bill.LineItem("maintained", "", BigDecimal.valueOf(0.0), "", "", ZonedDateTime.now()); + } + } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/CostCalculator.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/CostCalculator.java index e7f87d3a628..ddcd5308986 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/CostCalculator.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/CostCalculator.java @@ -5,6 +5,8 @@ import com.yahoo.config.provision.NodeResources; import com.yahoo.vespa.hosted.controller.api.integration.resource.CostInfo; import com.yahoo.vespa.hosted.controller.api.integration.resource.ResourceUsage; +import java.math.BigDecimal; + /** * @author ogronnesby */ @@ -16,4 +18,15 @@ public interface CostCalculator { /** Estimate the cost for the given resources */ double calculate(NodeResources resources); + /** CPU unit price */ + BigDecimal getCpuPrice(); + + /** Memory unit price */ + BigDecimal getMemoryPrice(); + + /** Disk unit price */ + BigDecimal getDiskPrice(); + + /** GPU unit price */ + BigDecimal getGpuPrice(); } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/InvoiceUpdate.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/InvoiceUpdate.java new file mode 100644 index 00000000000..6ca3cf6ebb1 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/InvoiceUpdate.java @@ -0,0 +1,45 @@ +package com.yahoo.vespa.hosted.controller.api.integration.billing; + +/** + * Helper to track changes to an invoice. + * + * @author gjoranv + */ +public record InvoiceUpdate(int itemsAdded, int itemsRemoved, int itemsModified) { + public boolean isEmpty() { + return itemsAdded == 0 && itemsRemoved == 0 && itemsModified == 0; + } + + public static InvoiceUpdate empty() { + return new InvoiceUpdate(0, 0, 0); + } + + public static class Counter { + private int itemsAdded = 0; + private int itemsRemoved = 0; + private int itemsModified = 0; + + public void addedItem() { + itemsAdded++; + } + + public void removedItem() { + itemsRemoved++; + } + + public void modifiedItem() { + itemsModified++; + } + + public void add(InvoiceUpdate other) { + itemsAdded += other.itemsAdded; + itemsRemoved += other.itemsRemoved; + itemsModified += other.itemsModified; + } + + public InvoiceUpdate finish() { + return new InvoiceUpdate(itemsAdded, itemsRemoved, itemsModified); + } + } + +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/MockBillingController.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/MockBillingController.java index 8ef14dd60ba..18dd339b4a1 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/MockBillingController.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/MockBillingController.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.hosted.controller.api.integration.billing; import com.yahoo.config.provision.TenantName; +import com.yahoo.vespa.hosted.controller.tenant.TaxId; import java.math.BigDecimal; import java.time.Clock; @@ -22,6 +23,7 @@ import java.util.stream.Stream; public class MockBillingController implements BillingController { private final Clock clock; + private final BillingDatabaseClient dbClient; PlanId defaultPlan = PlanId.from("trial"); List<TenantName> tenants = new ArrayList<>(); @@ -32,8 +34,9 @@ public class MockBillingController implements BillingController { Map<TenantName, List<Bill.LineItem>> unusedLineItems = new HashMap<>(); Map<TenantName, CollectionMethod> collectionMethod = new HashMap<>(); - public MockBillingController(Clock clock) { + public MockBillingController(Clock clock, BillingDatabaseClient dbClient) { this.clock = clock; + this.dbClient = dbClient; } @Override @@ -71,7 +74,7 @@ public class MockBillingController implements BillingController { .add(new Bill( billId, tenant, - Bill.StatusHistory.open(clock), + StatusHistory.open(clock), List.of(), startTime, endTime @@ -116,7 +119,7 @@ public class MockBillingController implements BillingController { } @Override - public void updateBillStatus(Bill.Id billId, String agent, String status) { + public void updateBillStatus(Bill.Id billId, String agent, BillStatus status) { var now = clock.instant().atZone(ZoneOffset.UTC); committedBills.values().stream() .flatMap(List::stream) @@ -134,7 +137,7 @@ public class MockBillingController implements BillingController { "line-item-id", description, amount, - "some-plan", + "paid", agent, ZonedDateTime.now())); } @@ -203,6 +206,29 @@ public class MockBillingController implements BillingController { return count < limit; } + @Override + public AcceptedCountries getAcceptedCountries() { + return new AcceptedCountries(List.of( + new AcceptedCountries.Country( + "NO", "Norway", + List.of(new AcceptedCountries.TaxType("no_vat", "Norwegian VAT number", "[0-9]{9}MVA", "123456789MVA"))), + new AcceptedCountries.Country( + "CA", "Canada", + List.of(new AcceptedCountries.TaxType("ca_gst_hst", "Canadian GST/HST number", "([0-9]{9}) ?RT ?([0-9]{4})", "123456789RT0002"), + new AcceptedCountries.TaxType("ca_pst_bc", "Canadian PST number (British Columbia)", "PST-?([0-9]{4})-?([0-9]{4})", "PST-1234-5678"))) + )); + } + + @Override + public void validateTaxId(TaxId id) throws IllegalArgumentException { + if (id.isEmpty() || id.isLegacy()) return; + if (!List.of("eu_vat", "no_vat").contains(id.type().value())) + throw new IllegalArgumentException("Unknown tax id type '%s'".formatted(id.type().value())); + if (!id.code().value().matches("\\w+")) + throw new IllegalArgumentException("Invalid tax id code '%s'".formatted(id.code().value())); + } + + public void setTenants(List<TenantName> tenants) { this.tenants = tenants; } @@ -234,6 +260,6 @@ public class MockBillingController implements BillingController { private Bill emptyBill() { var start = clock.instant().atZone(ZoneOffset.UTC); var end = clock.instant().atZone(ZoneOffset.UTC); - return new Bill(Bill.Id.of("empty"), TenantName.defaultName(), Bill.StatusHistory.open(clock), List.of(), start, end); + return new Bill(Bill.Id.of("empty"), TenantName.defaultName(), StatusHistory.open(clock), List.of(), start, end); } } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/PlanRegistry.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/PlanRegistry.java index 686a239a138..c0bd0dd29cd 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/PlanRegistry.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/PlanRegistry.java @@ -20,6 +20,10 @@ public interface PlanRegistry { /** Get a set of all plans */ List<Plan> all(); + default Plan require(String planId) { + return plan(planId).orElseThrow(); + } + /** Get a plan give a plan ID */ default Optional<Plan> plan(String planId) { if (planId == null || planId.isBlank()) diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/PlanRegistryMock.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/PlanRegistryMock.java index 3ae2b0aa495..5af4d0cff29 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/PlanRegistryMock.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/PlanRegistryMock.java @@ -144,5 +144,25 @@ public class PlanRegistryMock implements PlanRegistry { public double calculate(NodeResources resources) { return resources.cost(); } + + @Override + public BigDecimal getCpuPrice() { + return cpuHourCost; + } + + @Override + public BigDecimal getMemoryPrice() { + return memHourCost; + } + + @Override + public BigDecimal getDiskPrice() { + return dgbHourCost; + } + + @Override + public BigDecimal getGpuPrice() { + return gpuHourCost; + } } } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/StatusHistory.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/StatusHistory.java new file mode 100644 index 00000000000..f0c7f806c8c --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/StatusHistory.java @@ -0,0 +1,61 @@ +package com.yahoo.vespa.hosted.controller.api.integration.billing; + +import java.time.Clock; +import java.time.ZoneOffset; +import java.time.ZonedDateTime; +import java.util.Map; +import java.util.SortedMap; +import java.util.TreeMap; + +/** + * @author ogronnesby + */ +public class StatusHistory { + SortedMap<ZonedDateTime, BillStatus> history; + + public StatusHistory(SortedMap<ZonedDateTime, BillStatus> history) { + // Validate the given history + var iter = history.values().iterator(); + BillStatus next = iter.hasNext() ? iter.next() : null; + while (iter.hasNext()) { + var current = next; + next = iter.next(); + if (! validateStatus(current, next)) { + throw new IllegalArgumentException("Invalid transition from " + current + " to " + next); + } + } + + this.history = history; + } + + public static StatusHistory open(Clock clock) { + var now = clock.instant().atZone(ZoneOffset.UTC); + return new StatusHistory( + new TreeMap<>(Map.of(now, BillStatus.OPEN)) + ); + } + + public BillStatus current() { + return history.get(history.lastKey()); + } + + public SortedMap<ZonedDateTime, BillStatus> getHistory() { + return history; + } + + public void checkValidTransition(BillStatus newStatus) { + if (! validateStatus(current(), newStatus)) { + throw new IllegalArgumentException("Invalid transition from " + current() + " to " + newStatus); + } + } + + private static boolean validateStatus(BillStatus current, BillStatus newStatus) { + return switch(current) { + case OPEN -> true; + case FROZEN -> newStatus != BillStatus.OPEN; // This could be subject to change. + case CLOSED -> newStatus == BillStatus.CLOSED; + case VOID -> newStatus == BillStatus.VOID; + }; + } + +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/EndpointCertificateValidatorImpl.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/EndpointCertificateValidatorImpl.java index 421ec99d6f2..13fa6c862a7 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/EndpointCertificateValidatorImpl.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/EndpointCertificateValidatorImpl.java @@ -67,7 +67,7 @@ public class EndpointCertificateValidatorImpl implements EndpointCertificateVali } catch (SecretNotFoundException s) { // Normally because the cert is in the process of being provisioned - this will cause a retry in InternalStepRunner - throw new EndpointCertificateException(EndpointCertificateException.Type.CERT_NOT_AVAILABLE, "Certificate not found in secret store"); + throw new EndpointCertificateException(EndpointCertificateException.Type.CERT_NOT_AVAILABLE, "Certificate not found in secret store", s); } catch (EndpointCertificateException e) { if (!e.type().equals(EndpointCertificateException.Type.CERT_NOT_AVAILABLE)) { // such failures are normal and will be retried, it takes some time to show up in the secret store log.log(Level.WARNING, "Certificate validation failure for " + serializedInstanceId, e); diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/DeploymentFailureMails.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/DeploymentFailureMails.java index ef08c3a9adc..72728966dbc 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/DeploymentFailureMails.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/DeploymentFailureMails.java @@ -1,9 +1,9 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.api.integration.organization; +import com.yahoo.vespa.hosted.controller.api.integration.ConsoleUrls; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; -import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneRegistry; import java.util.Collection; @@ -14,10 +14,10 @@ import java.util.Collection; */ public class DeploymentFailureMails { - private final ZoneRegistry registry; + private final ConsoleUrls consoleUrls; - public DeploymentFailureMails(ZoneRegistry registry) { - this.registry = registry; + public DeploymentFailureMails(ConsoleUrls consoleUrls) { + this.consoleUrls = consoleUrls; } public Mail nodeAllocationFailure(RunId id, Collection<String> recipients) { @@ -66,8 +66,8 @@ public class DeploymentFailureMails { jobToString(id.type()), id.application(), messageDetail, - registry.dashboardUrl(id), - registry.supportUrl())); + consoleUrls.deploymentRun(id), + consoleUrls.support())); } private String jobToString(JobType type) { diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/pricing/PriceInformation.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/pricing/PriceInformation.java deleted file mode 100644 index 887741f9196..00000000000 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/pricing/PriceInformation.java +++ /dev/null @@ -1,9 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.api.integration.pricing; - -import java.math.BigDecimal; - -public record PriceInformation(BigDecimal listPriceWithSupport, BigDecimal volumeDiscount, BigDecimal committedAmountDiscount, - BigDecimal enclaveDiscount, BigDecimal totalAmount) { - -} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/pricing/PricingController.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/pricing/PricingController.java deleted file mode 100644 index d8186f17796..00000000000 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/pricing/PricingController.java +++ /dev/null @@ -1,18 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.api.integration.pricing; - -import com.yahoo.config.provision.ClusterResources; -import com.yahoo.vespa.hosted.controller.api.integration.billing.Plan; - -import java.util.List; - -/** - * A service that calculates price information based on cluster resources, plan, service level etc. - * - * @author hmusum - */ -public interface PricingController { - - PriceInformation price(List<ClusterResources> clusterResources, PricingInfo pricingInfo, Plan plan); - -} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/pricing/PricingInfo.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/pricing/PricingInfo.java deleted file mode 100644 index 938991e2ed7..00000000000 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/pricing/PricingInfo.java +++ /dev/null @@ -1,10 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.api.integration.pricing; - -public record PricingInfo(boolean enclave, SupportLevel supportLevel, double committedHourlyAmount) { - - public enum SupportLevel { BASIC, COMMERCIAL, ENTERPRISE } - - public static PricingInfo empty() { return new PricingInfo(false, SupportLevel.COMMERCIAL, 0); } - -} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/pricing/package-info.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/pricing/package-info.java deleted file mode 100644 index 649ab2a80f4..00000000000 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/pricing/package-info.java +++ /dev/null @@ -1,5 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -@ExportPackage -package com.yahoo.vespa.hosted.controller.api.integration.pricing; - -import com.yahoo.osgi.annotation.ExportPackage; diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/zone/ZoneRegistry.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/zone/ZoneRegistry.java index 747c6b72172..92c0a6b1fbb 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/zone/ZoneRegistry.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/zone/ZoneRegistry.java @@ -1,8 +1,6 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.api.integration.zone; -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.ApplicationName; import com.yahoo.config.provision.AthenzDomain; import com.yahoo.config.provision.CloudAccount; import com.yahoo.config.provision.CloudName; @@ -18,7 +16,6 @@ import com.yahoo.config.provision.zone.ZoneFilter; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.athenz.api.AthenzIdentity; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; import java.net.URI; import java.time.Duration; @@ -97,24 +94,6 @@ public interface ZoneRegistry { /** Returns the routing method used by given zone */ RoutingMethod routingMethod(ZoneId zone); - /** Returns a URL where an informative dashboard can be found. */ - URI dashboardUrl(); - - /** Returns a URL which displays information about the given tenant. */ - URI dashboardUrl(TenantName id); - - /** Returns a URL which displays information about the given application. */ - URI dashboardUrl(TenantName tenantName, ApplicationName applicationName); - - /** Returns a URL which displays information about the given application instance. */ - URI dashboardUrl(ApplicationId id); - - /** Returns a URL which displays information about the given job run. */ - URI dashboardUrl(RunId id); - - /** Returns a URL used to request support from the Vespa team. */ - URI supportUrl(); - /** Returns a URL to the controller's api endpoint */ URI apiUrl(); diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/PathGroup.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/PathGroup.java index 52900f83203..54f53d64f76 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/PathGroup.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/PathGroup.java @@ -72,25 +72,11 @@ enum PathGroup { "/application/v4/tenant/{tenant}/archive-access/aws", "/application/v4/tenant/{tenant}/archive-access/gcp"), - - billingToken(Matcher.tenant, - "/billing/v1/tenant/{tenant}/token"), - - billingInstrument(Matcher.tenant, - "/billing/v1/tenant/{tenant}/instrument/{*}"), - - billingPlan(Matcher.tenant, - "/billing/v1/tenant/{tenant}/plan/{*}"), - - billingCollection(Matcher.tenant, - "/billing/v1/tenant/{tenant}/collection/{*}"), - - billingList(Matcher.tenant, - "/billing/v1/tenant/{tenant}/billing/{*}"), - billing(Matcher.tenant, "/billing/v2/tenant/{tenant}/{*}"), + billingAux("/billing/v2/countries"), + accountant("/billing/v2/accountant/{*}"), userSearch("/user/v1/find"), @@ -247,11 +233,6 @@ enum PathGroup { /** Paths used for receiving payment callbacks */ paymentProcessor("/payment/notification"), - /** Paths used for invoice management */ - hostedAccountant("/billing/v1/invoice/{*}", - "/billing/v1/billing", - "/billing/v1/plans"), - /** Path used for listing endpoint certificate request and re-requesting endpoint certificates */ endpointCertificates("/endpointcertificates/"), @@ -322,20 +303,12 @@ enum PathGroup { static Set<PathGroup> operatorRestrictedPaths() { var paths = billingPathsNoToken(); - paths.add(PathGroup.billingToken); paths.add(accessRequestApproval); return paths; } static Set<PathGroup> billingPathsNoToken() { - return EnumSet.of( - PathGroup.billingCollection, - PathGroup.billingInstrument, - PathGroup.billingList, - PathGroup.billingPlan, - PathGroup.billing, - PathGroup.hostedAccountant - ); + return EnumSet.of(PathGroup.billing, PathGroup.billingAux); } /** Returns whether this group matches path in given context */ diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/Policy.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/Policy.java index 6b5130cf2e5..d1a8b2ef0c3 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/Policy.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/Policy.java @@ -26,10 +26,7 @@ enum Policy { .in(SystemName.all()), Privilege.grant(Action.read) .on(PathGroup.billingPathsNoToken()) - .in(SystemName.all()), - Privilege.grant(Action.read) - .on(PathGroup.billingToken) - .in(SystemName.PublicCd)), + .in(SystemName.all())), /** Full access to everything. */ supporter(Privilege.grant(Action.read) @@ -155,40 +152,14 @@ enum Policy { .on(PathGroup.paymentProcessor) .in(SystemName.PublicCd)), - /** Read your own instrument information */ - paymentInstrumentRead(Privilege.grant(Action.read) - .on(PathGroup.billingInstrument) - .in(SystemName.PublicCd, SystemName.Public)), - - /** Ability to update tenant payment instrument */ - paymentInstrumentUpdate(Privilege.grant(Action.update) - .on(PathGroup.billingInstrument) - .in(SystemName.PublicCd, SystemName.Public)), - - /** Ability to remove your own payment instrument */ - paymentInstrumentDelete(Privilege.grant(Action.delete) - .on(PathGroup.billingInstrument) - .in(SystemName.PublicCd, SystemName.Public)), - - /** Get the token to view instrument form */ - paymentInstrumentCreate(Privilege.grant(Action.read) - .on(PathGroup.billingToken) - .in(SystemName.PublicCd, SystemName.Public)), - /** Ability to update tenant payment instrument */ planUpdate(Privilege.grant(Action.update) - .on(PathGroup.billingPlan, PathGroup.billing) - .in(SystemName.PublicCd, SystemName.Public)), - - /** Ability to update tenant collection method */ - collectionMethodUpdate(Privilege.grant(Action.update) - .on(PathGroup.billingCollection) + .on(PathGroup.billing) .in(SystemName.PublicCd, SystemName.Public)), - /** Read the generated bills */ billingInformationRead(Privilege.grant(Action.read) - .on(PathGroup.billingList, PathGroup.billing) + .on(PathGroup.billing, PathGroup.billingAux) .in(SystemName.PublicCd, SystemName.Public)), accessRequests(Privilege.grant(Action.all()) @@ -197,7 +168,7 @@ enum Policy { /** Invoice management */ hostedAccountant(Privilege.grant(Action.all()) - .on(PathGroup.hostedAccountant, PathGroup.accountant, PathGroup.userSearch) + .on(PathGroup.accountant, PathGroup.userSearch) .in(SystemName.PublicCd, SystemName.Public)), /** Listing endpoint certificates and re-requesting certificates */ diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/RoleDefinition.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/RoleDefinition.java index d57e38df239..31c8560c908 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/RoleDefinition.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/RoleDefinition.java @@ -43,8 +43,6 @@ public enum RoleDefinition { Policy.applicationRead, Policy.deploymentRead, Policy.publicRead, - Policy.paymentInstrumentRead, - Policy.paymentInstrumentDelete, Policy.billingInformationRead, Policy.horizonProxyOperations), @@ -56,8 +54,6 @@ public enum RoleDefinition { Policy.developmentDeployment, Policy.keyManagement, Policy.submission, - Policy.paymentInstrumentRead, - Policy.paymentInstrumentDelete, Policy.billingInformationRead, Policy.secretStoreOperations, Policy.dataplaneToken), @@ -72,7 +68,6 @@ public enum RoleDefinition { Policy.tenantArchiveAccessManagement, Policy.applicationManager, Policy.keyRevokal, - Policy.paymentInstrumentRead, Policy.billingInformationRead, Policy.accessRequests ), @@ -99,7 +94,6 @@ public enum RoleDefinition { paymentProcessor(Policy.paymentProcessor), hostedAccountant(Policy.hostedAccountant, - Policy.collectionMethodUpdate, Policy.planUpdate, Policy.tenantUpdate); diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/SystemFlagsDataArchive.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/SystemFlagsDataArchive.java index b55157b90be..02bb669417c 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/SystemFlagsDataArchive.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/SystemFlagsDataArchive.java @@ -5,7 +5,6 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; -import com.fasterxml.jackson.databind.node.TextNode; import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ApplicationName; @@ -192,7 +191,7 @@ public class SystemFlagsDataArchive { flagData.rules().forEach(rule -> rule.conditions().forEach(condition -> { int force_switch_expression_dummy = switch (condition.type()) { case RELATIONAL -> switch (condition.dimension()) { - case APPLICATION_ID, CLOUD, CLOUD_ACCOUNT, CLUSTER_ID, CLUSTER_TYPE, CONSOLE_USER_EMAIL, + case APPLICATION, CLOUD, CLOUD_ACCOUNT, CLUSTER_ID, CLUSTER_TYPE, CONSOLE_USER_EMAIL, ENVIRONMENT, HOSTNAME, INSTANCE_ID, NODE_TYPE, SYSTEM, TENANT_ID, ZONE_ID -> throw new FlagValidationException(condition.type().toWire() + " " + DimensionHelper.toWire(condition.dimension()) + @@ -207,7 +206,7 @@ public class SystemFlagsDataArchive { }; case WHITELIST, BLACKLIST -> switch (condition.dimension()) { - case APPLICATION_ID -> validateConditionValues(condition, SystemFlagsDataArchive::validateTenantApplication); + case APPLICATION -> validateConditionValues(condition, SystemFlagsDataArchive::validateTenantApplication); case CONSOLE_USER_EMAIL -> validateConditionValues(condition, email -> { if (!email.contains("@")) throw new FlagValidationException("Invalid email address: " + email); diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/tenant/CloudTenant.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/tenant/CloudTenant.java index 4a61ff30c25..9ceeba32061 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/tenant/CloudTenant.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/tenant/CloudTenant.java @@ -51,11 +51,16 @@ public class CloudTenant extends Tenant { /** Creates a tenant with the given name, provided it passes validation. */ public static CloudTenant create(TenantName tenantName, Instant createdAt, Principal creator) { + // Initialize with creator as verified contact + var info = TenantInfo.empty().withContacts(new TenantContacts(List.of( + new TenantContacts.EmailContact( + List.of(TenantContacts.Audience.TENANT, TenantContacts.Audience.NOTIFICATIONS), + new Email(creator.getName(), true))))); return new CloudTenant(requireName(tenantName), createdAt, LastLoginInfo.EMPTY, Optional.ofNullable(creator).map(SimplePrincipal::of), - ImmutableBiMap.of(), TenantInfo.empty(), List.of(), new ArchiveAccess(), Optional.empty(), + ImmutableBiMap.of(), info, List.of(), new ArchiveAccess(), Optional.empty(), Instant.EPOCH, List.of(), Optional.empty(), PlanId.from("none")); } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/tenant/Email.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/tenant/Email.java index 995f1b1864f..702a183e7af 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/tenant/Email.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/tenant/Email.java @@ -25,7 +25,7 @@ public class Email { } public static Email empty() { - return new Email("", true); + return new Email("", false); } public Email withEmailAddress(String emailAddress) { @@ -36,6 +36,10 @@ public class Email { return new Email(emailAddress, isVerified); } + public boolean isBlank() { + return emailAddress.isBlank(); + } + @Override public boolean equals(Object o) { if (this == o) return true; diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/tenant/PendingMailVerification.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/tenant/PendingMailVerification.java index 057c8bad89b..9c4bbc88f1f 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/tenant/PendingMailVerification.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/tenant/PendingMailVerification.java @@ -76,6 +76,7 @@ public class PendingMailVerification { public enum MailType { TENANT_CONTACT, - NOTIFICATIONS + NOTIFICATIONS, + BILLING } } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/tenant/PurchaseOrder.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/tenant/PurchaseOrder.java new file mode 100644 index 00000000000..d222864a388 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/tenant/PurchaseOrder.java @@ -0,0 +1,21 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.tenant; + +import ai.vespa.validation.StringWrapper; + +import static ai.vespa.validation.Validation.requireLength; + +/** + * @author olaa + */ +public class PurchaseOrder extends StringWrapper<PurchaseOrder> { + + public PurchaseOrder(String value) { + super(value); + requireLength(value, "purchase order length", 0, 64); + } + + public static PurchaseOrder empty() { + return new PurchaseOrder(""); + } +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/tenant/TaxId.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/tenant/TaxId.java new file mode 100644 index 00000000000..99c2400c58c --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/tenant/TaxId.java @@ -0,0 +1,41 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.tenant; + +import ai.vespa.validation.StringWrapper; + +import static ai.vespa.validation.Validation.requireLength; + +/** + * @author olaa + */ +public record TaxId(Type type, Code code) { + + public TaxId(String type, String code) { this(new Type(type), new Code(code)); } + + public static TaxId empty() { return new TaxId(Type.empty(), Code.empty()); } + public boolean isEmpty() { return type.isEmpty() && code.isEmpty(); } + + // TODO(bjorncs) Remove legacy once no longer present in ZK + public static TaxId legacy(String code) { return new TaxId(Type.empty(), new Code(code)); } + public boolean isLegacy() { return type.isEmpty() && !code.isEmpty(); } + + public static class Type extends StringWrapper<Type> { + public Type(String value) { + super(value); + requireLength(value, "tax code type length", 0, 16); + } + + public static Type empty() { return new Type(""); } + public boolean isEmpty() { return value().isEmpty(); } + } + + public static class Code extends StringWrapper<Code> { + public Code(String value) { + super(value); + requireLength(value, "tax code value length", 0, 64); + } + + public static Code empty() { return new Code(""); } + public boolean isEmpty() { return value().isEmpty(); } + } +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/tenant/TenantBilling.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/tenant/TenantBilling.java index 5377b820e18..6e3b26661e5 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/tenant/TenantBilling.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/tenant/TenantBilling.java @@ -10,14 +10,20 @@ public class TenantBilling { private final TenantContact contact; private final TenantAddress address; + private final TaxId taxId; + private final PurchaseOrder purchaseOrder; + private final Email invoiceEmail; - public TenantBilling(TenantContact contact, TenantAddress address) { + public TenantBilling(TenantContact contact, TenantAddress address, TaxId taxId, PurchaseOrder purchaseOrder, Email invoiceEmail) { this.contact = Objects.requireNonNull(contact); this.address = Objects.requireNonNull(address); + this.taxId = Objects.requireNonNull(taxId); + this.purchaseOrder = Objects.requireNonNull(purchaseOrder); + this.invoiceEmail = Objects.requireNonNull(invoiceEmail); } public static TenantBilling empty() { - return new TenantBilling(TenantContact.empty(), TenantAddress.empty()); + return new TenantBilling(TenantContact.empty(), TenantAddress.empty(), TaxId.empty(), PurchaseOrder.empty(), Email.empty()); } public TenantContact contact() { @@ -28,12 +34,36 @@ public class TenantBilling { return address; } + public TaxId getTaxId() { + return taxId; + } + + public PurchaseOrder getPurchaseOrder() { + return purchaseOrder; + } + + public Email getInvoiceEmail() { + return invoiceEmail; + } + public TenantBilling withContact(TenantContact updatedContact) { - return new TenantBilling(updatedContact, this.address); + return new TenantBilling(updatedContact, this.address, this.taxId, this.purchaseOrder, this.invoiceEmail); } public TenantBilling withAddress(TenantAddress updatedAddress) { - return new TenantBilling(this.contact, updatedAddress); + return new TenantBilling(this.contact, updatedAddress, this.taxId, this.purchaseOrder, this.invoiceEmail); + } + + public TenantBilling withTaxId(TaxId updatedTaxId) { + return new TenantBilling(this.contact, this.address, updatedTaxId, this.purchaseOrder, this.invoiceEmail); + } + + public TenantBilling withPurchaseOrder(PurchaseOrder updatedPurchaseOrder) { + return new TenantBilling(this.contact, this.address, this.taxId, updatedPurchaseOrder, this.invoiceEmail); + } + + public TenantBilling withInvoiceEmail(Email updatedInvoiceEmail) { + return new TenantBilling(this.contact, this.address, this.taxId, this.purchaseOrder, updatedInvoiceEmail); } public boolean isEmpty() { @@ -45,19 +75,26 @@ public class TenantBilling { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; TenantBilling that = (TenantBilling) o; - return Objects.equals(contact, that.contact) && Objects.equals(address, that.address); + return Objects.equals(contact, that.contact) && + Objects.equals(address, that.address) && + Objects.equals(taxId, that.taxId) && + Objects.equals(purchaseOrder, that.purchaseOrder) && + Objects.equals(invoiceEmail, that.invoiceEmail); } @Override public int hashCode() { - return Objects.hash(contact, address); + return Objects.hash(contact, address, taxId, purchaseOrder, invoiceEmail); } @Override public String toString() { - return "TenantInfoBillingContact{" + + return "TenantBilling{" + "contact=" + contact + ", address=" + address + + ", taxId='" + taxId + '\'' + + ", purchaseOrder='" + purchaseOrder + '\'' + + ", invoiceEmail=" + invoiceEmail + '}'; } } diff --git a/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/integration/ConsoleUrlsTest.java b/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/integration/ConsoleUrlsTest.java new file mode 100644 index 00000000000..259a279671b --- /dev/null +++ b/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/integration/ConsoleUrlsTest.java @@ -0,0 +1,44 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.integration; + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.Environment; +import com.yahoo.config.provision.zone.ZoneId; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; +import org.junit.jupiter.api.Test; + +import java.net.URI; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +/** + * @author freva + */ +class ConsoleUrlsTest { + + private final ConsoleUrls urls = new ConsoleUrls(URI.create("https://console.tld/")); + + @Test + void urls() { + ApplicationId app = ApplicationId.from("t1", "a1", "i1"); + ZoneId prod = ZoneId.from("prod", "us-north-1"); + ZoneId dev = ZoneId.from("dev", "eu-west-2"); + ZoneId test = ZoneId.from("test", "ap-east-3"); + ClusterSpec.Id cluster = ClusterSpec.Id.from("c1"); + + assertEquals("https://console.tld", urls.root()); + assertEquals("https://console.tld/tenant/t1", urls.tenantOverview(app.tenant())); + assertEquals("https://console.tld/tenant/t1/account/notifications", urls.tenantNotifications(app.tenant())); + assertEquals("https://console.tld/tenant/t1/account/billing", urls.tenantBilling(app.tenant())); + assertEquals("https://console.tld/tenant/t1/application/a1/prod/instance", urls.prodApplicationOverview(app.tenant(), app.application())); + assertEquals("https://console.tld/tenant/t1/application/a1/prod/instance/i1", urls.instanceOverview(app, Environment.test)); + assertEquals("https://console.tld/tenant/t1/application/a1/dev/instance/i1?i1.dev.eu-west-2=clusters%2Cc1", urls.clusterOverview(app, dev, cluster)); + assertEquals("https://console.tld/tenant/t1/application/a1/prod/instance/i1?i1.prod.us-north-1=clusters%2Cc1%3Dreindexing", urls.clusterReindexing(app, prod, cluster)); + assertEquals("https://console.tld/tenant/t1/application/a1/prod/instance/i1/job/production-us-north-1/run/1", urls.deploymentRun(new RunId(app, JobType.deploymentTo(prod), 1))); + assertEquals("https://console.tld/tenant/t1/application/a1/prod/instance/i1/job/system-test/run/1", urls.deploymentRun(new RunId(app, JobType.deploymentTo(test), 1))); + assertEquals("https://console.tld/tenant/t1/application/a1/dev/instance/i1/job/dev-eu-west-2/run/1", urls.deploymentRun(new RunId(app, JobType.deploymentTo(dev), 1))); + assertEquals("https://console.tld/verify?code=test123", urls.verifyEmail("test123")); + } +} diff --git a/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/integration/billing/BillStatusTest.java b/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/integration/billing/BillStatusTest.java new file mode 100644 index 00000000000..022474406b9 --- /dev/null +++ b/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/integration/billing/BillStatusTest.java @@ -0,0 +1,19 @@ +package com.yahoo.vespa.hosted.controller.api.integration.billing; + +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +/** + * @author gjoranv + */ +public class BillStatusTest { + + @Test + void legacy_states_are_converted() { + assertEquals(BillStatus.OPEN, BillStatus.from("ISSUED")); + assertEquals(BillStatus.OPEN, BillStatus.from("EXPORTED")); + assertEquals(BillStatus.VOID, BillStatus.from("CANCELED")); + } + +} diff --git a/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/integration/billing/StatusHistoryTest.java b/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/integration/billing/StatusHistoryTest.java new file mode 100644 index 00000000000..46a4c7e199c --- /dev/null +++ b/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/integration/billing/StatusHistoryTest.java @@ -0,0 +1,89 @@ +package com.yahoo.vespa.hosted.controller.api.integration.billing; + +import org.junit.jupiter.api.Test; + +import java.time.Clock; +import java.time.ZonedDateTime; +import java.util.Map; +import java.util.SortedMap; +import java.util.TreeMap; + +import static org.junit.jupiter.api.Assertions.assertThrows; + +/** + * @author gjoranv + */ +public class StatusHistoryTest { + + private final Clock clock = Clock.systemUTC(); + + @Test + void open_can_change_to_any_status() { + var history = StatusHistory.open(clock); + history.checkValidTransition(BillStatus.FROZEN); + history.checkValidTransition(BillStatus.CLOSED); + history.checkValidTransition(BillStatus.VOID); + } + + @Test + void frozen_cannot_change_to_open() { + var history = new StatusHistory(historyWith(BillStatus.FROZEN)); + + history.checkValidTransition(BillStatus.CLOSED); + history.checkValidTransition(BillStatus.VOID); + + assertThrows(IllegalArgumentException.class, () -> history.checkValidTransition(BillStatus.OPEN)); + } + + @Test + void closed_cannot_change() { + var history = new StatusHistory(historyWith(BillStatus.CLOSED)); + + assertThrows(IllegalArgumentException.class, () -> history.checkValidTransition(BillStatus.OPEN)); + assertThrows(IllegalArgumentException.class, () -> history.checkValidTransition(BillStatus.FROZEN)); + assertThrows(IllegalArgumentException.class, () -> history.checkValidTransition(BillStatus.VOID)); + } + + @Test + void void_cannot_change() { + var history = new StatusHistory(historyWith(BillStatus.VOID)); + + assertThrows(IllegalArgumentException.class, () -> history.checkValidTransition(BillStatus.OPEN)); + assertThrows(IllegalArgumentException.class, () -> history.checkValidTransition(BillStatus.FROZEN)); + assertThrows(IllegalArgumentException.class, () -> history.checkValidTransition(BillStatus.CLOSED)); + } + + @Test + void any_status_can_change_to_itself() { + var history = new StatusHistory(historyWith(BillStatus.OPEN)); + history.checkValidTransition(BillStatus.OPEN); + + history = new StatusHistory(historyWith(BillStatus.FROZEN)); + history.checkValidTransition(BillStatus.FROZEN); + + history = new StatusHistory(historyWith(BillStatus.CLOSED)); + history.checkValidTransition(BillStatus.CLOSED); + + history = new StatusHistory(historyWith(BillStatus.VOID)); + history.checkValidTransition(BillStatus.VOID); + } + + @Test + void it_validates_status_history_in_constructor() { + assertThrows(IllegalArgumentException.class, () -> new StatusHistory(historyWith(BillStatus.FROZEN, BillStatus.OPEN))); + assertThrows(IllegalArgumentException.class, () -> new StatusHistory(historyWith(BillStatus.CLOSED, BillStatus.OPEN))); + assertThrows(IllegalArgumentException.class, () -> new StatusHistory(historyWith(BillStatus.CLOSED, BillStatus.FROZEN))); + assertThrows(IllegalArgumentException.class, () -> new StatusHistory(historyWith(BillStatus.CLOSED, BillStatus.VOID))); + assertThrows(IllegalArgumentException.class, () -> new StatusHistory(historyWith(BillStatus.VOID, BillStatus.OPEN))); + assertThrows(IllegalArgumentException.class, () -> new StatusHistory(historyWith(BillStatus.VOID, BillStatus.FROZEN))); + assertThrows(IllegalArgumentException.class, () -> new StatusHistory(historyWith(BillStatus.VOID, BillStatus.CLOSED))); + } + + private SortedMap<ZonedDateTime, BillStatus> historyWith(BillStatus... statuses) { + var history = new TreeMap<>(Map.of(ZonedDateTime.now(clock), BillStatus.OPEN)); + for (var status : statuses) { + history.put(ZonedDateTime.now(clock), status); + } + return history; + } +} diff --git a/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/role/RoleTest.java b/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/role/RoleTest.java index 24539d7c158..c8020666906 100644 --- a/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/role/RoleTest.java +++ b/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/role/RoleTest.java @@ -8,7 +8,6 @@ import org.junit.jupiter.api.Test; import java.net.URI; import java.util.List; -import java.util.stream.Stream; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -146,139 +145,6 @@ public class RoleTest { } } - @Test - void payment_instrument() { - URI paymentInstrumentUri = URI.create("/billing/v1/tenant/t1/instrument/foobar"); - URI tenantPaymentInstrumentUri = URI.create("/billing/v1/tenant/t1/instrument"); - URI tokenUri = URI.create("/billing/v1/tenant/t1/token"); - - Role user = Role.reader(TenantName.from("t1")); - assertTrue(publicCdEnforcer.allows(user, Action.read, paymentInstrumentUri)); - assertTrue(publicCdEnforcer.allows(user, Action.delete, paymentInstrumentUri)); - assertFalse(publicCdEnforcer.allows(user, Action.update, tenantPaymentInstrumentUri)); - assertFalse(publicCdEnforcer.allows(user, Action.read, tokenUri)); - - Role developer = Role.developer(TenantName.from("t1")); - assertTrue(publicCdEnforcer.allows(developer, Action.read, paymentInstrumentUri)); - assertTrue(publicCdEnforcer.allows(developer, Action.delete, paymentInstrumentUri)); - assertFalse(publicCdEnforcer.allows(developer, Action.update, tenantPaymentInstrumentUri)); - assertFalse(publicCdEnforcer.allows(developer, Action.read, tokenUri)); - - Role admin = Role.administrator(TenantName.from("t1")); - assertTrue(publicCdEnforcer.allows(admin, Action.read, paymentInstrumentUri)); - assertFalse(publicCdEnforcer.allows(admin, Action.delete, paymentInstrumentUri)); - assertFalse(publicCdEnforcer.allows(admin, Action.update, tenantPaymentInstrumentUri)); - assertFalse(publicCdEnforcer.allows(admin, Action.read, tokenUri)); - } - - @Test - void billing_tenant() { - URI billing = URI.create("/billing/v1/tenant/t1/billing"); - - Role user = Role.reader(TenantName.from("t1")); - Role developer = Role.developer(TenantName.from("t1")); - Role admin = Role.administrator(TenantName.from("t1")); - - Stream.of(user, developer, admin).forEach(role -> { - assertTrue(publicCdEnforcer.allows(role, Action.read, billing)); - assertFalse(publicCdEnforcer.allows(role, Action.update, billing)); - assertFalse(publicCdEnforcer.allows(role, Action.delete, billing)); - assertFalse(publicCdEnforcer.allows(role, Action.create, billing)); - }); - - } - - @Test - void billing_test() { - var tester = new EnforcerTester(publicEnforcer); - - var accountant = Role.hostedAccountant(); - var operator = Role.hostedOperator(); - var reader = Role.reader(TenantName.from("t1")); - var developer = Role.developer(TenantName.from("t1")); - var admin = Role.administrator(TenantName.from("t1")); - var otherAdmin = Role.administrator(TenantName.from("t2")); - - tester.on("/billing/v1/tenant/t1/token") - .assertAction(accountant) - .assertAction(operator) - .assertAction(reader) - .assertAction(developer) - .assertAction(otherAdmin); - - tester.on("/billing/v1/tenant/t1/instrument") - .assertAction(accountant) - .assertAction(operator, Action.read) - .assertAction(reader, Action.read, Action.delete) - .assertAction(developer, Action.read, Action.delete) - .assertAction(admin, Action.read) - .assertAction(otherAdmin); - - tester.on("/billing/v1/tenant/t1/instrument/i1") - .assertAction(accountant) - .assertAction(operator, Action.read) - .assertAction(reader, Action.read, Action.delete) - .assertAction(developer, Action.read, Action.delete) - .assertAction(admin, Action.read) - .assertAction(otherAdmin); - - tester.on("/billing/v1/tenant/t1/billing") - .assertAction(accountant) - .assertAction(operator, Action.read) - .assertAction(reader, Action.read) - .assertAction(developer, Action.read) - .assertAction(admin, Action.read) - .assertAction(otherAdmin); - - tester.on("/billing/v1/tenant/t1/plan") - .assertAction(accountant, Action.update) - .assertAction(operator, Action.read) - .assertAction(reader) - .assertAction(developer) - .assertAction(admin) - .assertAction(otherAdmin); - - tester.on("/billing/v1/tenant/t1/collection") - .assertAction(accountant, Action.update) - .assertAction(operator, Action.read) - .assertAction(reader) - .assertAction(developer) - .assertAction(admin) - .assertAction(otherAdmin); - - tester.on("/billing/v1/billing") - .assertAction(accountant, Action.create, Action.read, Action.update, Action.delete) - .assertAction(operator, Action.read) - .assertAction(reader) - .assertAction(developer) - .assertAction(admin) - .assertAction(otherAdmin); - - tester.on("/billing/v1/invoice/tenant/t1/line-item") - .assertAction(accountant, Action.create, Action.read, Action.update, Action.delete) - .assertAction(operator, Action.read) - .assertAction(reader) - .assertAction(developer) - .assertAction(admin) - .assertAction(otherAdmin); - - tester.on("/billing/v1/invoice") - .assertAction(accountant, Action.create, Action.read, Action.update, Action.delete) - .assertAction(operator, Action.read) - .assertAction(reader) - .assertAction(developer) - .assertAction(admin) - .assertAction(otherAdmin); - - tester.on("/billing/v1/invoice/i1/status") - .assertAction(accountant, Action.create, Action.read, Action.update, Action.delete) - .assertAction(operator, Action.read) - .assertAction(reader) - .assertAction(developer) - .assertAction(admin) - .assertAction(otherAdmin); - } - private static class EnforcerTester { private final Enforcer enforcer; private final URI resource; diff --git a/controller-server/pom.xml b/controller-server/pom.xml index 6671b71c73f..a9db2cace85 100644 --- a/controller-server/pom.xml +++ b/controller-server/pom.xml @@ -118,6 +118,33 @@ <!-- compile --> <dependency> + <groupId>org.apache.velocity</groupId> + <artifactId>velocity-engine-core</artifactId> + <exclusions> + <exclusion> + <!-- Must use the one provided by Jdisc to prevent two instances of slf4j classes. --> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-api</artifactId> + </exclusion> + </exclusions> + </dependency> + <dependency> + <groupId>org.apache.velocity.tools</groupId> + <artifactId>velocity-tools-generic</artifactId> + <exclusions> + <exclusion> + <!-- Must use the one provided by Jdisc to prevent two instances of slf4j classes. --> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-api</artifactId> + </exclusion> + <exclusion> + <groupId>commons-logging</groupId> + <artifactId>commons-logging</artifactId> + </exclusion> + </exclusions> + </dependency> + + <dependency> <groupId>org.apache.commons</groupId> <artifactId>commons-csv</artifactId> </dependency> diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java index 0de0ea06904..d7a3d4fb9e5 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java @@ -554,7 +554,7 @@ public class ApplicationController { if (warnings.isEmpty()) controller.notificationsDb().removeNotification(source, Notification.Type.applicationPackage); else - controller.notificationsDb().setNotification(source, Notification.Type.applicationPackage, Notification.Level.warning, warnings); + controller.notificationsDb().setApplicationPackageNotification(source, warnings); } lockApplicationOrThrow(applicationId, application -> diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java index 87885bc5f21..0b693bb9894 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java @@ -131,10 +131,10 @@ public class Controller extends AbstractComponent { auditLogger = new AuditLogger(curator, clock); jobControl = new JobControl(new JobControlFlags(curator, flagSource)); archiveBucketDb = new CuratorArchiveBucketDb(this); - notifier = new Notifier(curator, serviceRegistry.zoneRegistry(), serviceRegistry.mailer(), flagSource); + notifier = new Notifier(curator, serviceRegistry.consoleUrls(), serviceRegistry.mailer(), flagSource); notificationsDb = new NotificationsDb(this); supportAccessControl = new SupportAccessControl(this); - mailVerifier = new MailVerifier(serviceRegistry.zoneRegistry().dashboardUrl(), tenantController, serviceRegistry.mailer(), curator, clock); + mailVerifier = new MailVerifier(serviceRegistry.consoleUrls(), tenantController, serviceRegistry.mailer(), curator, clock); dataplaneTokenService = new DataplaneTokenService(this); // Record the version of this controller diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java index 51e20d0017c..5dec1449507 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java @@ -14,9 +14,9 @@ import com.yahoo.config.provision.zone.AuthMethod; import com.yahoo.config.provision.zone.RoutingMethod; import com.yahoo.config.provision.zone.ZoneApi; import com.yahoo.config.provision.zone.ZoneId; -import com.yahoo.vespa.flags.BooleanFlag; import com.yahoo.vespa.flags.FetchVector; import com.yahoo.vespa.flags.Flags; +import com.yahoo.vespa.flags.StringFlag; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificate; import com.yahoo.vespa.hosted.controller.api.integration.dns.Record; @@ -64,6 +64,8 @@ import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.TreeMap; +import java.util.logging.Level; +import java.util.logging.Logger; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -79,11 +81,12 @@ import static java.util.stream.Collectors.toMap; */ public class RoutingController { + private static final Logger LOG = Logger.getLogger(RoutingController.class.getName()); + private final Controller controller; private final RoutingPolicies routingPolicies; private final RotationRepository rotationRepository; - private final BooleanFlag generatedEndpoints; - private final BooleanFlag legacyEndpoints; + private final StringFlag endpointConfig; public RoutingController(Controller controller, RotationsConfig rotationsConfig) { this.controller = Objects.requireNonNull(controller, "controller must be non-null"); @@ -91,8 +94,7 @@ public class RoutingController { this.rotationRepository = new RotationRepository(Objects.requireNonNull(rotationsConfig, "rotationsConfig must be non-null"), controller.applications(), controller.curator()); - this.generatedEndpoints = Flags.RANDOMIZED_ENDPOINT_NAMES.bindTo(controller.flagSource()); - this.legacyEndpoints = Flags.LEGACY_ENDPOINTS.bindTo(controller.flagSource()); + this.endpointConfig = Flags.ENDPOINT_CONFIG.bindTo(controller.flagSource()); } /** Create a routing context for given deployment */ @@ -124,15 +126,17 @@ public class RoutingController { /** Returns the endpoint config to use for given instance */ public EndpointConfig endpointConfig(ApplicationId instance) { - // TODO(mpolden): Switch to reading endpoint-config flag - if (legacyEndpointsEnabled(instance)) { - if (generatedEndpointsEnabled(instance)) { - return EndpointConfig.combined; - } else { - return EndpointConfig.legacy; - } - } - return EndpointConfig.generated; + String flagValue = endpointConfig.with(FetchVector.Dimension.TENANT_ID, instance.tenant().value()) + .with(FetchVector.Dimension.APPLICATION, TenantAndApplicationId.from(instance).serialized()) + .with(FetchVector.Dimension.INSTANCE_ID, instance.serializedForm()) + .value(); + return switch (flagValue) { + case "legacy" -> EndpointConfig.legacy; + case "combined" -> EndpointConfig.combined; + case "generated" -> EndpointConfig.generated; + default -> throw new IllegalArgumentException("Invalid endpoint-config flag value: '" + flagValue + "', must be " + + "'legacy', 'combined' or 'generated'"); + }; } /** Prepares and returns the endpoints relevant for given deployment */ @@ -213,6 +217,8 @@ public class RoutingController { // Register rotation-backed endpoints in DNS registerRotationEndpointsInDns(prepared); + LOG.log(Level.FINE, () -> "Prepared endpoints: " + prepared); + return prepared; } @@ -600,20 +606,6 @@ public class RoutingController { return Collections.unmodifiableList(routingMethods); } - private boolean generatedEndpointsEnabled(ApplicationId instance) { - return generatedEndpoints.with(FetchVector.Dimension.INSTANCE_ID, instance.serializedForm()) - .with(FetchVector.Dimension.TENANT_ID, instance.tenant().value()) - .with(FetchVector.Dimension.APPLICATION_ID, TenantAndApplicationId.from(instance).serialized()) - .value(); - } - - private boolean legacyEndpointsEnabled(ApplicationId instance) { - return legacyEndpoints.with(FetchVector.Dimension.INSTANCE_ID, instance.serializedForm()) - .with(FetchVector.Dimension.TENANT_ID, instance.tenant().value()) - .with(FetchVector.Dimension.APPLICATION_ID, TenantAndApplicationId.from(instance).serialized()) - .value(); - } - private static void requireGeneratedEndpoints(GeneratedEndpointList generatedEndpoints, boolean declared) { if (generatedEndpoints.asList().stream().anyMatch(ge -> ge.declared() != declared)) { throw new IllegalStateException("All generated endpoints require declared=" + declared + diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/AssignedRotation.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/AssignedRotation.java index 7fa2a03d0a9..c2949e395e9 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/AssignedRotation.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/AssignedRotation.java @@ -23,16 +23,6 @@ public record AssignedRotation(ClusterSpec.Id clusterId, EndpointId endpointId, this.regions = Set.copyOf(Objects.requireNonNull(regions)); } - @Override - public String toString() { - return "AssignedRotation{" + - "clusterId=" + clusterId + - ", endpointId='" + endpointId + '\'' + - ", rotationId=" + rotationId + - ", regions=" + regions + - '}'; - } - private static <T> T requireNonEmpty(T object, String value, String field) { Objects.requireNonNull(object); Objects.requireNonNull(value); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/MailVerifier.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/MailVerifier.java index 4d29abaa212..9ff3206ee06 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/MailVerifier.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/MailVerifier.java @@ -4,24 +4,23 @@ package com.yahoo.vespa.hosted.controller.application; import com.yahoo.config.provision.TenantName; import com.yahoo.vespa.hosted.controller.LockedTenant; import com.yahoo.vespa.hosted.controller.TenantController; +import com.yahoo.vespa.hosted.controller.api.integration.ConsoleUrls; import com.yahoo.vespa.hosted.controller.api.integration.organization.Mail; import com.yahoo.vespa.hosted.controller.api.integration.organization.Mailer; +import com.yahoo.vespa.hosted.controller.notification.MailTemplating; import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; import com.yahoo.vespa.hosted.controller.tenant.CloudTenant; +import com.yahoo.vespa.hosted.controller.tenant.PendingMailVerification; import com.yahoo.vespa.hosted.controller.tenant.Tenant; import com.yahoo.vespa.hosted.controller.tenant.TenantContacts; import com.yahoo.vespa.hosted.controller.tenant.TenantInfo; -import com.yahoo.vespa.hosted.controller.tenant.PendingMailVerification; -import java.net.URI; import java.time.Clock; import java.time.Duration; import java.util.List; import java.util.Optional; import java.util.UUID; -import static com.yahoo.yolean.Exceptions.uncheck; - /** * @author olaa @@ -34,14 +33,14 @@ public class MailVerifier { private final Mailer mailer; private final CuratorDb curatorDb; private final Clock clock; - private final URI dashboardUri; + private final MailTemplating mailTemplating; - public MailVerifier(URI dashboardUri, TenantController tenantController, Mailer mailer, CuratorDb curatorDb, Clock clock) { + public MailVerifier(ConsoleUrls consoleUrls, TenantController tenantController, Mailer mailer, CuratorDb curatorDb, Clock clock) { this.tenantController = tenantController; this.mailer = mailer; this.curatorDb = curatorDb; this.clock = clock; - this.dashboardUri = dashboardUri; + this.mailTemplating = new MailTemplating(consoleUrls); } public PendingMailVerification sendMailVerification(TenantName tenantName, String email, PendingMailVerification.MailType mailType) { @@ -86,6 +85,7 @@ public class MailVerifier { case NOTIFICATIONS -> withTenantContacts(oldTenantInfo, pendingMailVerification); case TENANT_CONTACT -> oldTenantInfo.withContact(oldTenantInfo.contact() .withEmail(oldTenantInfo.contact().email().withVerification(true))); + case BILLING -> withVerifiedBillingMail(oldTenantInfo); }; tenantController.lockOrThrow(tenant.name(), LockedTenant.Cloud.class, lockedTenant -> { @@ -111,6 +111,13 @@ public class MailVerifier { return oldInfo.withContacts(new TenantContacts(newContacts)); } + private TenantInfo withVerifiedBillingMail(TenantInfo oldInfo) { + var verifiedMail = oldInfo.billingContact().contact().email().withVerification(true); + var billingContact = oldInfo.billingContact() + .withContact(oldInfo.billingContact().contact().withEmail(verifiedMail)); + return oldInfo.withBilling(billingContact); + } + private void writePendingVerification(PendingMailVerification pendingMailVerification) { try (var lock = curatorDb.lockPendingMailVerification(pendingMailVerification.getVerificationCode())) { curatorDb.writePendingMailVerification(pendingMailVerification); @@ -125,12 +132,7 @@ public class MailVerifier { } private Mail mailOf(PendingMailVerification pendingMailVerification) { - var classLoader = this.getClass().getClassLoader(); - var template = uncheck(() -> classLoader.getResourceAsStream("mail/mail-verification.tmpl").readAllBytes()); - var message = new String(template) - .replaceAll("%\\{consoleUrl}", dashboardUri.getHost()) - .replaceAll("%\\{email}", pendingMailVerification.getMailAddress()) - .replaceAll("%\\{code}", pendingMailVerification.getVerificationCode()); + var message = mailTemplating.generateMailVerificationHtml(pendingMailVerification); return new Mail(List.of(pendingMailVerification.getMailAddress()), "Please verify your email", "", message); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java index 1080b379c4d..9bfa2674754 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java @@ -86,7 +86,6 @@ import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.running; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.success; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.testFailure; import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.succeeded; -import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.unfinished; import static com.yahoo.vespa.hosted.controller.deployment.Step.copyVespaLogs; import static com.yahoo.vespa.hosted.controller.deployment.Step.deactivateReal; import static com.yahoo.vespa.hosted.controller.deployment.Step.deactivateTester; @@ -128,7 +127,7 @@ public class InternalStepRunner implements StepRunner { public InternalStepRunner(Controller controller) { this.controller = controller; this.testConfigSerializer = new TestConfigSerializer(controller.system()); - this.mails = new DeploymentFailureMails(controller.zoneRegistry()); + this.mails = new DeploymentFailureMails(controller.serviceRegistry().consoleUrls()); this.timeouts = Timeouts.of(controller.system()); } @@ -186,7 +185,7 @@ public class InternalStepRunner implements StepRunner { return deploy(() -> controller.applications().deploy(id.job(), setTheStage, logger::log, - account -> getCloudAccountWithOverrideForStaging(id, account)), + account -> getAndSetCloudAccountWithOverrideForStaging(id, account)), controller.jobController().run(id) .stepInfo(setTheStage ? deployInitialReal : deployReal).get() .startTime().get(), @@ -224,7 +223,7 @@ public class InternalStepRunner implements StepRunner { return account; } - private Optional<CloudAccount> getCloudAccountWithOverrideForStaging(RunId id, Optional<CloudAccount> account) { + private Optional<CloudAccount> getAndSetCloudAccountWithOverrideForStaging(RunId id, Optional<CloudAccount> account) { if (id.type().environment() == Environment.staging) { Instant doom = controller.clock().instant().plusSeconds(60); // Sleeping is bad, but we're already in a sleepy code path: deployment. while (true) { @@ -233,10 +232,6 @@ public class InternalStepRunner implements StepRunner { if (stored.isPresent()) return stored.filter(not(CloudAccount.empty::equals)); - // TODO jonmv: remove with next release - if (run.stepStatus(deployTester).get() != unfinished) - return account; // Use original value for runs which started prior to this code change, and resumed after. Extremely unlikely :> - long millisToDoom = Duration.between(controller.clock().instant(), doom).toMillis(); if (millisToDoom > 0) uncheckInterruptedAndRestoreFlag(() -> Thread.sleep(min(millisToDoom, 5000))); @@ -244,6 +239,7 @@ public class InternalStepRunner implements StepRunner { throw new CloudAccountNotSetException("Cloud account not yet set; must deploy tests first"); } } + account.ifPresent(cloudAccount -> controller.jobController().locked(id, run -> run.with(cloudAccount))); return account; } @@ -292,8 +288,8 @@ public class InternalStepRunner implements StepRunner { case LOAD_BALANCER_NOT_READY, PARENT_HOST_NOT_READY -> { logger.log(e.message()); // Consider splitting these messages in summary and details, on config server. Instant someTimeAfterStart = startTime.plusSeconds(200); - Instant inALittleWhile = controller.clock().instant().plusSeconds(60); - controller.jobController().locked(id, run -> run.sleepingUntil(someTimeAfterStart.isAfter(inALittleWhile) ? someTimeAfterStart : inALittleWhile)); + if (someTimeAfterStart.isAfter(controller.clock().instant())) + controller.jobController().locked(id, run -> run.sleepingUntil(someTimeAfterStart)); return result; } case NODE_ALLOCATION_FAILURE -> { @@ -860,7 +856,7 @@ public class InternalStepRunner implements StepRunner { private void updateConsoleNotification(Run run, boolean isRemoved) { NotificationSource source = NotificationSource.from(run.id()); - Consumer<String> updater = msg -> controller.notificationsDb().setNotification(source, Notification.Type.deployment, Notification.Level.error, msg); + Consumer<String> updater = msg -> controller.notificationsDb().setDeploymentNotification(run.id(), msg); switch (isRemoved ? success : run.status()) { case aborted, cancelled: return; // wait and see how the next run goes. case noTests: diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java index 0dc30f54d61..ae6bcdea00c 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java @@ -37,7 +37,6 @@ import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage; import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackageDiff; import com.yahoo.vespa.hosted.controller.application.pkg.TestPackage; import com.yahoo.vespa.hosted.controller.deployment.Run.Reason; -import com.yahoo.vespa.hosted.controller.notification.Notification; import com.yahoo.vespa.hosted.controller.notification.Notification.Type; import com.yahoo.vespa.hosted.controller.notification.NotificationSource; import com.yahoo.vespa.hosted.controller.persistence.BufferedLogStore; @@ -625,19 +624,15 @@ public class JobController { private void validateTests(TenantAndApplicationId id, Submission submission) { var testSummary = TestPackage.validateTests(submission.applicationPackage().deploymentSpec(), submission.testPackage()); if ( ! testSummary.problems().isEmpty()) - controller.notificationsDb().setNotification(NotificationSource.from(id), - Type.testPackage, - Notification.Level.warning, - testSummary.problems()); - + controller.notificationsDb().setTestPackageNotification(id, testSummary.problems()); } private void validateMajorVersion(TenantAndApplicationId id, Submission submission) { submission.applicationPackage().deploymentSpec().majorVersion().ifPresent(explicitMajor -> { if ( ! controller.readVersionStatus().isOnCurrentMajor(new Version(explicitMajor))) - controller.notificationsDb().setNotification(NotificationSource.from(id), Type.submission, Notification.Level.warning, - "Vespa " + explicitMajor + " will soon reach end of life, upgrade to Vespa " + (explicitMajor + 1) + " now: " + - "https://cloud.vespa.ai/en/vespa" + (explicitMajor + 1) + "-release-notes.html"); // ∠( á› ã€âˆ )＿ + controller.notificationsDb().setSubmissionNotification(id, + "Vespa " + explicitMajor + " will soon reach end of life, upgrade to [Vespa " + (explicitMajor + 1) + " now](" + + "https://cloud.vespa.ai/en/vespa" + (explicitMajor + 1) + "-release-notes.html)"); // ∠( á› ã€âˆ )＿ }); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Run.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Run.java index e92a70c3b4e..2b207e6662b 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Run.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Run.java @@ -276,7 +276,7 @@ public class Run { /** Whether this is a dry run deployment. */ public boolean isDryRun() { return dryRun; } - /** Cloud account override to use for this run, if set. This should only be used by staging tests. */ + /** Cloud account used for deployments in this run. This is set by the first deployment. */ public Optional<CloudAccount> cloudAccount() { return cloudAccount; } /** The specific reason for triggering this run, if any. This should be empty for jobs triggered bvy deployment orchestration. */ diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/BcpGroupUpdater.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/BcpGroupUpdater.java index 1327bfb09b2..92aaacaa1f0 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/BcpGroupUpdater.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/BcpGroupUpdater.java @@ -71,6 +71,24 @@ public class BcpGroupUpdater extends ControllerMaintainer { var patch = new ApplicationPatch(); addTrafficShare(deployment, bcpGroups, patch); addBcpGroupInfo(deployment.zone().region(), metrics.get(instance.id()), bcpGroups, patch); + + StringBuilder patchAsStringBuilder = new StringBuilder("Patch of instance ").append(instance.id().serializedForm()).append(": ") + .append("\n\tcurrentReadShare: ") + .append(patch.currentReadShare) + .append("\n\tmaxReadShare: ") + .append(patch.maxReadShare); + for (Map.Entry<String, ApplicationPatch.ClusterPatch> entry : patch.clusters.entrySet()) { + String key = entry.getKey(); + ApplicationPatch.ClusterPatch value = entry.getValue(); + patchAsStringBuilder.append("\n\tbcpGroupInfo for ").append(key).append(": ") + .append("\n\t\tcpuCostPerQuery: ") + .append(value.bcpGroupInfo.cpuCostPerQuery) + .append("\n\t\tqueryRate: ") + .append(value.bcpGroupInfo.queryRate) + .append("\n\t\tgrowthRateHeadroom: ") + .append(value.bcpGroupInfo.growthRateHeadroom); + } + log.log(Level.FINER, patchAsStringBuilder.toString()); nodeRepository.patchApplication(deployment.zone(), instance.id(), patch); } catch (Exception e) { @@ -84,7 +102,7 @@ public class BcpGroupUpdater extends ControllerMaintainer { double successFactorDeviation = asSuccessFactorDeviation(attempts, failures); if ( successFactorDeviation == -successFactorBaseline ) log.log(Level.WARNING, "Could not update traffic share on any applications", lastException); - else if ( successFactorDeviation < -0.1 ) + else if ( successFactorDeviation < 0 ) log.log(Level.FINE, "Could not update traffic share on all applications", lastException); return successFactorDeviation; } @@ -103,7 +121,9 @@ public class BcpGroupUpdater extends ControllerMaintainer { currentReadShare += groupQps == 0 ? 0 : fraction * deploymentQps / groupQps; maxReadShare += group.size() == 1 ? currentReadShare - : fraction * ( deploymentQps + group.maxQpsExcluding(deployment.zone().region()) / (group.size() - 1) ) / groupQps; + : groupQps != 0 + ? fraction * (deploymentQps + group.maxQpsExcluding(deployment.zone().region()) / (group.size() - 1)) / groupQps + : 0; } patch.currentReadShare = currentReadShare; patch.maxReadShare = maxReadShare; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/BillingReportMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/BillingReportMaintainer.java index 3aeba07630b..7868c3fe611 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/BillingReportMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/BillingReportMaintainer.java @@ -5,8 +5,11 @@ import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.TenantName; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.LockedTenant; +import com.yahoo.vespa.hosted.controller.api.integration.billing.BillStatus; import com.yahoo.vespa.hosted.controller.api.integration.billing.BillingController; +import com.yahoo.vespa.hosted.controller.api.integration.billing.BillingDatabaseClient; import com.yahoo.vespa.hosted.controller.api.integration.billing.BillingReporter; +import com.yahoo.vespa.hosted.controller.api.integration.billing.InvoiceUpdate; import com.yahoo.vespa.hosted.controller.api.integration.billing.Plan; import com.yahoo.vespa.hosted.controller.api.integration.billing.PlanRegistry; import com.yahoo.vespa.hosted.controller.tenant.CloudTenant; @@ -23,18 +26,25 @@ public class BillingReportMaintainer extends ControllerMaintainer { private final BillingReporter reporter; private final BillingController billing; + private final BillingDatabaseClient databaseClient; + private final PlanRegistry plans; public BillingReportMaintainer(Controller controller, Duration interval) { - super(controller, interval, null, Set.of(SystemName.PublicCd)); - this.reporter = controller.serviceRegistry().billingReporter(); - this.billing = controller.serviceRegistry().billingController(); - this.plans = controller.serviceRegistry().planRegistry(); + super(controller, interval, null, Set.of(SystemName.Public, SystemName.PublicCd)); + reporter = controller.serviceRegistry().billingReporter(); + billing = controller.serviceRegistry().billingController(); + databaseClient = controller.serviceRegistry().billingDatabase(); + plans = controller.serviceRegistry().planRegistry(); } @Override protected double maintain() { maintainTenants(); + + var updates = maintainInvoices(); + log.fine("Updated invoices: " + updates); + return 0.0; } @@ -53,6 +63,19 @@ public class BillingReportMaintainer extends ControllerMaintainer { }); } + InvoiceUpdate maintainInvoices() { + var billsNeedingMaintenance = databaseClient.readBills().stream() + .filter(bill -> bill.getExportedId().isPresent()) + .filter(exported -> exported.status() == BillStatus.OPEN) + .toList(); + + var updates = new InvoiceUpdate.Counter(); + for (var bill : billsNeedingMaintenance) { + updates.add(reporter.maintainInvoice(bill)); + } + return updates.finish(); + } + private Map<TenantName, CloudTenant> cloudTenants() { return controller().tenants().asList() .stream() @@ -74,4 +97,5 @@ public class BillingReportMaintainer extends ControllerMaintainer { .flatMap(p -> billing.tenantsWithPlan(tenants, p.id()).stream()) .toList(); } + } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CloudTrialExpirer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CloudTrialExpirer.java index 6c50afe7fb2..55428e80493 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CloudTrialExpirer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CloudTrialExpirer.java @@ -3,24 +3,40 @@ package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.TenantName; +import com.yahoo.vespa.flags.BooleanFlag; +import com.yahoo.vespa.flags.FetchVector; +import com.yahoo.vespa.flags.Flags; import com.yahoo.vespa.flags.ListFlag; import com.yahoo.vespa.flags.PermanentFlags; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.api.integration.billing.PlanId; +import com.yahoo.vespa.hosted.controller.notification.MailTemplating; +import com.yahoo.vespa.hosted.controller.notification.Notification; +import com.yahoo.vespa.hosted.controller.notification.NotificationSource; +import com.yahoo.vespa.hosted.controller.persistence.TrialNotifications; import com.yahoo.vespa.hosted.controller.tenant.LastLoginInfo; import com.yahoo.vespa.hosted.controller.tenant.Tenant; import java.time.Duration; +import java.time.Instant; +import java.util.ArrayList; +import java.util.HashMap; import java.util.List; import java.util.Optional; import java.util.function.Predicate; +import java.util.logging.Level; import java.util.logging.Logger; import java.util.stream.Collectors; +import static com.yahoo.vespa.hosted.controller.persistence.TrialNotifications.State.EXPIRED; +import static com.yahoo.vespa.hosted.controller.persistence.TrialNotifications.State.EXPIRES_IMMEDIATELY; +import static com.yahoo.vespa.hosted.controller.persistence.TrialNotifications.State.EXPIRES_SOON; +import static com.yahoo.vespa.hosted.controller.persistence.TrialNotifications.State.MID_CHECK_IN; +import static com.yahoo.vespa.hosted.controller.persistence.TrialNotifications.State.SIGNED_UP; +import static com.yahoo.vespa.hosted.controller.persistence.TrialNotifications.State.UNKNOWN; + /** * Expires unused tenants from Vespa Cloud. - * <p> - * TODO: Should support sending notifications some time before the various expiry events happen. * * @author ogronnesby */ @@ -30,17 +46,20 @@ public class CloudTrialExpirer extends ControllerMaintainer { private static final Duration nonePlanAfter = Duration.ofDays(14); private static final Duration tombstoneAfter = Duration.ofDays(91); private final ListFlag<String> extendedTrialTenants; + private final BooleanFlag cloudTrialNotificationEnabled; public CloudTrialExpirer(Controller controller, Duration interval) { super(controller, interval, null, SystemName.allOf(SystemName::isPublic)); this.extendedTrialTenants = PermanentFlags.EXTENDED_TRIAL_TENANTS.bindTo(controller().flagSource()); + this.cloudTrialNotificationEnabled = Flags.CLOUD_TRIAL_NOTIFICATIONS.bindTo(controller().flagSource()); } @Override protected double maintain() { var a = tombstoneNonePlanTenants(); var b = moveInactiveTenantsToNonePlan(); - return (a ? 0.0 : -0.5) + (b ? 0.0 : -0.5); + var c = notifyTenants(); + return (a ? 0.0 : -(1D/3)) + (b ? 0.0 : -(1D/3) + (c ? 0.0 : -(1D/3))); } private boolean moveInactiveTenantsToNonePlan() { @@ -76,6 +95,116 @@ public class CloudTrialExpirer extends ControllerMaintainer { return tombstoneTenants(idleOldPlanTenants); } + private boolean notifyTenants() { + try { + var currentStatus = controller().curator().readTrialNotifications() + .map(TrialNotifications::tenants).orElse(List.of()); + log.fine(() -> "Current: %s".formatted(currentStatus)); + var currentStatusByTenant = new HashMap<TenantName, TrialNotifications.Status>(); + currentStatus.forEach(status -> currentStatusByTenant.put(status.tenant(), status)); + var updatedStatus = new ArrayList<TrialNotifications.Status>(); + var now = controller().clock().instant(); + + for (var tenant : controller().tenants().asList()) { + + var status = currentStatusByTenant.get(tenant.name()); + var state = status == null ? UNKNOWN : status.state(); + var plan = controller().serviceRegistry().billingController().getPlan(tenant.name()).value(); + var ageInDays = Duration.between(tenant.createdAt(), now).toDays(); + + // TODO Replace stubs with proper email content stored in templates. + + var enabled = cloudTrialNotificationEnabled.with(FetchVector.Dimension.TENANT_ID, tenant.name().value()).value(); + if (!enabled) { + if (status != null) updatedStatus.add(status); + } else if (!List.of("none", "trial").contains(plan)) { + // Ignore tenants that are on a paid plan and skip from inclusion in updated data structure + } else if (status == null && "trial".equals(plan) && ageInDays <= 1) { + updatedStatus.add(updatedStatus(tenant, now, SIGNED_UP)); + notifySignup(tenant); + } else if ("none".equals(plan) && !List.of(EXPIRED).contains(state)) { + updatedStatus.add(updatedStatus(tenant, now, EXPIRED)); + notifyExpired(tenant); + } else if ("trial".equals(plan) && ageInDays >= 13 + && !List.of(EXPIRES_IMMEDIATELY, EXPIRED).contains(state)) { + updatedStatus.add(updatedStatus(tenant, now, EXPIRES_IMMEDIATELY)); + notifyExpiresImmediately(tenant); + } else if ("trial".equals(plan) && ageInDays >= 12 + && !List.of(EXPIRES_SOON, EXPIRES_IMMEDIATELY, EXPIRED).contains(state)) { + updatedStatus.add(updatedStatus(tenant, now, EXPIRES_SOON)); + notifyExpiresSoon(tenant); + } else if ("trial".equals(plan) && ageInDays >= 7 + && !List.of(MID_CHECK_IN, EXPIRES_SOON, EXPIRES_IMMEDIATELY, EXPIRED).contains(state)) { + updatedStatus.add(updatedStatus(tenant, now, MID_CHECK_IN)); + notifyMidCheckIn(tenant); + } else { + updatedStatus.add(status); + } + } + log.fine(() -> "Updated: %s".formatted(updatedStatus)); + controller().curator().writeTrialNotifications(new TrialNotifications(updatedStatus)); + return true; + } catch (Exception e) { + log.log(Level.WARNING, "Failed to process trial notifications", e); + return false; + } + } + + private void notifySignup(Tenant tenant) { + var consoleMsg = "Welcome to Vespa Cloud trial! [Manage plan](%s)".formatted(billingUrl(tenant)); + queueNotification(tenant, consoleMsg, "Welcome to Vespa Cloud", + "Welcome to Vespa Cloud! We hope you will enjoy your trial. " + + "Please reach out to us if you have any questions or feedback."); + } + + private void notifyMidCheckIn(Tenant tenant) { + var consoleMsg = "You're halfway through the **14 day** trial period. [Manage plan](%s)".formatted(billingUrl(tenant)); + queueNotification(tenant, consoleMsg, "How is your Vespa Cloud trial going?", + "How is your Vespa Cloud trial going? " + + "Please reach out to us if you have any questions or feedback."); + } + + private void notifyExpiresSoon(Tenant tenant) { + var consoleMsg = "Your Vespa Cloud trial expires in **2** days. [Manage plan](%s)".formatted(billingUrl(tenant)); + queueNotification(tenant, consoleMsg, "Your Vespa Cloud trial expires in 2 days", + "Your Vespa Cloud trial expires in 2 days. " + + "Please reach out to us if you have any questions or feedback."); + } + + private void notifyExpiresImmediately(Tenant tenant) { + var consoleMsg = "Your Vespa Cloud trial expires **tomorrow**. [Manage plan](%s)".formatted(billingUrl(tenant)); + queueNotification(tenant, consoleMsg, "Your Vespa Cloud trial expires tomorrow", + "Your Vespa Cloud trial expires tomorrow. " + + "Please reach out to us if you have any questions or feedback."); + } + + private void notifyExpired(Tenant tenant) { + var consoleMsg = "Your Vespa Cloud trial has expired. [Upgrade plan](%s)".formatted(billingUrl(tenant)); + queueNotification(tenant, consoleMsg, "Your Vespa Cloud trial has expired", + "Your Vespa Cloud trial has expired. " + + "Please reach out to us if you have any questions or feedback."); + } + + private void queueNotification(Tenant tenant, String consoleMsg, String emailSubject, String emailMsg) { + var mail = Optional.of(Notification.MailContent.fromTemplate(MailTemplating.Template.DEFAULT_MAIL_CONTENT) + .subject(emailSubject) + .with("mailMessageTemplate", "cloud-trial-notification") + .with("cloudTrialMessage", emailMsg) + .with("mailTitle", emailSubject) + .with("consoleLink", controller().serviceRegistry().consoleUrls().tenantOverview(tenant.name())) + .build()); + var source = NotificationSource.from(tenant.name()); + // Remove previous notification to ensure new notification is sent by email + controller().notificationsDb().removeNotification(source, Notification.Type.account); + controller().notificationsDb().setNotification( + source, Notification.Type.account, Notification.Level.info, consoleMsg, List.of(), mail); + } + + private String billingUrl(Tenant t) { return controller().serviceRegistry().consoleUrls().tenantBilling(t.name()); } + + private static TrialNotifications.Status updatedStatus(Tenant t, Instant i, TrialNotifications.State s) { + return new TrialNotifications.Status(t.name(), s, i); + } private boolean tenantIsCloudTenant(Tenant tenant) { return tenant.type() == Tenant.Type.cloud; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/FormattedNotification.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/FormattedNotification.java index 9402092f789..bed053d592f 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/FormattedNotification.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/FormattedNotification.java @@ -1,7 +1,6 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.notification; -import java.net.URI; import java.util.Objects; /** @@ -10,7 +9,7 @@ import java.util.Objects; * * @author enygaard */ -public record FormattedNotification(Notification notification, String prettyType, String messagePrefix, URI uri) { +public record FormattedNotification(Notification notification, String prettyType, String messagePrefix, String uri) { public FormattedNotification { Objects.requireNonNull(prettyType); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/MailTemplating.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/MailTemplating.java new file mode 100644 index 00000000000..1c05330702e --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/MailTemplating.java @@ -0,0 +1,101 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.notification; + +import com.yahoo.config.provision.TenantName; +import com.yahoo.vespa.hosted.controller.api.integration.ConsoleUrls; +import com.yahoo.vespa.hosted.controller.tenant.PendingMailVerification; +import com.yahoo.yolean.Exceptions; +import org.apache.velocity.VelocityContext; +import org.apache.velocity.app.Velocity; +import org.apache.velocity.app.VelocityEngine; +import org.apache.velocity.runtime.resource.loader.StringResourceLoader; +import org.apache.velocity.runtime.resource.util.StringResourceRepository; +import org.apache.velocity.tools.generic.EscapeTool; + +import java.io.StringWriter; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.Map; +import java.util.Optional; + +/** + * @author bjorncs + */ +public class MailTemplating { + + public enum Template { + MAIL("mail"), DEFAULT_MAIL_CONTENT("default-mail-content"), NOTIFICATION_MESSAGE("notification-message"), + CLOUD_TRIAL_NOTIFICATION("cloud-trial-notification"), MAIL_VERIFICATION("mail-verification"); + + public static Optional<Template> fromId(String id) { + return Arrays.stream(values()).filter(t -> t.id.equals(id)).findAny(); + } + + private final String id; + + Template(String id) { this.id = id; } + + public String getId() { return id; } + } + + private final VelocityEngine velocity; + private final EscapeTool escapeTool = new EscapeTool(); + private final ConsoleUrls consoleUrls; + + public MailTemplating(ConsoleUrls consoleUrls) { + this.velocity = createTemplateEngine(); + this.consoleUrls = consoleUrls; + } + + public String generateDefaultMailHtml(Template mailBodyTemplate, Map<String, Object> params, TenantName tenant) { + var ctx = createVelocityContext(); + ctx.put("accountNotificationLink", consoleUrls.tenantNotifications(tenant)); + ctx.put("privacyPolicyLink", "https://legal.yahoo.com/xw/en/yahoo/privacy/topic/b2bprivacypolicy/index.html"); + ctx.put("termsOfServiceLink", consoleUrls.termsOfService()); + ctx.put("supportLink", consoleUrls.support()); + ctx.put("mailBodyTemplate", mailBodyTemplate.getId()); + params.forEach(ctx::put); + return render(ctx, Template.MAIL); + } + + public String generateMailVerificationHtml(PendingMailVerification pmf) { + var ctx = createVelocityContext(); + ctx.put("verifyLink", consoleUrls.verifyEmail(pmf.getVerificationCode())); + ctx.put("email", pmf.getMailAddress()); + return render(ctx, Template.MAIL_VERIFICATION); + } + + public String escapeHtml(String s) { return escapeTool.html(s); } + + private VelocityContext createVelocityContext() { + var ctx = new VelocityContext(); + ctx.put("esc", escapeTool); + return ctx; + } + + private String render(VelocityContext ctx, Template template) { + var writer = new StringWriter(); + // Ignoring return value - implementation either returns 'true' or throws, never 'false' + velocity.mergeTemplate(template.getId(), StandardCharsets.UTF_8.name(), ctx, writer); + return writer.toString(); + } + + private static VelocityEngine createTemplateEngine() { + var v = new VelocityEngine(); + v.setProperty(Velocity.RESOURCE_LOADERS, "string"); + v.setProperty(Velocity.RESOURCE_LOADER + ".string.class", StringResourceLoader.class.getName()); + v.setProperty(Velocity.RESOURCE_LOADER + ".string.repository.static", "false"); + v.init(); + var repo = (StringResourceRepository) v.getApplicationAttribute(StringResourceLoader.REPOSITORY_NAME_DEFAULT); + Arrays.stream(Template.values()).forEach(t -> registerTemplate(repo, t.getId())); + return v; + } + + private static void registerTemplate(StringResourceRepository repo, String name) { + var templateStr = Exceptions.uncheck(() -> { + var in = MailTemplating.class.getResourceAsStream("/mail/%s.vm".formatted(name)); + return new String(in.readAllBytes()); + }); + repo.putStringResource(name, templateStr); + } +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/Notification.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/Notification.java index d22efdc5f6e..897e0be2d22 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/Notification.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/Notification.java @@ -2,8 +2,15 @@ package com.yahoo.vespa.hosted.controller.notification; import java.time.Instant; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Objects; +import java.util.Optional; +import java.util.SortedMap; +import java.util.TreeMap; /** * Represents an event that we want to notify the tenant about. The message(s) should be short @@ -13,15 +20,30 @@ import java.util.Objects; * * @author freva */ -public record Notification(Instant at, com.yahoo.vespa.hosted.controller.notification.Notification.Type type, com.yahoo.vespa.hosted.controller.notification.Notification.Level level, NotificationSource source, List<String> messages) { +public record Notification(Instant at, Notification.Type type, Notification.Level level, NotificationSource source, + String title, List<String> messages, Optional<MailContent> mailContent) { + + public Notification(Instant at, Type type, Level level, NotificationSource source, String title, List<String> messages) { + this(at, type, level, source, title, messages, Optional.empty()); + } public Notification(Instant at, Type type, Level level, NotificationSource source, List<String> messages) { - this.at = Objects.requireNonNull(at, "at cannot be null"); - this.type = Objects.requireNonNull(type, "type cannot be null"); - this.level = Objects.requireNonNull(level, "level cannot be null"); - this.source = Objects.requireNonNull(source, "source cannot be null"); - this.messages = List.copyOf(Objects.requireNonNull(messages, "messages cannot be null")); - if (messages.size() < 1) throw new IllegalArgumentException("messages cannot be empty"); + this(at, type, level, source, "", messages); + } + + public Notification { + Objects.requireNonNull(at, "at cannot be null"); + Objects.requireNonNull(type, "type cannot be null"); + Objects.requireNonNull(level, "level cannot be null"); + Objects.requireNonNull(source, "source cannot be null"); + Objects.requireNonNull(title, "title cannot be null"); + messages = List.copyOf(Objects.requireNonNull(messages, "messages cannot be null")); + + // Allowing empty title temporarily until all notifications have a title + // if (title.isBlank()) throw new IllegalArgumentException("title cannot be empty"); + if (messages.isEmpty() && title.isBlank()) throw new IllegalArgumentException("messages cannot be empty when title is empty"); + + Objects.requireNonNull(mailContent); } public enum Level { @@ -31,36 +53,81 @@ public record Notification(Instant at, com.yahoo.vespa.hosted.controller.notific public enum Type { - /** - * Related to contents of application package, e.g., usage of deprecated features/syntax - */ + /** Related to contents of application package, e.g., usage of deprecated features/syntax */ applicationPackage, - /** - * Related to contents of application package detectable by the controller on submission - */ + /** Related to contents of application package detectable by the controller on submission */ submission, - /** - * Related to contents of application test package, e.g., mismatch between deployment spec and provided tests - */ + /** Related to contents of application test package, e.g., mismatch between deployment spec and provided tests */ testPackage, - /** - * Related to deployment of application, e.g., system test failure, node allocation failure, internal errors, etc. - */ + /** Related to deployment of application, e.g., system test failure, node allocation failure, internal errors, etc. */ deployment, - /** - * Application cluster is (near) external feed blocked - */ + /** Application cluster is (near) external feed blocked */ feedBlock, - /** - * Application cluster is reindexing document(s) - */ - reindex + /** Application cluster is reindexing document(s) */ + reindex, + + /** Account, e.g. expiration of trial plan */ + account, + } + + public static class MailContent { + private final MailTemplating.Template template; + private final SortedMap<String, Object> values; + private final String subject; + + private MailContent(Builder b) { + template = Objects.requireNonNull(b.template); + values = new TreeMap<>(b.values); + subject = b.subject; + } + + public MailTemplating.Template template() { return template; } + public SortedMap<String, Object> values() { return Collections.unmodifiableSortedMap(values); } + public Optional<String> subject() { return Optional.ofNullable(subject); } + + public static Builder fromTemplate(MailTemplating.Template template) { return new Builder(template); } + + public static class Builder { + private final MailTemplating.Template template; + private final Map<String, Object> values = new HashMap<>(); + private String subject; + + private Builder(MailTemplating.Template template) { + this.template = template; + } + + public Builder with(String name, String value) { values.put(name, value); return this; } + public Builder with(String name, Collection<String> items) { values.put(name, List.copyOf(items)); return this; } + public Builder subject(String s) { this.subject = s; return this; } + public MailContent build() { return new MailContent(this); } + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + MailContent that = (MailContent) o; + return Objects.equals(template, that.template) && Objects.equals(values, that.values) && Objects.equals(subject, that.subject); + } + + @Override + public int hashCode() { + return Objects.hash(template, values, subject); + } + @Override + public String toString() { + return "MailContent{" + + "template='" + template + '\'' + + ", values=" + values + + ", subject='" + subject + '\'' + + '}'; + } } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/NotificationFormatter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/NotificationFormatter.java index de99d03cc82..e9b38f7a122 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/NotificationFormatter.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/NotificationFormatter.java @@ -2,17 +2,13 @@ package com.yahoo.vespa.hosted.controller.notification; import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.Environment; import com.yahoo.text.Text; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; -import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneRegistry; -import org.apache.http.client.utils.URIBuilder; +import com.yahoo.vespa.hosted.controller.api.integration.ConsoleUrls; -import java.net.URI; -import java.net.URISyntaxException; import java.util.Objects; import java.util.Optional; -import java.util.function.Function; + +import static com.yahoo.vespa.hosted.controller.notification.Notifier.notificationLink; /** * Created a NotificationContent for a given Notification. @@ -22,10 +18,10 @@ import java.util.function.Function; * @author enygaard */ public class NotificationFormatter { - private final ZoneRegistry zoneRegistry; + private final ConsoleUrls consoleUrls; - public NotificationFormatter(ZoneRegistry zoneRegistry) { - this.zoneRegistry = Objects.requireNonNull(zoneRegistry); + public NotificationFormatter(ConsoleUrls consoleUrls) { + this.consoleUrls = Objects.requireNonNull(consoleUrls); } public FormattedNotification format(Notification n) { @@ -35,20 +31,18 @@ public class NotificationFormatter { case testPackage -> testPackage(n); case reindex -> reindex(n); case feedBlock -> feedBlock(n); - default -> new FormattedNotification(n, n.type().name(), "", zoneRegistry.dashboardUrl(n.source().tenant())); + default -> new FormattedNotification(n, n.type().name(), "", consoleUrls.tenantOverview(n.source().tenant())); }; } private FormattedNotification applicationPackage(Notification n) { var source = n.source(); var application = requirePresent(source.application(), "application"); - var instance = requirePresent(source.instance(), "instance"); - var message = Text.format("Application package for %s.%s has %s", + var message = Text.format("Application package for %s%s has %s", application, - instance, + source.instance().map(instance -> "." + instance.value()).orElse(""), levelText(n.level(), n.messages().size())); - var uri = zoneRegistry.dashboardUrl(ApplicationId.from(source.tenant(), application, instance)); - return new FormattedNotification(n, "Application package", message, uri); + return new FormattedNotification(n, "Application package", message, notificationLink(consoleUrls, n.source())); } private FormattedNotification deployment(Notification n) { @@ -58,7 +52,7 @@ public class NotificationFormatter { requirePresent(source.application(), "application"), requirePresent(source.instance(), "instance"), levelText(n.level(), n.messages().size())); - return new FormattedNotification(n,"Deployment", message, jobLink(n.source())); + return new FormattedNotification(n,"Deployment", message, notificationLink(consoleUrls, n.source())); } private FormattedNotification testPackage(Notification n) { @@ -68,68 +62,23 @@ public class NotificationFormatter { n.messages().size() > 1 ? "are problems" : "is a problem", application, source.instance().map(i -> "."+i).orElse("")); - var uri = zoneRegistry.dashboardUrl(source.tenant(), application); - return new FormattedNotification(n, "Test package", message, uri); + return new FormattedNotification(n, "Test package", message, notificationLink(consoleUrls, n.source())); } private FormattedNotification reindex(Notification n) { var message = Text.format("%s is reindexing", clusterInfo(n.source())); - var source = n.source(); - var application = requirePresent(source.application(), "application"); - var instance = requirePresent(source.instance(), "instance"); - var clusterId = requirePresent(source.clusterId(), "clusterId"); - var zone = requirePresent(source.zoneId(), "zoneId"); - var instanceURI = zoneRegistry.dashboardUrl(ApplicationId.from(source.tenant(), application, instance)); - try { - var uri = new URIBuilder(instanceURI) - .setParameter( - String.format("%s.%s.%s", instance, zone.environment(), zone.region()), - String.format("clusters,%s=status", clusterId.value())) - .build(); - return new FormattedNotification(n, "Reindex", message, uri); - } catch (URISyntaxException e) { - throw new IllegalArgumentException(e); - } + var application = requirePresent(n.source().application(), "application"); + var instance = requirePresent(n.source().instance(), "instance"); + var clusterId = requirePresent(n.source().clusterId(), "clusterId"); + var zone = requirePresent(n.source().zoneId(), "zoneId"); + return new FormattedNotification(n, "Reindex", message, + consoleUrls.clusterReindexing(ApplicationId.from(n.source().tenant(), application, instance), zone, clusterId)); } private FormattedNotification feedBlock(Notification n) { - String type; - if (n.level() == Notification.Level.warning) { - type = "Nearly feed blocked"; - } else { - type = "Feed blocked"; - } + String type = n.level() == Notification.Level.warning ? "Nearly feed blocked" : "Feed blocked"; var message = Text.format("%s is %s", clusterInfo(n.source()), type.toLowerCase()); - var source = n.source(); - var application = requirePresent(source.application(), "application"); - var instance = requirePresent(source.instance(), "instance"); - var clusterId = requirePresent(source.clusterId(), "clusterId"); - var zone = requirePresent(source.zoneId(), "zoneId"); - var instanceURI = zoneRegistry.dashboardUrl(ApplicationId.from(source.tenant(), application, instance)); - try { - var uri = new URIBuilder(instanceURI) - .setParameter( - String.format("%s.%s.%s", instance, zone.environment(), zone.region()), - String.format("clusters,%s", clusterId.value())) - .build(); - return new FormattedNotification(n, type, message, uri); - } catch (URISyntaxException e) { - throw new IllegalArgumentException(e); - } - } - - private URI jobLink(NotificationSource source) { - var application = requirePresent(source.application(), "application"); - var instance = requirePresent(source.instance(), "instance"); - var jobType = requirePresent(source.jobType(), "jobType"); - var runNumber = source.runNumber().orElseThrow(() -> new MissingOptionalException("runNumber")); - var applicationId = ApplicationId.from(source.tenant(), application, instance); - Function<Environment, URI> link = (Environment env) -> zoneRegistry.dashboardUrl(new RunId(applicationId, jobType, runNumber)); - var environment = jobType.zone().environment(); - return switch (environment) { - case dev, perf -> link.apply(environment); - default -> link.apply(Environment.prod); - }; + return new FormattedNotification(n, type, message, notificationLink(consoleUrls, n.source())); } private String jobText(NotificationSource source) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/NotificationsDb.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/NotificationsDb.java index 287342f1290..e279e4feacd 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/NotificationsDb.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/NotificationsDb.java @@ -9,7 +9,11 @@ import com.yahoo.transaction.Mutex; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.api.application.v4.model.ClusterMetrics; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; +import com.yahoo.vespa.hosted.controller.api.integration.ConsoleUrls; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ApplicationReindexing; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; +import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; +import com.yahoo.vespa.hosted.controller.notification.Notification.MailContent; import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; import java.time.Clock; @@ -25,6 +29,7 @@ import java.util.stream.Stream; import static com.yahoo.vespa.hosted.controller.api.integration.configserver.ApplicationReindexing.Cluster; import static com.yahoo.vespa.hosted.controller.notification.Notification.Level; import static com.yahoo.vespa.hosted.controller.notification.Notification.Type; +import static com.yahoo.vespa.hosted.controller.notification.Notifier.notificationLink; /** * Adds, updates and removes tenant notifications in ZK @@ -38,15 +43,17 @@ public class NotificationsDb { private final Clock clock; private final CuratorDb curatorDb; private final Notifier notifier; + private final ConsoleUrls consoleUrls; public NotificationsDb(Controller controller) { - this(controller.clock(), controller.curator(), controller.notifier()); + this(controller.clock(), controller.curator(), controller.notifier(), controller.serviceRegistry().consoleUrls()); } - NotificationsDb(Clock clock, CuratorDb curatorDb, Notifier notifier) { + NotificationsDb(Clock clock, CuratorDb curatorDb, Notifier notifier, ConsoleUrls consoleUrls) { this.clock = clock; this.curatorDb = curatorDb; this.notifier = notifier; + this.consoleUrls = consoleUrls; } public List<TenantName> listTenantsWithNotifications() { @@ -59,22 +66,59 @@ public class NotificationsDb { .toList(); } - public void setNotification(NotificationSource source, Type type, Level level, String message) { - setNotification(source, type, level, List.of(message)); + public void setSubmissionNotification(TenantAndApplicationId tenantApp, String message) { + NotificationSource source = NotificationSource.from(tenantApp); + String title = "Application package for [%s](%s) has a warning".formatted( + tenantApp.application().value(), notificationLink(consoleUrls, source)); + setNotification(source, Type.submission, Level.warning, title, List.of(message), Optional.empty()); + } + + public void setApplicationPackageNotification(NotificationSource source, List<String> messages) { + String title = "Application package for [%s%s](%s) has %s".formatted( + source.application().get().value(), source.instance().map(i -> "." + i.value()).orElse(""), notificationLink(consoleUrls, source), + messages.size() == 1 ? "a warning" : "warnings"); + setNotification(source, Type.applicationPackage, Level.warning, title, messages, Optional.empty()); + } + + public void setTestPackageNotification(TenantAndApplicationId tenantApp, List<String> messages) { + NotificationSource source = NotificationSource.from(tenantApp); + String title = "There %s with tests for [%s](%s)".formatted( + messages.size() == 1 ? "is a problem" : "are problems", tenantApp.application().value(), + notificationLink(consoleUrls, source)); + setNotification(source, Type.testPackage, Level.warning, title, messages, Optional.empty()); + } + + public void setDeploymentNotification(RunId runId, String message) { + String description, linkText; + if (runId.type().isProduction()) { + description = runId.type().isTest() ? "Test job " : "Deployment job "; + linkText = "#" + runId.number() + " to " + runId.type().zone().region().value(); + } else if (runId.type().isTest()) { + description = ""; + linkText = (runId.type().isStagingTest() ? "Staging" : "System") + " test #" + runId.number(); + } else if (runId.type().isDeployment()) { + description = "Deployment job "; + linkText = "#" + runId.number() + " to " + runId.type().zone().value(); + } else throw new IllegalStateException("Unexpected job type " + runId.type()); + NotificationSource source = NotificationSource.from(runId); + String title = "%s[%s](%s) for application **%s.%s** has failed".formatted( + description, linkText, notificationLink(consoleUrls, source), runId.application().application().value(), runId.application().instance().value()); + setNotification(source, Type.deployment, Level.error, title, List.of(message), Optional.empty()); } /** * Add a notification with given source and type. If a notification with same source and type - * already exists, it'll be replaced by this one instead + * already exists, it'll be replaced by this one instead. */ - public void setNotification(NotificationSource source, Type type, Level level, List<String> messages) { + public void setNotification(NotificationSource source, Type type, Level level, String title, List<String> messages, + Optional<MailContent> mailContent) { Optional<Notification> changed = Optional.empty(); try (Mutex lock = curatorDb.lockNotifications(source.tenant())) { var existingNotifications = curatorDb.readNotifications(source.tenant()); List<Notification> notifications = existingNotifications.stream() .filter(notification -> !source.equals(notification.source()) || type != notification.type()) .collect(Collectors.toCollection(ArrayList::new)); - var notification = new Notification(clock.instant(), type, level, source, messages); + var notification = new Notification(clock.instant(), type, level, source, title, messages, mailContent); if (!notificationExists(notification, existingNotifications, false)) { changed = Optional.of(notification); } @@ -128,14 +172,9 @@ public class NotificationsDb { Instant now = clock.instant(); List<Notification> changed = List.of(); List<Notification> newNotifications = Stream.concat( - clusterMetrics.stream().map(metric -> { - NotificationSource source = NotificationSource.from(deploymentId, ClusterSpec.Id.from(metric.getClusterId())); - return createFeedBlockNotification(source, now, metric); - }), - applicationReindexing.clusters().entrySet().stream().map(entry -> { - NotificationSource source = NotificationSource.from(deploymentId, ClusterSpec.Id.from(entry.getKey())); - return createReindexNotification(source, now, entry.getValue()); - })) + clusterMetrics.stream().map(metric -> createFeedBlockNotification(consoleUrls, deploymentId, metric.getClusterId(), now, metric)), + applicationReindexing.clusters().entrySet().stream().map(entry -> + createReindexNotification(consoleUrls, deploymentId, entry.getKey(), now, entry.getValue()))) .flatMap(Optional::stream) .toList(); @@ -169,25 +208,34 @@ public class NotificationsDb { return exists; } - private static Optional<Notification> createFeedBlockNotification(NotificationSource source, Instant at, ClusterMetrics metric) { + private static Optional<Notification> createFeedBlockNotification(ConsoleUrls consoleUrls, DeploymentId deployment, String clusterId, Instant at, ClusterMetrics metric) { Optional<Pair<Level, String>> memoryStatus = resourceUtilToFeedBlockStatus("memory", metric.memoryUtil(), metric.memoryFeedBlockLimit()); Optional<Pair<Level, String>> diskStatus = resourceUtilToFeedBlockStatus("disk", metric.diskUtil(), metric.diskFeedBlockLimit()); if (memoryStatus.isEmpty() && diskStatus.isEmpty()) return Optional.empty(); + NotificationSource source = NotificationSource.from(deployment, ClusterSpec.Id.from(clusterId)); // Find the max among levels Level level = Stream.of(memoryStatus, diskStatus) .flatMap(status -> status.stream().map(Pair::getFirst)) .max(Comparator.comparing(Enum::ordinal)).get(); + String title = "Cluster [%s](%s) in **%s** for **%s.%s** is %sfeed blocked".formatted( + clusterId, notificationLink(consoleUrls, source), deployment.zoneId().value(), deployment.applicationId().application().value(), + deployment.applicationId().instance().value(), level == Level.warning ? "nearly " : ""); List<String> messages = Stream.concat(memoryStatus.stream(), diskStatus.stream()) .filter(status -> status.getFirst() == level) // Do not mix message from different levels .map(Pair::getSecond) .toList(); - return Optional.of(new Notification(at, Type.feedBlock, level, source, messages)); + + return Optional.of(new Notification(at, Type.feedBlock, level, source, title, messages)); } - private static Optional<Notification> createReindexNotification(NotificationSource source, Instant at, Cluster cluster) { + private static Optional<Notification> createReindexNotification(ConsoleUrls consoleUrls, DeploymentId deployment, String clusterId, Instant at, Cluster cluster) { + NotificationSource source = NotificationSource.from(deployment, ClusterSpec.Id.from(clusterId)); + String title = "Cluster [%s](%s) in **%s** for **%s.%s** is [reindexing](https://docs.vespa.ai/en/operations/reindexing.html)".formatted( + clusterId, consoleUrls.clusterReindexing(deployment.applicationId(), deployment.zoneId(), source.clusterId().get()), + deployment.zoneId().value(), deployment.applicationId().application().value(), deployment.applicationId().instance().value()); List<String> messages = cluster.ready().entrySet().stream() .filter(entry -> entry.getValue().progress().isPresent()) .map(entry -> Text.format("document type '%s'%s (%.1f%% done)", @@ -195,7 +243,7 @@ public class NotificationsDb { .sorted() .toList(); if (messages.isEmpty()) return Optional.empty(); - return Optional.of(new Notification(at, Type.reindex, Level.info, source, messages)); + return Optional.of(new Notification(at, Type.reindex, Level.info, source, title, messages)); } /** diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/Notifier.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/Notifier.java index afb260bf765..f27e69c4636 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/Notifier.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/Notifier.java @@ -2,22 +2,22 @@ package com.yahoo.vespa.hosted.controller.notification; import com.google.common.annotations.VisibleForTesting; +import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Environment; -import com.yahoo.config.provision.TenantName; -import com.yahoo.restapi.UriBuilder; +import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.text.Text; import com.yahoo.vespa.flags.FetchVector; import com.yahoo.vespa.flags.FlagSource; import com.yahoo.vespa.flags.PermanentFlags; +import com.yahoo.vespa.hosted.controller.api.integration.ConsoleUrls; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; import com.yahoo.vespa.hosted.controller.api.integration.organization.Mail; import com.yahoo.vespa.hosted.controller.api.integration.organization.Mailer; import com.yahoo.vespa.hosted.controller.api.integration.organization.MailerException; -import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneRegistry; import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; import com.yahoo.vespa.hosted.controller.tenant.CloudTenant; import com.yahoo.vespa.hosted.controller.tenant.TenantContacts; -import java.net.URI; import java.util.Collection; import java.util.List; import java.util.Objects; @@ -26,8 +26,6 @@ import java.util.logging.Logger; import java.util.regex.Pattern; import java.util.stream.Collectors; -import static com.yahoo.yolean.Exceptions.uncheck; - /** * Notifier is responsible for dispatching user notifications to their chosen Contact points. * @@ -37,20 +35,22 @@ public class Notifier { private final CuratorDb curatorDb; private final Mailer mailer; private final FlagSource flagSource; + private final ConsoleUrls consoleUrls; private final NotificationFormatter formatter; - private final URI dashboardUri; + private final MailTemplating mailTemplating; private static final Logger log = Logger.getLogger(Notifier.class.getName()); // Minimal url pattern matcher to detect hardcoded URLs in Notification messages private static final Pattern urlPattern = Pattern.compile("https://[\\w\\d./]+"); - public Notifier(CuratorDb curatorDb, ZoneRegistry zoneRegistry, Mailer mailer, FlagSource flagSource) { + public Notifier(CuratorDb curatorDb, ConsoleUrls consoleUrls, Mailer mailer, FlagSource flagSource) { this.curatorDb = Objects.requireNonNull(curatorDb); this.mailer = Objects.requireNonNull(mailer); this.flagSource = Objects.requireNonNull(flagSource); - this.formatter = new NotificationFormatter(zoneRegistry); - this.dashboardUri = zoneRegistry.dashboardUrl(); + this.consoleUrls = Objects.requireNonNull(consoleUrls); + this.formatter = new NotificationFormatter(consoleUrls); + this.mailTemplating = new MailTemplating(consoleUrls); } public void dispatch(List<Notification> notifications, NotificationSource source) { @@ -101,10 +101,13 @@ public class Notifier { log.fine(() -> "Sending notification " + notification + " to " + contacts.stream().map(c -> c.email().getEmailAddress()).toList()); var content = formatter.format(notification); - mailer.send(mailOf(content, contacts.stream() - .filter(c -> c.email().isVerified()) - .map(c -> c.email().getEmailAddress()) - .toList())); + var verifiedContacts = contacts.stream() + .filter(c -> c.email().isVerified()).map(c -> c.email().getEmailAddress()).toList(); + if (verifiedContacts.isEmpty()) { + log.fine(() -> "None of the %d contact(s) are verified - skipping delivery of %s".formatted(contacts.size(), notification)); + return; + } + mailer.send(mailOf(content, verifiedContacts)); } catch (MailerException e) { log.log(Level.SEVERE, "Failed sending email", e); } catch (MissingOptionalException e) { @@ -114,23 +117,30 @@ public class Notifier { public Mail mailOf(FormattedNotification content, Collection<String> recipients) { var notification = content.notification(); - var subject = Text.format("[%s] %s Vespa Notification for %s", notification.level().toString().toUpperCase(), content.prettyType(), applicationIdSource(notification.source())); - var template = uncheck(() -> Notifier.class.getResourceAsStream("/mail/mail-notification.tmpl").readAllBytes()); - var html = new String(template) - .replace("[[NOTIFICATION_HEADER]]", content.messagePrefix()) - .replace("[[NOTIFICATION_ITEMS]]", notification.messages().stream() - .map(Notifier::linkify) - .map(Notifier::capitalise) - .map(m -> "<p>" + m + "</p>") - .collect(Collectors.joining())) - .replace("[[LINK_TO_NOTIFICATION]]", notificationLink(notification.source())) - .replace("[[LINK_TO_ACCOUNT_NOTIFICATIONS]]", accountNotificationsUri(content.notification().source().tenant())) - .replace("[[LINK_TO_PRIVACY_POLICY]]", "https://legal.yahoo.com/xw/en/yahoo/privacy/topic/b2bprivacypolicy/index.html") - .replace("[[LINK_TO_TERMS_OF_SERVICE]]", consoleUri("terms-of-service-trial.html")) - .replace("[[LINK_TO_SUPPORT]]", consoleUri("support")); + var subject = content.notification().mailContent().flatMap(Notification.MailContent::subject) + .orElseGet(() -> Text.format( + "[%s] %s Vespa Notification for %s", notification.level().toString().toUpperCase(), + content.prettyType(), applicationIdSource(notification.source()))); + var html = generateHtml(content); return new Mail(recipients, subject, "", html); } + private String generateHtml(FormattedNotification content) { + var mailContent = content.notification().mailContent().orElseGet(() -> generateContentFromMessages(content)); + return mailTemplating.generateDefaultMailHtml(mailContent.template(), mailContent.values(), content.notification().source().tenant()); + } + + private Notification.MailContent generateContentFromMessages(FormattedNotification f) { + var items = f.notification().messages().stream().map(m -> capitalise(linkify(mailTemplating.escapeHtml(m)))).toList(); + return Notification.MailContent.fromTemplate(MailTemplating.Template.DEFAULT_MAIL_CONTENT) + .with("mailMessageTemplate", "notification-message") + .with("mailTitle", "Vespa Cloud Notifications") + .with("notificationHeader", f.messagePrefix()) + .with("notificationItems", items) + .with("consoleLink", notificationLink(consoleUrls, f.notification().source())) + .build(); + } + @VisibleForTesting static String linkify(String text) { return urlPattern.matcher(text).replaceAll((res) -> String.format("<a href=\"%s\">%s</a>", res.group(), res.group())); @@ -144,36 +154,16 @@ public class Notifier { return sb.toString(); } - private String accountNotificationsUri(TenantName tenant) { - return new UriBuilder(dashboardUri) - .append("tenant/") - .append(tenant.value()) - .append("account/notifications") - .toString(); - } + static String notificationLink(ConsoleUrls consoleUrls, NotificationSource source) { + if (source.application().isEmpty()) return consoleUrls.tenantOverview(source.tenant()); + if (source.instance().isEmpty()) return consoleUrls.prodApplicationOverview(source.tenant(), source.application().get()); - private String consoleUri(String path) { - return new UriBuilder(dashboardUri).append(path).toString(); - } - - private String notificationLink(NotificationSource source) { - var uri = new UriBuilder(dashboardUri); - uri = uri.append("tenant").append(source.tenant().value()); - if (source.application().isPresent()) - uri = uri.append("application").append(source.application().get().value()); - if (source.isProduction()) { - uri = uri.append("prod/instance"); - if (source.jobType().isPresent()) { - uri = uri.append(source.instance().get().value()); - } - } - else { - uri = uri.append("dev/instance/").append(source.instance().get().value()); - } - if (source.jobType().isPresent()) { - uri = uri.append("job").append(source.jobType().get().jobName()).append("run").append(String.valueOf(source.runNumber().getAsLong())); - } - return uri.toString(); + ApplicationId application = ApplicationId.from(source.tenant(), source.application().get(), source.instance().get()); + if (source.jobType().isPresent()) + return consoleUrls.deploymentRun(new RunId(application, source.jobType().get(), source.runNumber().getAsLong())); + if (source.clusterId().isPresent()) + return consoleUrls.clusterOverview(application, source.zoneId().get(), source.clusterId().get()); + return consoleUrls.instanceOverview(application, source.zoneId().map(ZoneId::environment).orElse(Environment.prod)); } private static String capitalise(String m) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java index a2a4cf809b1..cef62438a53 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java @@ -111,6 +111,7 @@ public class CuratorDb { private static final Path mailVerificationRoot = root.append("mailVerification"); private static final Path dataPlaneTokenRoot = root.append("dataplaneTokens"); private static final Path certificatePoolRoot = root.append("certificatePool"); + private static final Path trialNotificationsRoot = root.append("trialNotifications"); private final NodeVersionSerializer nodeVersionSerializer = new NodeVersionSerializer(); private final VersionStatusSerializer versionStatusSerializer = new VersionStatusSerializer(nodeVersionSerializer); @@ -816,6 +817,16 @@ public class CuratorDb { return curator.getChildren(certificatePoolRoot).stream().flatMap(id -> readUnassignedCertificate(id).stream()).toList(); } + // -------------- Cloud trial notification -------------------------------- + + public void writeTrialNotifications(TrialNotifications tn) { + curator.set(trialNotificationsRoot, asJson(tn.toSlime())); + } + + public Optional<TrialNotifications> readTrialNotifications() { + return readSlime(trialNotificationsRoot).map(TrialNotifications::fromSlime); + } + // -------------- Paths --------------------------------------------------- private static Path upgradesPerMinutePath() { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/NotificationsSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/NotificationsSerializer.java index fa688436256..d5be4d22dc2 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/NotificationsSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/NotificationsSerializer.java @@ -8,13 +8,16 @@ import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.slime.Cursor; import com.yahoo.slime.Inspector; +import com.yahoo.slime.ObjectTraverser; import com.yahoo.slime.Slime; import com.yahoo.slime.SlimeUtils; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; +import com.yahoo.vespa.hosted.controller.notification.MailTemplating; import com.yahoo.vespa.hosted.controller.notification.Notification; import com.yahoo.vespa.hosted.controller.notification.NotificationSource; import java.util.List; +import java.util.Optional; /** * (de)serializes notifications for a tenant @@ -34,6 +37,7 @@ public class NotificationsSerializer { private static final String atFieldName = "at"; private static final String typeField = "type"; private static final String levelField = "level"; + private static final String titleField = "title"; private static final String messagesField = "messages"; private static final String applicationField = "application"; private static final String instanceField = "instance"; @@ -51,6 +55,7 @@ public class NotificationsSerializer { notificationObject.setLong(atFieldName, notification.at().toEpochMilli()); notificationObject.setString(typeField, asString(notification.type())); notificationObject.setString(levelField, asString(notification.level())); + notificationObject.setString(titleField, notification.title()); Cursor messagesArray = notificationObject.setArray(messagesField); notification.messages().forEach(messagesArray::addString); @@ -60,6 +65,22 @@ public class NotificationsSerializer { notification.source().clusterId().ifPresent(clusterId -> notificationObject.setString(clusterIdField, clusterId.value())); notification.source().jobType().ifPresent(jobType -> notificationObject.setString(jobTypeField, jobType.serialized())); notification.source().runNumber().ifPresent(runNumber -> notificationObject.setLong(runNumberField, runNumber)); + + notification.mailContent().ifPresent(mc -> { + notificationObject.setString("mail-template", mc.template().getId()); + mc.subject().ifPresent(s -> notificationObject.setString("mail-subject", s)); + var mailParamsCursor = notificationObject.setObject("mail-params"); + mc.values().forEach((key, value) -> { + if (value instanceof String str) { + mailParamsCursor.setString(key, str); + } else if (value instanceof List<?> l) { + var array = mailParamsCursor.setArray(key); + l.forEach(elem -> array.addString((String) elem)); + } else { + throw new ClassCastException("Unsupported param type: " + value.getClass()); + } + }); + }); } return slime; @@ -92,7 +113,24 @@ public class NotificationsSerializer { SlimeUtils.optionalString(inspector.field(clusterIdField)).map(ClusterSpec.Id::from), SlimeUtils.optionalString(inspector.field(jobTypeField)).map(jobName -> JobType.ofSerialized(jobName)), SlimeUtils.optionalLong(inspector.field(runNumberField))), - SlimeUtils.entriesStream(inspector.field(messagesField)).map(Inspector::asString).toList()); + SlimeUtils.optionalString(inspector.field(titleField)).orElse(""), + SlimeUtils.entriesStream(inspector.field(messagesField)).map(Inspector::asString).toList(), + mailContentFrom(inspector)); + } + + private Optional<Notification.MailContent> mailContentFrom(final Inspector inspector) { + return SlimeUtils.optionalString(inspector.field("mail-template")).map(template -> { + var builder = Notification.MailContent.fromTemplate(MailTemplating.Template.fromId(template).orElseThrow()); + SlimeUtils.optionalString(inspector.field("mail-subject")).ifPresent(builder::subject); + inspector.field("mail-params").traverse((ObjectTraverser) (name, insp) -> { + switch (insp.type()) { + case STRING -> builder.with(name, insp.asString()); + case ARRAY -> builder.with(name, SlimeUtils.entriesStream(insp).map(Inspector::asString).toList()); + default -> throw new IllegalArgumentException("Unsupported param type: " + insp.type()); + } + }); + return builder.build(); + }); } private static String asString(Notification.Type type) { @@ -103,6 +141,7 @@ public class NotificationsSerializer { case deployment -> "deployment"; case feedBlock -> "feedBlock"; case reindex -> "reindex"; + case account -> "account"; }; } @@ -114,6 +153,7 @@ public class NotificationsSerializer { case "deployment" -> Notification.Type.deployment; case "feedBlock" -> Notification.Type.feedBlock; case "reindex" -> Notification.Type.reindex; + case "account" -> Notification.Type.account; default -> throw new IllegalArgumentException("Unknown serialized notification type value '" + field.asString() + "'"); }; } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializer.java index 381a5eaaa26..85b7acd603a 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializer.java @@ -28,6 +28,8 @@ import com.yahoo.vespa.hosted.controller.tenant.CloudTenant; import com.yahoo.vespa.hosted.controller.tenant.DeletedTenant; import com.yahoo.vespa.hosted.controller.tenant.Email; import com.yahoo.vespa.hosted.controller.tenant.LastLoginInfo; +import com.yahoo.vespa.hosted.controller.tenant.PurchaseOrder; +import com.yahoo.vespa.hosted.controller.tenant.TaxId; import com.yahoo.vespa.hosted.controller.tenant.Tenant; import com.yahoo.vespa.hosted.controller.tenant.TenantAddress; import com.yahoo.vespa.hosted.controller.tenant.TenantBilling; @@ -93,6 +95,11 @@ public class TenantSerializer { private static final String cloudAccountsField = "cloudAccounts"; private static final String accountField = "account"; private static final String templateVersionField = "templateVersion"; + private static final String taxIdField = "taxId"; + private static final String taxIdTypeField = "type"; + private static final String taxIdCodeField = "code"; + private static final String purchaseOrderField = "purchaseOrder"; + private static final String invoiceEmailField = "invoiceEmail"; private static final String awsIdField = "awsId"; private static final String roleField = "role"; @@ -282,12 +289,29 @@ public class TenantSerializer { } private TenantBilling tenantInfoBillingContactFromSlime(Inspector billingObject) { + var taxIdInspector = billingObject.field(taxIdField); + var taxId = switch (taxIdInspector.type()) { + case STRING -> TaxId.legacy(taxIdInspector.asString()); + case OBJECT -> { + var taxIdType = taxIdInspector.field(taxIdTypeField).asString(); + var taxIdCode = taxIdInspector.field(taxIdCodeField).asString(); + yield new TaxId(new TaxId.Type(taxIdType), new TaxId.Code(taxIdCode)); + } + case NIX -> TaxId.empty(); + default -> throw new IllegalStateException(taxIdInspector.type().name()); + }; + var purchaseOrder = new PurchaseOrder(billingObject.field(purchaseOrderField).asString()); + var invoiceEmail = new Email(billingObject.field(invoiceEmailField).asString(), false); + return TenantBilling.empty() .withContact(TenantContact.from( billingObject.field("name").asString(), - new Email(billingObject.field("email").asString(), true), + new Email(billingObject.field("email").asString(), billingObject.field("emailVerified").asBool()), billingObject.field("phone").asString())) - .withAddress(tenantInfoAddressFromSlime(billingObject.field("address"))); + .withAddress(tenantInfoAddressFromSlime(billingObject.field("address"))) + .withTaxId(taxId) + .withPurchaseOrder(purchaseOrder) + .withInvoiceEmail(invoiceEmail); } private List<TenantSecretStore> secretStoresFromSlime(Inspector secretStoresObject) { @@ -344,11 +368,17 @@ public class TenantSerializer { private void toSlime(TenantBilling billingContact, Cursor parentCursor) { if (billingContact.isEmpty()) return; - Cursor addressCursor = parentCursor.setObject("billingContact"); - addressCursor.setString("name", billingContact.contact().name()); - addressCursor.setString("email", billingContact.contact().email().getEmailAddress()); - addressCursor.setString("phone", billingContact.contact().phone()); - toSlime(billingContact.address(), addressCursor); + Cursor billingCursor = parentCursor.setObject("billingContact"); + billingCursor.setString("name", billingContact.contact().name()); + billingCursor.setString("email", billingContact.contact().email().getEmailAddress()); + billingCursor.setBool("emailVerified", billingContact.contact().email().isVerified()); + billingCursor.setString("phone", billingContact.contact().phone()); + var taxIdCursor = billingCursor.setObject(taxIdField); + taxIdCursor.setString(taxIdTypeField, billingContact.getTaxId().type().value()); + taxIdCursor.setString(taxIdCodeField, billingContact.getTaxId().code().value()); + billingCursor.setString(purchaseOrderField, billingContact.getPurchaseOrder().value()); + billingCursor.setString(invoiceEmailField, billingContact.getInvoiceEmail().getEmailAddress()); + toSlime(billingContact.address(), billingCursor); } private void toSlime(List<TenantSecretStore> tenantSecretStores, Cursor parentCursor) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/TrialNotifications.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/TrialNotifications.java new file mode 100644 index 00000000000..a205e6c4173 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/TrialNotifications.java @@ -0,0 +1,57 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package com.yahoo.vespa.hosted.controller.persistence; + +import com.yahoo.config.provision.TenantName; +import com.yahoo.slime.Slime; +import com.yahoo.slime.SlimeUtils; + +import java.time.Instant; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.logging.Logger; + +/** + * @author bjorncs + */ +public record TrialNotifications(List<Status> tenants) { + private static final Logger log = Logger.getLogger(TrialNotifications.class.getName()); + + public TrialNotifications { tenants = List.copyOf(tenants); } + + public record Status(TenantName tenant, State state, Instant lastUpdate) {} + public enum State { SIGNED_UP, MID_CHECK_IN, EXPIRES_SOON, EXPIRES_IMMEDIATELY, EXPIRED, UNKNOWN } + + public Slime toSlime() { + var slime = new Slime(); + var rootCursor = slime.setObject(); + var tenantsCursor = rootCursor.setArray("tenants"); + for (Status t : tenants) { + var tenantCursor = tenantsCursor.addObject(); + tenantCursor.setString("tenant", t.tenant().value()); + tenantCursor.setString("state", t.state().name()); + tenantCursor.setString("lastUpdate", t.lastUpdate().toString()); + } + log.fine(() -> "Generated json '%s' from '%s'".formatted(SlimeUtils.toJson(slime), this)); + return slime; + } + + public static TrialNotifications fromSlime(Slime slime) { + var rootCursor = slime.get(); + var tenantsCursor = rootCursor.field("tenants"); + var tenants = new ArrayList<Status>(); + for (int i = 0; i < tenantsCursor.entries(); i++) { + var tenantCursor = tenantsCursor.entry(i); + var name = TenantName.from(tenantCursor.field("tenant").asString()); + var stateStr = tenantCursor.field("state").asString(); + var state = Arrays.stream(State.values()) + .filter(s -> s.name().equals(stateStr)).findFirst().orElse(State.UNKNOWN); + var lastUpdate = Instant.parse(tenantCursor.field("lastUpdate").asString()); + tenants.add(new Status(name, state, lastUpdate)); + } + var tn = new TrialNotifications(tenants); + log.fine(() -> "Parsed '%s' from '%s'".formatted(tn, SlimeUtils.toJson(slime))); + return tn; + } +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java index d274d59c417..078c27f5d00 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java @@ -127,6 +127,8 @@ import com.yahoo.vespa.hosted.controller.tenant.DeletedTenant; import com.yahoo.vespa.hosted.controller.tenant.Email; import com.yahoo.vespa.hosted.controller.tenant.LastLoginInfo; import com.yahoo.vespa.hosted.controller.tenant.PendingMailVerification; +import com.yahoo.vespa.hosted.controller.tenant.PurchaseOrder; +import com.yahoo.vespa.hosted.controller.tenant.TaxId; import com.yahoo.vespa.hosted.controller.tenant.Tenant; import com.yahoo.vespa.hosted.controller.tenant.TenantAddress; import com.yahoo.vespa.hosted.controller.tenant.TenantBilling; @@ -692,7 +694,13 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { var contact = root.setObject("contact"); contact.setString("name", billingContact.contact().name()); contact.setString("email", billingContact.contact().email().getEmailAddress()); + contact.setBool("emailVerified", billingContact.contact().email().isVerified()); contact.setString("phone", billingContact.contact().phone()); + var taxIdCursor = root.setObject("taxId"); + taxIdCursor.setString("type", billingContact.getTaxId().type().value()); + taxIdCursor.setString("code", billingContact.getTaxId().code().value()); + root.setString("purchaseOrder", billingContact.getPurchaseOrder().value()); + root.setString("invoiceEmail", billingContact.getInvoiceEmail().getEmailAddress()); toSlime(billingContact.address(), root); // will create "address" on the parent } @@ -702,15 +710,26 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { private SlimeJsonResponse putTenantInfoBilling(CloudTenant cloudTenant, Inspector inspector) { var info = cloudTenant.info(); - var contact = info.billingContact().contact(); - var address = info.billingContact().address(); + var billing = info.billingContact(); + var contact = billing.contact(); + var address = billing.address(); - var mergedContact = updateTenantInfoContact(inspector.field("contact"), cloudTenant.name(), contact, false); - var mergedAddress = updateTenantInfoAddress(inspector.field("address"), info.billingContact().address()); + var mergedContact = updateBillingContact(inspector.field("contact"), cloudTenant.name(), contact); + var mergedAddress = updateTenantInfoAddress(inspector.field("address"), billing.address()); + var mergedTaxId = updateAndValidateTaxId(inspector.field("taxId"), billing.getTaxId()); + var mergedPurchaseOrder = optional("purchaseOrder", inspector).map(PurchaseOrder::new).orElse(billing.getPurchaseOrder()); + var mergedInvoiceEmail = optional("invoiceEmail", inspector).map(mail -> new Email(mail, false)).orElse(billing.getInvoiceEmail()); + + if (!inspector.field("taxId").valid() && inspector.field("address").valid()) { + throw new IllegalArgumentException("Tax ID information is mandatory for setting up billing"); + } var mergedBilling = info.billingContact() .withContact(mergedContact) - .withAddress(mergedAddress); + .withAddress(mergedAddress) + .withTaxId(mergedTaxId) + .withPurchaseOrder(mergedPurchaseOrder) + .withInvoiceEmail(mergedInvoiceEmail); var mergedInfo = info.withBilling(mergedBilling); @@ -763,6 +782,11 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { throw new IllegalArgumentException("'website' needs to be a valid address"); } } + if (! mergedInfo.billingContact().getInvoiceEmail().isBlank()) { + // TODO: Validate invoice email is set if collection method is INVOICE + if (! mergedInfo.billingContact().getInvoiceEmail().getEmailAddress().contains("@")) + throw new IllegalArgumentException("'Invoice email' needs to be an email address"); + } } private void toSlime(TenantAddress address, Cursor parentCursor) { @@ -779,11 +803,17 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { private void toSlime(TenantBilling billingContact, Cursor parentCursor) { if (billingContact.isEmpty()) return; - Cursor addressCursor = parentCursor.setObject("billingContact"); - addressCursor.setString("name", billingContact.contact().name()); - addressCursor.setString("email", billingContact.contact().email().getEmailAddress()); - addressCursor.setString("phone", billingContact.contact().phone()); - toSlime(billingContact.address(), addressCursor); + Cursor billingCursor = parentCursor.setObject("billingContact"); + billingCursor.setString("name", billingContact.contact().name()); + billingCursor.setString("email", billingContact.contact().email().getEmailAddress()); + billingCursor.setBool("emailVerified", billingContact.contact().email().isVerified()); + billingCursor.setString("phone", billingContact.contact().phone()); + var taxIdCursor = billingCursor.setObject("taxId"); + taxIdCursor.setString("type", billingContact.getTaxId().type().value()); + taxIdCursor.setString("code", billingContact.getTaxId().code().value()); + billingCursor.setString("purchaseOrder", billingContact.getPurchaseOrder().value()); + billingCursor.setString("invoiceEmail", billingContact.getInvoiceEmail().getEmailAddress()); + toSlime(billingContact.address(), billingCursor); } private void toSlime(TenantContacts contacts, Cursor parentCursor) { @@ -892,15 +922,21 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { throw new IllegalArgumentException("All address fields must be set"); } - private TenantContact updateTenantInfoContact(Inspector insp, TenantName tenantName, TenantContact oldContact, boolean isBillingContact) { + private TaxId updateAndValidateTaxId(Inspector insp, TaxId old) { + if (!insp.valid()) return old; + var taxId = new TaxId(getString(insp.field("type"), old.type().value()), + getString(insp.field("code"), old.code().value())); + controller.serviceRegistry().billingController().validateTaxId(taxId); + return taxId; + } + + private TenantContact updateBillingContact(Inspector insp, TenantName tenantName, TenantContact oldContact) { if (!insp.valid()) return oldContact; var mergedEmail = optional("email", insp) .filter(address -> !address.equals(oldContact.email().getEmailAddress())) .map(address -> { - if (isBillingContact) - return new Email(address, true); - controller.mailVerifier().sendMailVerification(tenantName, address, PendingMailVerification.MailType.TENANT_CONTACT); + controller.mailVerifier().sendMailVerification(tenantName, address, PendingMailVerification.MailType.BILLING); return new Email(address, false); }) .orElse(oldContact.email()); @@ -914,9 +950,14 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { private TenantBilling updateTenantInfoBillingContact(Inspector insp, TenantName tenantName, TenantBilling oldContact) { if (!insp.valid()) return oldContact; + var purchaseOrder = optional("purchaseOrder", insp).map(PurchaseOrder::new).orElse(oldContact.getPurchaseOrder()); + var invoiceEmail = optional("invoiceEmail", insp).map(mail -> new Email(mail, false)).orElse(oldContact.getInvoiceEmail()); return TenantBilling.empty() - .withContact(updateTenantInfoContact(insp, tenantName, oldContact.contact(), true)) - .withAddress(updateTenantInfoAddress(insp.field("address"), oldContact.address())); + .withContact(updateBillingContact(insp, tenantName, oldContact.contact())) + .withAddress(updateTenantInfoAddress(insp.field("address"), oldContact.address())) + .withTaxId(updateAndValidateTaxId(insp.field("taxId"), oldContact.getTaxId())) + .withPurchaseOrder(purchaseOrder) + .withInvoiceEmail(invoiceEmail); } private TenantContacts updateTenantInfoContacts(Inspector insp, TenantName tenantName, TenantContacts oldContacts) { @@ -1048,6 +1089,7 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { cursor.setString("level", notificationLevelAsString(notification.level())); cursor.setString("type", notificationTypeAsString(notification.type())); if (!excludeMessages) { + cursor.setString("title", notification.title()); Cursor messagesArray = cursor.setArray("messages"); notification.messages().forEach(messagesArray::addString); } @@ -1071,6 +1113,7 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { case deployment: yield "deployment"; case feedBlock: yield "feedBlock"; case reindex: yield "reindex"; + case account: yield "account"; }; } @@ -1700,6 +1743,7 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { var mailType = switch (type) { case "contact" -> PendingMailVerification.MailType.TENANT_CONTACT; case "notifications" -> PendingMailVerification.MailType.NOTIFICATIONS; + case "billing" -> PendingMailVerification.MailType.BILLING; default -> throw new IllegalArgumentException("Unknown mail type " + type); }; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java index 3e147459dd0..18221d82e44 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java @@ -519,6 +519,8 @@ class JobControllerApiHandlerHelper { run.end().ifPresent(end -> runObject.setLong("end", end.toEpochMilli())); runObject.setString("status", nameOf(run.status())); toSlime(runObject, run.versions(), run.reason(), application); + run.cloudAccount().filter(account -> ! account.isUnspecified()) + .ifPresent(cloudAccount -> runObject.setObject("enclave").setString("cloudAccount", cloudAccount.value())); Cursor runStepsArray = runObject.setArray("steps"); run.steps().forEach((step, info) -> { Cursor runStepObject = runStepsArray.addObject(); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandler.java deleted file mode 100644 index ac3a8f2ee23..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandler.java +++ /dev/null @@ -1,515 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.restapi.billing; - -import com.yahoo.config.provision.TenantName; -import com.yahoo.container.jdisc.HttpRequest; -import com.yahoo.container.jdisc.HttpResponse; -import com.yahoo.container.jdisc.ThreadedHttpRequestHandler; -import com.yahoo.restapi.ErrorResponse; -import com.yahoo.restapi.JacksonJsonResponse; -import com.yahoo.restapi.MessageResponse; -import com.yahoo.restapi.Path; -import com.yahoo.restapi.SlimeJsonResponse; -import com.yahoo.restapi.StringResponse; -import com.yahoo.slime.Cursor; -import com.yahoo.slime.Inspector; -import com.yahoo.slime.Slime; -import com.yahoo.slime.SlimeUtils; -import com.yahoo.vespa.hosted.controller.ApplicationController; -import com.yahoo.vespa.hosted.controller.Controller; -import com.yahoo.vespa.hosted.controller.TenantController; -import com.yahoo.vespa.hosted.controller.api.integration.billing.Bill; -import com.yahoo.vespa.hosted.controller.api.integration.billing.BillingController; -import com.yahoo.vespa.hosted.controller.api.integration.billing.CollectionMethod; -import com.yahoo.vespa.hosted.controller.api.integration.billing.InstrumentOwner; -import com.yahoo.vespa.hosted.controller.api.integration.billing.PaymentInstrument; -import com.yahoo.vespa.hosted.controller.api.integration.billing.PlanId; -import com.yahoo.vespa.hosted.controller.api.integration.billing.PlanRegistry; -import com.yahoo.vespa.hosted.controller.api.role.Role; -import com.yahoo.vespa.hosted.controller.api.role.SecurityContext; -import com.yahoo.vespa.hosted.controller.restapi.ErrorResponses; -import com.yahoo.vespa.hosted.controller.tenant.Tenant; -import com.yahoo.yolean.Exceptions; - -import java.io.IOException; -import java.math.BigDecimal; -import java.security.Principal; -import java.time.LocalDate; -import java.time.format.DateTimeFormatter; -import java.time.format.DateTimeParseException; -import java.util.Comparator; -import java.util.List; -import java.util.Optional; -import java.util.Set; -import java.util.concurrent.Executor; - -/** - * @author andreer - * @author olaa - */ -public class BillingApiHandler extends ThreadedHttpRequestHandler { - - private static final DateTimeFormatter DATE_TIME_FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd"); - - private final BillingController billingController; - private final ApplicationController applicationController; - private final TenantController tenantController; - private final PlanRegistry planRegistry; - - public BillingApiHandler(Executor executor, - Controller controller) { - super(executor); - this.billingController = controller.serviceRegistry().billingController(); - this.planRegistry = controller.serviceRegistry().planRegistry(); - this.applicationController = controller.applications(); - this.tenantController = controller.tenants(); - } - - @Override - public HttpResponse handle(HttpRequest request) { - try { - Optional<String> userId = Optional.ofNullable(request.getJDiscRequest().getUserPrincipal()).map(Principal::getName); - if (userId.isEmpty()) - return ErrorResponse.unauthorized("Must be authenticated to use this API"); - - Path path = new Path(request.getUri()); - return switch (request.getMethod()) { - case GET -> handleGET(request, path, userId.get()); - case PATCH -> handlePATCH(request, path, userId.get()); - case DELETE -> handleDELETE(path, userId.get()); - case POST -> handlePOST(path, request, userId.get()); - default -> ErrorResponse.methodNotAllowed("Method '" + request.getMethod() + "' is not supported"); - }; - } - catch (IllegalArgumentException e) { - return ErrorResponse.badRequest(Exceptions.toMessageString(e)); - } catch (Exception e) { - return ErrorResponses.logThrowing(request, log, e); - } - } - - private HttpResponse handleGET(HttpRequest request, Path path, String userId) { - if (path.matches("/billing/v1/tenant/{tenant}/token")) return getToken(path.get("tenant"), userId); - if (path.matches("/billing/v1/tenant/{tenant}/instrument")) return getInstruments(path.get("tenant"), userId); - if (path.matches("/billing/v1/tenant/{tenant}/billing")) return getBilling(path.get("tenant"), request.getProperty("until")); - if (path.matches("/billing/v1/tenant/{tenant}/plan")) return getPlan(path.get("tenant")); - if (path.matches("/billing/v1/billing")) return getBillingAllTenants(request.getProperty("until")); - if (path.matches("/billing/v1/invoice/export")) return getAllBills(); - if (path.matches("/billing/v1/invoice/tenant/{tenant}/line-item")) return getLineItems(path.get("tenant")); - if (path.matches("/billing/v1/plans")) return getPlans(); - return ErrorResponse.notFoundError("Nothing at " + path); - } - - private HttpResponse getAllBills() { - var bills = billingController.getBills(); - var headers = new String[]{ "ID", "Tenant", "From", "To", "CpuHours", "MemoryHours", "DiskHours", "Cpu", "Memory", "Disk", "Additional" }; - var rows = bills.stream() - .map(bill -> { - return new Object[] { - bill.id().value(), bill.tenant().value(), - bill.getStartDate().format(DateTimeFormatter.ISO_LOCAL_DATE), - bill.getEndDate().format(DateTimeFormatter.ISO_LOCAL_DATE), - bill.sumCpuHours(), bill.sumMemoryHours(), bill.sumDiskHours(), - bill.sumCpuCost(), bill.sumMemoryCost(), bill.sumDiskCost(), - bill.sumAdditionalCost() - }; - }) - .toList(); - return new CsvResponse(headers, rows); - } - - private HttpResponse handlePATCH(HttpRequest request, Path path, String userId) { - if (path.matches("/billing/v1/tenant/{tenant}/instrument")) return patchActiveInstrument(request, path.get("tenant"), userId); - if (path.matches("/billing/v1/tenant/{tenant}/plan")) return patchPlan(request, path.get("tenant")); - if (path.matches("/billing/v1/tenant/{tenant}/collection")) return patchCollectionMethod(request, path.get("tenant")); - return ErrorResponse.notFoundError("Nothing at " + path); - - } - - private HttpResponse handleDELETE(Path path, String userId) { - if (path.matches("/billing/v1/tenant/{tenant}/instrument/{instrument}")) return deleteInstrument(path.get("tenant"), userId, path.get("instrument")); - if (path.matches("/billing/v1/invoice/line-item/{line-item-id}")) return deleteLineItem(path.get("line-item-id")); - return ErrorResponse.notFoundError("Nothing at " + path); - - } - - private HttpResponse handlePOST(Path path, HttpRequest request, String userId) { - if (path.matches("/billing/v1/invoice")) return createBill(request, userId); - if (path.matches("/billing/v1/invoice/{invoice-id}/status")) return setBillStatus(request, path.get("invoice-id"), userId); - if (path.matches("/billing/v1/invoice/tenant/{tenant}/line-item")) return addLineItem(request, path.get("tenant"), userId); - return ErrorResponse.notFoundError("Nothing at " + path); - - } - - private HttpResponse getPlan(String tenant) { - var plan = billingController.getPlan(TenantName.from(tenant)); - var slime = new Slime(); - var root = slime.setObject(); - root.setString("tenant", tenant); - root.setString("plan", plan.value()); - return new SlimeJsonResponse(slime); - } - - private HttpResponse patchPlan(HttpRequest request, String tenant) { - var tenantName = TenantName.from(tenant); - var slime = inspectorOrThrow(request); - var planId = PlanId.from(slime.field("plan").asString()); - var roles = requestRoles(request); - var isAccountant = roles.contains(Role.hostedAccountant()); - - var hasDeployments = hasDeployments(tenantName); - var result = billingController.setPlan(tenantName, planId, hasDeployments, isAccountant); - - if (result.isSuccess()) - return new StringResponse("Plan: " + planId.value()); - - return ErrorResponse.forbidden(result.getErrorMessage().orElse("Invalid plan change")); - } - - private HttpResponse patchCollectionMethod(HttpRequest request, String tenant) { - var tenantName = TenantName.from(tenant); - var slime = inspectorOrThrow(request); - var newMethod = slime.field("collection").valid() ? - slime.field("collection").asString().toUpperCase() : - slime.field("collectionMethod").asString().toUpperCase(); - if (newMethod.isEmpty()) return ErrorResponse.badRequest("No collection method specified"); - - try { - var result = billingController.setCollectionMethod(tenantName, CollectionMethod.valueOf(newMethod)); - if (result.isSuccess()) - return new StringResponse("Collection method updated to " + newMethod); - - return ErrorResponse.forbidden(result.getErrorMessage().orElse("Invalid collection method change")); - } catch (IllegalArgumentException iea){ - return ErrorResponse.badRequest("Invalid collection method: " + newMethod); - } - } - - private HttpResponse getBillingAllTenants(String until) { - try { - var untilDate = untilParameter(until); - var uncommittedBills = billingController.createUncommittedBills(untilDate); - - var slime = new Slime(); - var root = slime.setObject(); - root.setString("until", untilDate.format(DateTimeFormatter.ISO_DATE)); - var tenants = root.setArray("tenants"); - - tenantController.asList().stream().sorted(Comparator.comparing(Tenant::name)).forEach(tenant -> { - var bill = uncommittedBills.get(tenant.name()); - var tc = tenants.addObject(); - tc.setString("tenant", tenant.name().value()); - getPlanForTenant(tc, tenant.name()); - getCollectionForTenant(tc, tenant.name()); - renderCurrentUsage(tc.setObject("current"), bill); - renderAdditionalItems(tc.setObject("additional").setArray("items"), billingController.getUnusedLineItems(tenant.name())); - - billingController.getDefaultInstrument(tenant.name()).ifPresent(card -> - renderInstrument(tc.setObject("payment"), card) - ); - }); - - return new SlimeJsonResponse(slime); - } catch (DateTimeParseException e) { - return ErrorResponse.badRequest("Could not parse date: " + until); - } - } - - private void getCollectionForTenant(Cursor tc, TenantName tenant) { - var collection = billingController.getCollectionMethod(tenant); - tc.setString("collection", collection.name()); - } - - private HttpResponse addLineItem(HttpRequest request, String tenant, String userId) { - Inspector inspector = inspectorOrThrow(request); - - Optional<Bill.Id> billId = SlimeUtils.optionalString(inspector.field("billId")).map(Bill.Id::of); - - billingController.addLineItem( - TenantName.from(tenant), - getInspectorFieldOrThrow(inspector, "description"), - new BigDecimal(getInspectorFieldOrThrow(inspector, "amount")), - billId, - userId); - - return new MessageResponse("Added line item for tenant " + tenant); - } - - private HttpResponse setBillStatus(HttpRequest request, String billId, String userId) { - Inspector inspector = inspectorOrThrow(request); - String status = getInspectorFieldOrThrow(inspector, "status"); - billingController.updateBillStatus(Bill.Id.of(billId), userId, status); - return new MessageResponse("Updated status of invoice " + billId); - } - - private HttpResponse createBill(HttpRequest request, String userId) { - Inspector inspector = inspectorOrThrow(request); - TenantName tenantName = TenantName.from(getInspectorFieldOrThrow(inspector, "tenant")); - - LocalDate startDate = LocalDate.parse(getInspectorFieldOrThrow(inspector, "startTime")); - LocalDate endDate = LocalDate.parse(getInspectorFieldOrThrow(inspector, "endTime")); - - var billId = billingController.createBillForPeriod(tenantName, startDate, endDate, userId); - - Slime slime = new Slime(); - Cursor root = slime.setObject(); - root.setString("message", "Created invoice with ID " + billId.value()); - root.setString("id", billId.value()); - return new SlimeJsonResponse(slime); - } - - private HttpResponse getInstruments(String tenant, String userId) { - var instrumentListResponse = billingController.listInstruments(TenantName.from(tenant), userId); - return new JacksonJsonResponse<>(200, instrumentListResponse); - } - - private HttpResponse getToken(String tenant, String userId) { - return new StringResponse(billingController.createClientToken(tenant, userId)); - } - - private HttpResponse getBilling(String tenant, String until) { - try { - var untilDate = untilParameter(until); - var tenantId = TenantName.from(tenant); - var slimeResponse = new Slime(); - var root = slimeResponse.setObject(); - - root.setString("until", untilDate.format(DateTimeFormatter.ISO_DATE)); - - getPlanForTenant(root, tenantId); - renderCurrentUsage(root.setObject("current"), getCurrentUsageForTenant(tenantId, untilDate)); - renderAdditionalItems(root.setObject("additional").setArray("items"), billingController.getUnusedLineItems(tenantId)); - renderBills(root.setArray("bills"), getBillsForTenant(tenantId)); - - billingController.getDefaultInstrument(tenantId).ifPresent( card -> - renderInstrument(root.setObject("payment"), card) - ); - - root.setString("collection", billingController.getCollectionMethod(tenantId).name()); - return new SlimeJsonResponse(slimeResponse); - } catch (DateTimeParseException e) { - return ErrorResponse.badRequest("Could not parse date: " + until); - } - } - - private HttpResponse getPlans() { - var slime = new Slime(); - var root = slime.setObject(); - var plans = root.setArray("plans"); - for (var plan : planRegistry.all()) { - var p = plans.addObject(); - p.setString("id", plan.id().value()); - p.setString("name", plan.displayName()); - } - return new SlimeJsonResponse(slime); - } - - private HttpResponse getLineItems(String tenant) { - var slimeResponse = new Slime(); - var root = slimeResponse.setObject(); - var lineItems = root.setArray("lineItems"); - - billingController.getUnusedLineItems(TenantName.from(tenant)) - .forEach(lineItem -> { - var itemCursor = lineItems.addObject(); - renderLineItemToCursor(itemCursor, lineItem); - }); - - return new SlimeJsonResponse(slimeResponse); - } - - private void getPlanForTenant(Cursor cursor, TenantName tenant) { - PlanId plan = billingController.getPlan(tenant); - cursor.setString("plan", plan.value()); - cursor.setString("planName", billingController.getPlanDisplayName(plan)); - } - - private void renderInstrument(Cursor cursor, PaymentInstrument instrument) { - cursor.setString("pi-id", instrument.getId()); - cursor.setString("type", instrument.getType()); - cursor.setString("brand", instrument.getBrand()); - cursor.setString("endingWith", instrument.getEndingWith()); - cursor.setString("expiryDate", instrument.getExpiryDate()); - cursor.setString("displayText", instrument.getDisplayText()); - cursor.setString("nameOnCard", instrument.getNameOnCard()); - cursor.setString("addressLine1", instrument.getAddressLine1()); - cursor.setString("addressLine2", instrument.getAddressLine2()); - cursor.setString("zip", instrument.getZip()); - cursor.setString("city", instrument.getCity()); - cursor.setString("state", instrument.getState()); - cursor.setString("country", instrument.getCountry()); - - } - - private void renderCurrentUsage(Cursor cursor, Bill currentUsage) { - if (currentUsage == null) return; - cursor.setString("amount", currentUsage.sum().toPlainString()); - cursor.setString("status", "accrued"); - cursor.setString("from", currentUsage.getStartDate().format(DATE_TIME_FORMATTER)); - var itemsCursor = cursor.setArray("items"); - currentUsage.lineItems().forEach(lineItem -> { - var itemCursor = itemsCursor.addObject(); - renderLineItemToCursor(itemCursor, lineItem); - }); - } - - private void renderAdditionalItems(Cursor cursor, List<Bill.LineItem> items) { - items.forEach(item -> { - renderLineItemToCursor(cursor.addObject(), item); - }); - } - - private Bill getCurrentUsageForTenant(TenantName tenant, LocalDate until) { - return billingController.createUncommittedBill(tenant, until); - } - - private List<Bill> getBillsForTenant(TenantName tenant) { - return billingController.getBillsForTenant(tenant); - } - - private void renderBills(Cursor cursor, List<Bill> bills) { - bills.forEach(bill -> { - var billCursor = cursor.addObject(); - renderBillToCursor(billCursor, bill); - }); - } - - private void renderBillToCursor(Cursor billCursor, Bill bill) { - billCursor.setString("id", bill.id().value()); - billCursor.setString("from", bill.getStartDate().format(DATE_TIME_FORMATTER)); - billCursor.setString("to", bill.getEndDate().format(DATE_TIME_FORMATTER)); - - billCursor.setString("amount", bill.sum().toString()); - billCursor.setString("status", bill.status()); - var statusCursor = billCursor.setArray("statusHistory"); - renderStatusHistory(statusCursor, bill.statusHistory()); - - - var lineItemsCursor = billCursor.setArray("items"); - bill.lineItems().forEach(lineItem -> { - var itemCursor = lineItemsCursor.addObject(); - renderLineItemToCursor(itemCursor, lineItem); - }); - } - - private void renderStatusHistory(Cursor cursor, Bill.StatusHistory statusHistory) { - statusHistory.getHistory() - .entrySet() - .stream() - .forEach(entry -> { - var c = cursor.addObject(); - c.setString("at", entry.getKey().format(DATE_TIME_FORMATTER)); - c.setString("status", entry.getValue()); - }); - } - - private void renderLineItemToCursor(Cursor cursor, Bill.LineItem lineItem) { - cursor.setString("id", lineItem.id()); - cursor.setString("description", lineItem.description()); - cursor.setString("amount", lineItem.amount().toString()); - cursor.setString("plan", lineItem.plan()); - cursor.setString("planName", billingController.getPlanDisplayName(PlanId.from(lineItem.plan()))); - - lineItem.applicationId().ifPresent(appId -> { - cursor.setString("application", appId.application().value()); - cursor.setString("instance", appId.instance().value()); - }); - lineItem.zoneId().ifPresent(zoneId -> - cursor.setString("zone", zoneId.value()) - ); - - lineItem.getArchitecture().ifPresent(architecture -> { - cursor.setString("architecture", architecture.name()); - }); - - cursor.setLong("majorVersion", lineItem.getMajorVersion()); - - if (! lineItem.getCloudAccount().isUnspecified()) - cursor.setString("cloudAccount", lineItem.getCloudAccount().account()); - - lineItem.getCpuHours().ifPresent(cpuHours -> - cursor.setString("cpuHours", cpuHours.toString()) - ); - lineItem.getMemoryHours().ifPresent(memoryHours -> - cursor.setString("memoryHours", memoryHours.toString()) - ); - lineItem.getDiskHours().ifPresent(diskHours -> - cursor.setString("diskHours", diskHours.toString()) - ); - lineItem.getGpuHours().ifPresent(gpuHours -> - cursor.setString("gpuHours", gpuHours.toString()) - ); - lineItem.getCpuCost().ifPresent(cpuCost -> - cursor.setString("cpuCost", cpuCost.toString()) - ); - lineItem.getMemoryCost().ifPresent(memoryCost -> - cursor.setString("memoryCost", memoryCost.toString()) - ); - lineItem.getDiskCost().ifPresent(diskCost -> - cursor.setString("diskCost", diskCost.toString()) - ); - lineItem.getGpuCost().ifPresent(gpuCost -> - cursor.setString("gpuCost", gpuCost.toString()) - ); - } - - private HttpResponse deleteInstrument(String tenant, String userId, String instrument) { - if (billingController.deleteInstrument(TenantName.from(tenant), userId, instrument)) { - return new StringResponse("OK"); - } else { - return ErrorResponse.forbidden("Cannot delete payment instrument you don't own"); - } - } - - private HttpResponse deleteLineItem(String lineItemId) { - billingController.deleteLineItem(lineItemId); - return new MessageResponse("Succesfully deleted line item " + lineItemId); - } - - private HttpResponse patchActiveInstrument(HttpRequest request, String tenant, String userId) { - var inspector = inspectorOrThrow(request); - String instrumentId = getInspectorFieldOrThrow(inspector, "active"); - InstrumentOwner paymentInstrument = new InstrumentOwner(TenantName.from(tenant), userId, instrumentId, true); - boolean success = billingController.setActivePaymentInstrument(paymentInstrument); - return success ? new StringResponse("OK") : ErrorResponse.internalServerError("Failed to patch active instrument"); - } - - private Inspector inspectorOrThrow(HttpRequest request) { - try { - return SlimeUtils.jsonToSlime(request.getData().readAllBytes()).get(); - } catch (IOException e) { - throw new IllegalArgumentException("Failed to parse request body"); - } - } - - private static String getInspectorFieldOrThrow(Inspector inspector, String field) { - if (!inspector.field(field).valid()) - throw new IllegalArgumentException("Field " + field + " cannot be null"); - return inspector.field(field).asString(); - } - - private LocalDate untilParameter(String until) { - if (until == null || until.isEmpty() || until.isBlank()) - return LocalDate.now(); - return LocalDate.parse(until); - } - - private boolean hasDeployments(TenantName tenantName) { - return applicationController.asList(tenantName) - .stream() - .flatMap(app -> app.instances().values() - .stream() - .flatMap(instance -> instance.deployments().values().stream()) - ) - .count() > 0; - } - - private static Set<Role> requestRoles(HttpRequest request) { - return Optional.ofNullable(request.getJDiscRequest().context().get(SecurityContext.ATTRIBUTE_NAME)) - .filter(SecurityContext.class::isInstance) - .map(SecurityContext.class::cast) - .map(SecurityContext::roles) - .orElseThrow(() -> new IllegalArgumentException("Attribute '" + SecurityContext.ATTRIBUTE_NAME + "' was not set on request")); - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandlerV2.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandlerV2.java index da83073609d..b4237a46c5a 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandlerV2.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandlerV2.java @@ -12,20 +12,24 @@ import com.yahoo.restapi.SlimeJsonResponse; import com.yahoo.slime.Cursor; import com.yahoo.slime.Inspector; import com.yahoo.slime.Slime; +import com.yahoo.slime.SlimeUtils; import com.yahoo.slime.Type; import com.yahoo.vespa.hosted.controller.ApplicationController; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.TenantController; import com.yahoo.vespa.hosted.controller.api.integration.billing.Bill; import com.yahoo.vespa.hosted.controller.api.integration.billing.BillingController; +import com.yahoo.vespa.hosted.controller.api.integration.billing.BillingReporter; import com.yahoo.vespa.hosted.controller.api.integration.billing.CollectionMethod; import com.yahoo.vespa.hosted.controller.api.integration.billing.Plan; import com.yahoo.vespa.hosted.controller.api.integration.billing.PlanId; import com.yahoo.vespa.hosted.controller.api.integration.billing.PlanRegistry; import com.yahoo.vespa.hosted.controller.api.integration.billing.Quota; +import com.yahoo.vespa.hosted.controller.api.integration.billing.StatusHistory; import com.yahoo.vespa.hosted.controller.api.role.Role; import com.yahoo.vespa.hosted.controller.api.role.SecurityContext; import com.yahoo.vespa.hosted.controller.restapi.ErrorResponses; +import com.yahoo.vespa.hosted.controller.tenant.BillingReference; import com.yahoo.vespa.hosted.controller.tenant.CloudTenant; import com.yahoo.vespa.hosted.controller.tenant.Tenant; @@ -53,6 +57,7 @@ public class BillingApiHandlerV2 extends RestApiRequestHandler<BillingApiHandler private final ApplicationController applications; private final TenantController tenants; private final BillingController billing; + private final BillingReporter billingReporter; private final PlanRegistry planRegistry; private final Clock clock; @@ -63,11 +68,18 @@ public class BillingApiHandlerV2 extends RestApiRequestHandler<BillingApiHandler this.billing = controller.serviceRegistry().billingController(); this.planRegistry = controller.serviceRegistry().planRegistry(); this.clock = controller.serviceRegistry().clock(); + this.billingReporter = controller.serviceRegistry().billingReporter(); } private static RestApi createRestApi(BillingApiHandlerV2 self) { return RestApi.builder() /* + * This is the API that is tenant agnostic + */ + .addRoute(RestApi.route("/billing/v2/countries") + .get(self::acceptedCountries)) + + /* * This is the API that is available to tenants to view their status */ .addRoute(RestApi.route("/billing/v2/tenant/{tenant}") @@ -86,9 +98,22 @@ public class BillingApiHandlerV2 extends RestApiRequestHandler<BillingApiHandler .get(self::accountant)) .addRoute(RestApi.route("/billing/v2/accountant/preview") .get(self::accountantPreview)) - .addRoute(RestApi.route("/billing/v2/accountant/preview/tenant/{tenant}") + .addRoute(RestApi.route("/billing/v2/accountant/tenant/{tenant}") + .get(self::accountantTenant)) + .addRoute(RestApi.route("/billing/v2/accountant/tenant/{tenant}/preview") .get(self::previewBill) .post(Slime.class, self::createBill)) + .addRoute(RestApi.route("/billing/v2/accountant/tenant/{tenant}/items") + .get(self::additionalItems) + .post(Slime.class, self::newAdditionalItem)) + .addRoute(RestApi.route("/billing/v2/accountant/tenant/{tenant}/item/{item}") + .delete(self::deleteAdditionalItem)) + .addRoute(RestApi.route("/billing/v2/accountant/tenant/{tenant}/plan") + .get(self::accountantTenantPlan) + .post(Slime.class, self::setAccountantTenantPlan)) + .addRoute(RestApi.route("/billing/v2/accountant/tenant/{tenant}/collection") + .get(self::accountantTenantCollection) + .post(Slime.class, self::setAccountantTenantCollection)) .addRoute(RestApi.route("/billing/v2/accountant/bill/{invoice}/export") .put(Slime.class, self::putAccountantInvoiceExport)) .addRoute(RestApi.route("/billing/v2/accountant/plans") @@ -97,6 +122,27 @@ public class BillingApiHandlerV2 extends RestApiRequestHandler<BillingApiHandler .build(); } + // ---------- AUX API ------------- + + private SlimeJsonResponse acceptedCountries(RestApi.RequestContext ctx) { + var response = new Slime(); + var countries = response.setObject().setArray("countries"); + billing.getAcceptedCountries().countries().forEach(country -> { + var countryCursor = countries.addObject(); + countryCursor.setString("code", country.code()); + countryCursor.setString("name", country.displayName()); + var taxTypesCursors = countryCursor.setArray("taxTypes"); + country.taxTypes().forEach(taxType -> { + var taxTypeCursor = taxTypesCursors.addObject(); + taxTypeCursor.setString("id", taxType.id()); + taxTypeCursor.setString("description", taxType.description()); + taxTypeCursor.setString("pattern", taxType.pattern()); + taxTypeCursor.setString("example", taxType.example()); + }); + }); + return new SlimeJsonResponse(response, /*compact*/false); + } + // ---------- TENANT API ---------- private Slime tenant(RestApi.RequestContext requestContext) { @@ -294,13 +340,141 @@ public class BillingApiHandlerV2 extends RestApiRequestHandler<BillingApiHandler var cloudTenant = tenants.require(bill.tenant(), CloudTenant.class); var exportMethod = slime.get().field("method").asString(); - var result = billing.exportBill(bill, exportMethod, cloudTenant); + var result = billingReporter.exportBill(bill, exportMethod, cloudTenant); var responseSlime = new Slime(); responseSlime.setObject().setString("invoiceId", result); return new SlimeJsonResponse(responseSlime); } + private MessageResponse deleteAdditionalItem(RestApi.RequestContext requestContext) { + var tenantName = TenantName.from(requestContext.pathParameters().getStringOrThrow("tenant")); + var tenant = tenants.get(tenantName).orElseThrow(() -> new RestApiException.NotFound("No such tenant: " + tenantName)); + + var itemId = requestContext.pathParameters().getStringOrThrow("item"); + + var items = billing.getUnusedLineItems(tenant.name()); + var candidate = items.stream().filter(item -> item.id().equals(itemId)).findAny(); + + if (candidate.isEmpty()) { + throw new RestApiException.NotFound("Could not find item with ID " + itemId); + } + + billing.deleteLineItem(itemId);; + + return new MessageResponse("Successfully deleted line item " + itemId); + } + + private MessageResponse newAdditionalItem(RestApi.RequestContext requestContext, Slime body) { + var tenantName = TenantName.from(requestContext.pathParameters().getStringOrThrow("tenant")); + var tenant = tenants.get(tenantName).orElseThrow(() -> new RestApiException.NotFound("No such tenant: " + tenantName)); + + var inspector = body.get(); + + var billId = SlimeUtils.optionalString(inspector.field("billId")).map(Bill.Id::of); + + billing.addLineItem( + tenant.name(), + getInspectorFieldOrThrow(inspector, "description"), + new BigDecimal(getInspectorFieldOrThrow(inspector, "amount")), + billId, + requestContext.userPrincipalOrThrow().getName()); + + return new MessageResponse("Added line item for tenant " + tenantName); + } + + private Slime additionalItems(RestApi.RequestContext requestContext) { + var tenantName = TenantName.from(requestContext.pathParameters().getStringOrThrow("tenant")); + var tenant = tenants.get(tenantName).orElseThrow(() -> new RestApiException.NotFound("No such tenant: " + tenantName)); + + var slime = new Slime(); + var items = slime.setObject().setArray("items"); + + billing.getUnusedLineItems(tenant.name()).forEach(item -> { + var itemCursor = items.addObject(); + toSlime(itemCursor, item); + }); + + return slime; + } + + private MessageResponse setAccountantTenantPlan(RestApi.RequestContext requestContext, Slime body) { + var tenantName = TenantName.from(requestContext.pathParameters().getStringOrThrow("tenant")); + var tenant = tenants.require(tenantName, CloudTenant.class); + + var planId = PlanId.from(getInspectorFieldOrThrow(body.get(), "id")); + var response = billing.setPlan(tenant.name(), planId, false, true); + + if (response.isSuccess()) { + return new MessageResponse("Plan: " + planId.value()); + } else { + throw new RestApiException.BadRequest("Could not change plan: " + response.getErrorMessage()); + } + } + + private Slime accountantTenantPlan(RestApi.RequestContext requestContext) { + var tenantName = TenantName.from(requestContext.pathParameters().getStringOrThrow("tenant")); + var tenant = tenants.require(tenantName, CloudTenant.class); + + var planId = billing.getPlan(tenant.name()); + var plan = planRegistry.plan(planId); + + if (plan.isEmpty()) { + throw new RestApiException.BadRequest("Plan with ID '" + planId.value() + "' does not exist"); + } + + var slime = new Slime(); + var root = slime.setObject(); + root.setString("id", plan.get().id().value()); + root.setString("name", plan.get().displayName()); + + return slime; + } + + private MessageResponse setAccountantTenantCollection(RestApi.RequestContext requestContext, Slime body) { + var tenantName = TenantName.from(requestContext.pathParameters().getStringOrThrow("tenant")); + var tenant = tenants.require(tenantName, CloudTenant.class); + + var collection = CollectionMethod.valueOf(getInspectorFieldOrThrow(body.get(), "collection")); + var result = billing.setCollectionMethod(tenant.name(), collection); + + if (result.isSuccess()) { + return new MessageResponse("Collection: " + collection.name()); + } else { + throw new RestApiException.BadRequest("Could not change collection method: " + result.getErrorMessage()); + } + } + + private Slime accountantTenantCollection(RestApi.RequestContext requestContext) { + var tenantName = TenantName.from(requestContext.pathParameters().getStringOrThrow("tenant")); + var tenant = tenants.require(tenantName, CloudTenant.class); + + var collection = billing.getCollectionMethod(tenant.name()); + + var slime = new Slime(); + var root = slime.setObject(); + root.setString("collection", collection.name()); + + return slime; + } + + private Slime accountantTenant(RestApi.RequestContext requestContext) { + var tenantName = TenantName.from(requestContext.pathParameters().getStringOrThrow("tenant")); + var tenant = tenants.require(tenantName, CloudTenant.class); + + var slime = new Slime(); + var root = slime.setObject(); + + var planId = billing.getPlan(tenant.name()); + var plan = planRegistry.plan(planId); + + var collection = billing.getCollectionMethod(tenant.name()); + + toSlime(root, tenant, planId, plan, collection); + + return slime; + } + // --------- INVOICE RENDERING ---------- private void invoicesSummaryToSlime(Cursor slime, List<Bill> bills) { @@ -312,7 +486,7 @@ public class BillingApiHandlerV2 extends RestApiRequestHandler<BillingApiHandler slime.setString("from", bill.getStartDate().format(DateTimeFormatter.ISO_LOCAL_DATE)); slime.setString("to", bill.getEndDate().format(DateTimeFormatter.ISO_LOCAL_DATE)); slime.setString("total", bill.sum().toString()); - slime.setString("status", bill.status()); + slime.setString("status", bill.status().value()); } private void usageToSlime(Cursor slime, Bill bill) { @@ -327,16 +501,16 @@ public class BillingApiHandlerV2 extends RestApiRequestHandler<BillingApiHandler slime.setString("from", bill.getStartDate().format(DateTimeFormatter.ISO_LOCAL_DATE)); slime.setString("to", bill.getEndDate().format(DateTimeFormatter.ISO_LOCAL_DATE)); slime.setString("total", bill.sum().toString()); - slime.setString("status", bill.status()); + slime.setString("status", bill.status().value()); toSlime(slime.setArray("statusHistory"), bill.statusHistory()); toSlime(slime.setArray("items"), bill.lineItems()); } - private void toSlime(Cursor slime, Bill.StatusHistory history) { + private void toSlime(Cursor slime, StatusHistory history) { history.getHistory().forEach((key, value) -> { var c = slime.addObject(); c.setString("at", key.format(DateTimeFormatter.ISO_OFFSET_DATE_TIME)); - c.setString("status", value); + c.setString("status", value.value()); }); } @@ -352,7 +526,7 @@ public class BillingApiHandlerV2 extends RestApiRequestHandler<BillingApiHandler item.getArchitecture().ifPresent(arch -> slime.setString("architecture", arch.name())); slime.setLong("majorVersion", item.getMajorVersion()); if (! item.getCloudAccount().isUnspecified()) - slime.setString("cloudAccount", item.getCloudAccount().account()); + slime.setString("cloudAccount", item.getCloudAccount().value()); item.applicationId().ifPresent(appId -> { slime.setString("application", appId.application().value()); @@ -364,6 +538,7 @@ public class BillingApiHandlerV2 extends RestApiRequestHandler<BillingApiHandler toSlime(slime.setObject("cpu"), item.getCpuHours(), item.getCpuCost()); toSlime(slime.setObject("memory"), item.getMemoryHours(), item.getMemoryCost()); toSlime(slime.setObject("disk"), item.getDiskHours(), item.getDiskCost()); + toSlime(slime.setObject("gpu"), item.getGpuHours(), item.getGpuCost()); } private void toSlime(Cursor slime, Optional<BigDecimal> hours, Optional<BigDecimal> cost) { @@ -371,6 +546,33 @@ public class BillingApiHandlerV2 extends RestApiRequestHandler<BillingApiHandler cost.ifPresent(c -> slime.setString("cost", c.toString())); } + private void toSlime(Cursor slime, CloudTenant tenant, PlanId planId, Optional<Plan> plan, CollectionMethod method) { + slime.setString("tenant", tenant.name().value()); + toSlime(slime.setObject("plan"), planId, plan); + toSlime(slime.setObject("billing"), tenant.billingReference()); + slime.setString("collection", method.name()); + } + + private void toSlime(Cursor slime, PlanId planId, Optional<Plan> plan) { + slime.setString("id", planId.value()); + if (plan.isPresent()) { + slime.setString("name", plan.get().displayName()); + slime.setBool("billed", plan.get().isBilled()); + slime.setBool("supported", plan.get().isSupported()); + } else { + slime.setString("name", "UNKNOWN"); + slime.setBool("billed", false); + slime.setBool("supported", false); + } + } + + private void toSlime(Cursor slime, Optional<BillingReference> billingReference) { + if (billingReference.isPresent()) { + slime.setString("id", billingReference.get().reference()); + slime.setLong("lastUpdated", billingReference.get().updated().toEpochMilli()); + } + } + private List<Object[]> toCsv(Bill bill) { return List.<Object[]>of(new Object[]{ bill.id().value(), bill.tenant().value(), diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/pricing/PricingApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/pricing/PricingApiHandler.java deleted file mode 100644 index 7136b77572c..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/pricing/PricingApiHandler.java +++ /dev/null @@ -1,191 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.restapi.pricing; - -import com.yahoo.collections.Pair; -import com.yahoo.component.annotation.Inject; -import com.yahoo.config.provision.ClusterResources; -import com.yahoo.config.provision.NodeResources; -import com.yahoo.container.jdisc.HttpRequest; -import com.yahoo.container.jdisc.HttpResponse; -import com.yahoo.container.jdisc.ThreadedHttpRequestHandler; -import com.yahoo.restapi.ErrorResponse; -import com.yahoo.restapi.Path; -import com.yahoo.restapi.SlimeJsonResponse; -import com.yahoo.slime.Cursor; -import com.yahoo.slime.Slime; -import com.yahoo.text.Text; -import com.yahoo.vespa.hosted.controller.Controller; -import com.yahoo.vespa.hosted.controller.api.integration.billing.Plan; -import com.yahoo.vespa.hosted.controller.api.integration.pricing.PriceInformation; -import com.yahoo.vespa.hosted.controller.api.integration.pricing.PricingInfo; -import com.yahoo.vespa.hosted.controller.restapi.ErrorResponses; -import com.yahoo.yolean.Exceptions; - -import java.math.BigDecimal; -import java.math.RoundingMode; -import java.net.URLDecoder; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.Optional; -import java.util.logging.Logger; - -import static com.yahoo.jdisc.http.HttpRequest.Method.GET; -import static com.yahoo.restapi.ErrorResponse.methodNotAllowed; -import static com.yahoo.vespa.hosted.controller.api.integration.pricing.PricingInfo.SupportLevel; -import static java.lang.Double.parseDouble; -import static java.lang.Integer.parseInt; -import static java.nio.charset.StandardCharsets.UTF_8; - -/** - * API for calculating price information - * - * @author hmusum - */ -@SuppressWarnings("unused") // Handler -public class PricingApiHandler extends ThreadedHttpRequestHandler { - - private static final Logger log = Logger.getLogger(PricingApiHandler.class.getName()); - - private final Controller controller; - - @Inject - public PricingApiHandler(Context parentCtx, Controller controller) { - super(parentCtx); - this.controller = controller; - } - - @Override - public HttpResponse handle(HttpRequest request) { - if (request.getMethod() != GET) - return methodNotAllowed("Method '" + request.getMethod() + "' is not supported"); - - try { - return handleGET(request); - } catch (IllegalArgumentException e) { - return ErrorResponse.badRequest(Exceptions.toMessageString(e)); - } catch (RuntimeException e) { - return ErrorResponses.logThrowing(request, log, e); - } - } - - private HttpResponse handleGET(HttpRequest request) { - Path path = new Path(request.getUri()); - if (path.matches("/pricing/v1/pricing")) return pricing(request); - - return ErrorResponse.notFoundError(Text.format("No '%s' handler at '%s'", request.getMethod(), - request.getUri().getPath())); - } - - private HttpResponse pricing(HttpRequest request) { - String rawQuery = request.getUri().getRawQuery(); - var priceParameters = parseQuery(rawQuery); - var price = controller.serviceRegistry().pricingController() - .price(priceParameters.clusterResources, priceParameters.pricingInfo, priceParameters.plan); - return response(price, priceParameters); - } - - private PriceParameters parseQuery(String rawQuery) { - if (rawQuery == null) throw new IllegalArgumentException("No price information found in query"); - String[] elements = URLDecoder.decode(rawQuery, UTF_8).split("&"); - - var supportLevel = SupportLevel.BASIC; - var enclave = false; - var committedSpend = 0d; - var plan = controller.serviceRegistry().planRegistry().defaultPlan(); // fallback to default plan if not supplied - List<ClusterResources> clusterResources = new ArrayList<>(); - - for (Pair<String, String> entry : keysAndValues(elements)) { - switch (entry.getFirst()) { - case "committedSpend" -> committedSpend = parseDouble(entry.getSecond()); - case "enclave" -> enclave = Boolean.parseBoolean(entry.getSecond()); - case "planId" -> plan = plan(entry.getSecond()) - .orElseThrow(() -> new IllegalArgumentException("Unknown plan id " + entry.getSecond())); - case "supportLevel" -> supportLevel = SupportLevel.valueOf(entry.getSecond().toUpperCase()); - case "resources" -> clusterResources.add(clusterResources(entry.getSecond())); - default -> throw new IllegalArgumentException("Unknown query parameter '" + entry.getFirst() + '\''); - } - } - if (clusterResources.isEmpty()) throw new IllegalArgumentException("No cluster resources found in query"); - - PricingInfo pricingInfo = new PricingInfo(enclave, supportLevel, committedSpend); - return new PriceParameters(clusterResources, pricingInfo, plan); - } - - private ClusterResources clusterResources(String resourcesString) { - String[] elements = resourcesString.split(","); - - var nodes = 0; - var vcpu = 0d; - var memoryGb = 0d; - var diskGb = 0d; - var gpuMemoryGb = 0d; - - for (var element : keysAndValues(elements)) { - switch (element.getFirst()) { - case "nodes" -> nodes = parseInt(element.getSecond()); - case "vcpu" -> vcpu = parseDouble(element.getSecond()); - case "memoryGb" -> memoryGb = parseDouble(element.getSecond()); - case "diskGb" -> diskGb = parseDouble(element.getSecond()); - case "gpuMemoryGb" -> gpuMemoryGb = parseDouble(element.getSecond()); - default -> throw new IllegalArgumentException("Unknown resource type '" + element.getFirst() + '\''); - } - } - - var nodeResources = new NodeResources(vcpu, memoryGb, diskGb, 0); // 0 bandwidth, not used in price calculation - if (gpuMemoryGb > 0) - nodeResources = nodeResources.with(new NodeResources.GpuResources(1, gpuMemoryGb)); - return new ClusterResources(nodes, 1, nodeResources); - } - - private List<Pair<String, String>> keysAndValues(String[] elements) { - return Arrays.stream(elements).map(element -> { - var index = element.indexOf("="); - if (index <= 0 || index == element.length() - 1) - throw new IllegalArgumentException("Error in query parameter, expected '=' between key and value: '" + element + '\''); - return new Pair<>(element.substring(0, index), element.substring(index + 1)); - }) - .toList(); - } - - private Optional<Plan> plan(String element) { - return controller.serviceRegistry().planRegistry().plan(element); - } - - private static SlimeJsonResponse response(PriceInformation priceInfo, PriceParameters priceParameters) { - var slime = new Slime(); - Cursor cursor = slime.setObject(); - - var array = cursor.setArray("priceInfo"); - addItem(array, supportLevelDescription(priceParameters), priceInfo.listPriceWithSupport()); - addItem(array, "Enclave discount", priceInfo.enclaveDiscount()); - addItem(array, "Volume discount", priceInfo.volumeDiscount()); - addItem(array, "Committed spend", priceInfo.committedAmountDiscount()); - - setBigDecimal(cursor, "totalAmount", priceInfo.totalAmount()); - - return new SlimeJsonResponse(slime); - } - - private static String supportLevelDescription(PriceParameters priceParameters) { - String supportLevel = priceParameters.pricingInfo.supportLevel().name(); - return supportLevel.substring(0,1).toUpperCase() + supportLevel.substring(1).toLowerCase() + " support unit price"; - } - - private static void addItem(Cursor array, String name, BigDecimal amount) { - if (amount.compareTo(BigDecimal.ZERO) != 0) { - var o = array.addObject(); - o.setString("description", name); - setBigDecimal(o, "amount", amount); - } - } - - private static void setBigDecimal(Cursor cursor, String name, BigDecimal value) { - cursor.setString(name, value.setScale(2, RoundingMode.HALF_UP).toPlainString()); - } - - private record PriceParameters(List<ClusterResources> clusterResources, PricingInfo pricingInfo, Plan plan) { - - } - -} diff --git a/controller-server/src/main/resources/mail/cloud-trial-notification.vm b/controller-server/src/main/resources/mail/cloud-trial-notification.vm new file mode 100644 index 00000000000..c1ba394bf8e --- /dev/null +++ b/controller-server/src/main/resources/mail/cloud-trial-notification.vm @@ -0,0 +1,3 @@ +<p> + $esc.html($cloudTrialMessage) +</p>
\ No newline at end of file diff --git a/controller-server/src/main/resources/mail/default-mail-content.vm b/controller-server/src/main/resources/mail/default-mail-content.vm new file mode 100644 index 00000000000..02de98b900d --- /dev/null +++ b/controller-server/src/main/resources/mail/default-mail-content.vm @@ -0,0 +1,131 @@ +<tbody> +<tr> + <td + align="left" + style=" + font-size: 0px; + padding: 0px 25px 0px 25px; + padding-top: 0px; + padding-right: 50px; + padding-bottom: 0px; + padding-left: 50px; + word-break: break-word; + " + > + <div + style=" + font-family: Open Sans, Helvetica, Arial, + sans-serif; + font-size: 13px; + line-height: 22px; + text-align: left; + color: #797e82; + " + > + <h1 + style=" + text-align: center; + color: #000000; + line-height: 32px; + " + > + $esc.html($mailTitle) + </h1> + </div> + </td> +</tr> +<tr> + <td + align="left" + style=" + font-size: 0px; + padding: 0px 25px 0px 25px; + padding-top: 0px; + padding-right: 50px; + padding-bottom: 0px; + padding-left: 50px; + word-break: break-word; + " + > + <div + style=" + font-family: Open Sans, Helvetica, Arial, + sans-serif; + font-size: 13px; + line-height: 22px; + text-align: left; + color: #797e82; + " + > + + #parse($mailMessageTemplate) + + </div> + </td> +</tr> +<tr> + <td + align="center" + vertical-align="middle" + style=" + font-size: 0px; + padding: 10px 25px; + padding-top: 20px; + padding-bottom: 20px; + word-break: break-word; + " + > + <table + border="0" + cellpadding="0" + cellspacing="0" + role="presentation" + style="border-collapse: separate; line-height: 100%" + > + <tbody> + <tr> + <td + align="center" + bgcolor="#005A8E" + role="presentation" + style=" + border: none; + border-radius: 100px; + cursor: auto; + mso-padding-alt: 15px 25px 15px 25px; + background: #005a8e; + " + valign="middle" + > + <a + href="$consoleLink" + style=" + display: inline-block; + background: #005a8e; + color: #ffffff; + font-family: Open Sans, Helvetica, Arial, + sans-serif; + font-size: 13px; + font-weight: normal; + line-height: 120%; + margin: 0; + text-decoration: none; + text-transform: none; + padding: 15px 25px 15px 25px; + mso-padding-alt: 0px; + border-radius: 100px; + " + target="_blank" + ><b style="font-weight: 700" + ><b style="font-weight: 700" + >Go to Console</b + ></b + ></a + > + </td> + </tr> + </tbody> + </table> + </td> +</tr> +</tbody>
\ No newline at end of file diff --git a/controller-server/src/main/resources/mail/mail-verification.tmpl b/controller-server/src/main/resources/mail/mail-verification.vm index 8a473e74755..340895812ca 100644 --- a/controller-server/src/main/resources/mail/mail-verification.tmpl +++ b/controller-server/src/main/resources/mail/mail-verification.vm @@ -366,7 +366,7 @@ " > <p style="margin: 10px 0; text-align: center"> - You have entered the email address <b>%{email}</b> in + You have entered the email address <b>$esc.html($email)</b> in Vespa Cloud. </p> <p style="margin: 10px 0; text-align: center"> @@ -411,7 +411,7 @@ valign="middle" > <a - href="https://%{consoleUrl}/verify?code=%{code}" + href="$verifyLink" style=" display: inline-block; background: #3b9fde; @@ -471,9 +471,9 @@ <a target="_blank" rel="noopener noreferrer" - href="https://%{consoleUrl}/verify?code=%{code}" + href="$verifyLink" style="color: #3b9fde" - >https://%{consoleUrl}/verify?code=%{code}</a + >$verifyLink</a > </p> </div> diff --git a/controller-server/src/main/resources/mail/mail.vm b/controller-server/src/main/resources/mail/mail.vm new file mode 100644 index 00000000000..1dbec781b3a --- /dev/null +++ b/controller-server/src/main/resources/mail/mail.vm @@ -0,0 +1,516 @@ +<!DOCTYPE html> +<html + xmlns="http://www.w3.org/1999/xhtml" + xmlns:v="urn:schemas-microsoft-com:vml" + xmlns:o="urn:schemas-microsoft-com:office:office" +> + <head> + <title></title> + <!--[if !mso]><!--> + <meta http-equiv="X-UA-Compatible" content="IE=edge" /> + <!--<![endif]--> + <meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /> + <meta name="viewport" content="width=device-width,initial-scale=1" /> + <style type="text/css"> + #outlook a { + padding: 0; + } + + body { + margin: 0; + padding: 0; + -webkit-text-size-adjust: 100%; + -ms-text-size-adjust: 100%; + } + + table, + td { + border-collapse: collapse; + mso-table-lspace: 0pt; + mso-table-rspace: 0pt; + } + + img { + border: 0; + height: auto; + line-height: 100%; + outline: none; + text-decoration: none; + -ms-interpolation-mode: bicubic; + } + + p { + display: block; + margin: 13px 0; + } + </style> + <!--[if mso]> + <noscript> + <xml> + <o:OfficeDocumentSettings> + <o:AllowPNG /> + <o:PixelsPerInch>96</o:PixelsPerInch> + </o:OfficeDocumentSettings> + </xml> + </noscript> + <![endif]--> + <!--[if lte mso 11]> + <style type="text/css"> + .mj-outlook-group-fix { + width: 100% !important; + } + </style> + <![endif]--> + <!--[if !mso]><!--> + <link + href="https://fonts.googleapis.com/css?family=Open Sans" + rel="stylesheet" + type="text/css" + /> + <style type="text/css"> + @import url(https://fonts.googleapis.com/css?family=Open Sans); + </style> + <!--<![endif]--> + <style type="text/css"> + @media only screen and (min-width: 480px) { + .mj-column-per-100 { + width: 100% !important; + max-width: 100%; + } + } + </style> + <style media="screen and (min-width:480px)"> + .moz-text-html .mj-column-per-100 { + width: 100% !important; + max-width: 100%; + } + </style> + <style type="text/css"> + [owa] .mj-column-per-100 { + width: 100% !important; + max-width: 100%; + } + </style> + <style type="text/css"> + @media only screen and (max-width: 480px) { + table.mj-full-width-mobile { + width: 100% !important; + } + + td.mj-full-width-mobile { + width: auto !important; + } + } + </style> + </head> + + <body style="word-spacing: normal; background-color: #f2f7fa"> + <div style="background-color: #f2f7fa"> + <!--[if mso | IE]><table align="center" border="0" cellpadding="0" cellspacing="0" class="" role="presentation" style="width:600px;" width="600" ><tr><td style="line-height:0px;font-size:0px;mso-line-height-rule:exactly;"><![endif]--> + <div style="margin: 0px auto; max-width: 600px"> + <table + align="center" + border="0" + cellpadding="0" + cellspacing="0" + role="presentation" + style="width: 100%" + > + <tbody> + <tr> + <td + style=" + direction: ltr; + font-size: 0px; + padding: 20px 0px 20px 0px; + padding-bottom: 0px; + padding-top: 0px; + text-align: center; + " + > + <!--[if mso | IE]><table role="presentation" border="0" cellpadding="0" cellspacing="0"><tr><td class="" style="vertical-align:top;width:600px;" ><![endif]--> + <div + class="mj-column-per-100 mj-outlook-group-fix" + style=" + font-size: 0px; + text-align: left; + direction: ltr; + display: inline-block; + vertical-align: top; + width: 100%; + " + > + <table + border="0" + cellpadding="0" + cellspacing="0" + role="presentation" + style="vertical-align: top" + width="100%" + > + <tbody> + <tr> + <td + align="left" + style=" + font-size: 0px; + padding: 0px 0px 0px 25px; + padding-top: 0px; + padding-bottom: 0px; + word-break: break-word; + " + > + <div + style=" + font-family: Open Sans, Helvetica, Arial, + sans-serif; + font-size: 11px; + line-height: 22px; + text-align: left; + color: #797e82; + " + > + <br /> + </div> + </td> + </tr> + </tbody> + </table> + </div> + <!--[if mso | IE]></td></tr></table><![endif]--> + </td> + </tr> + </tbody> + </table> + </div> + <!--[if mso | IE]></td></tr></table><table align="center" border="0" cellpadding="0" cellspacing="0" class="" role="presentation" style="width:600px;" width="600" bgcolor="#ffffff" ><tr><td style="line-height:0px;font-size:0px;mso-line-height-rule:exactly;"><![endif]--> + <div + style=" + background: #ffffff; + background-color: #ffffff; + margin: 0px auto; + max-width: 600px; + " + > + <table + align="center" + border="0" + cellpadding="0" + cellspacing="0" + role="presentation" + style="background: #ffffff; background-color: #ffffff; width: 100%" + > + <tbody> + <tr> + <td + style=" + direction: ltr; + font-size: 0px; + padding: 20px 0; + padding-bottom: 0px; + padding-left: 0px; + padding-right: 0px; + padding-top: 0px; + text-align: center; + " + > + <!--[if mso | IE]><table role="presentation" border="0" cellpadding="0" cellspacing="0"><tr><td class="" style="vertical-align:top;width:600px;" ><![endif]--> + <div + class="mj-column-per-100 mj-outlook-group-fix" + style=" + font-size: 0px; + text-align: left; + direction: ltr; + display: inline-block; + vertical-align: top; + width: 100%; + " + > + <table + border="0" + cellpadding="0" + cellspacing="0" + role="presentation" + style="vertical-align: top" + width="100%" + > + <tbody> + <tr> + <td + align="center" + style=" + font-size: 0px; + padding: 10px 25px; + padding-top: 0px; + padding-right: 0px; + padding-bottom: 40px; + padding-left: 0px; + word-break: break-word; + " + > + <p + style=" + border-top: solid 8px #005a8e; + font-size: 1px; + margin: 0px auto; + width: 100%; + " + ></p> + <!--[if mso | IE + ]><table + align="center" + border="0" + cellpadding="0" + cellspacing="0" + style=" + border-top: solid 8px #005a8e; + font-size: 1px; + margin: 0px auto; + width: 600px; + " + role="presentation" + width="600px" + > + <tr> + <td style="height: 0; line-height: 0"> + + </td> + </tr> + </table><! + [endif]--> + </td> + </tr> + <tr> + <td + align="center" + style=" + font-size: 0px; + padding: 10px 25px; + padding-top: 0px; + padding-bottom: 0px; + word-break: break-word; + " + > + <table + border="0" + cellpadding="0" + cellspacing="0" + role="presentation" + style=" + border-collapse: collapse; + border-spacing: 0px; + " + > + <tbody> + <tr> + <td style="width: 121px"> + <img + alt="" + height="auto" + src="https://data.vespa.oath.cloud/assets/vespa-cloud-logo.png" + style=" + border: none; + display: block; + outline: none; + text-decoration: none; + height: auto; + width: 100%; + font-size: 13px; + " + width="121" + /> + </td> + </tr> + </tbody> + </table> + </td> + </tr> + </tbody> + </table> + </div> + <!--[if mso | IE]></td></tr></table><![endif]--> + </td> + </tr> + </tbody> + </table> + </div> + <!--[if mso | IE]></td></tr></table><table align="center" border="0" cellpadding="0" cellspacing="0" class="" role="presentation" style="width:600px;" width="600" bgcolor="#ffffff" ><tr><td style="line-height:0px;font-size:0px;mso-line-height-rule:exactly;"><![endif]--> + <div + style=" + background: #ffffff; + background-color: #ffffff; + margin: 0px auto; + max-width: 600px; + " + > + <table + align="center" + border="0" + cellpadding="0" + cellspacing="0" + role="presentation" + style="background: #ffffff; background-color: #ffffff; width: 100%" + > + <tbody> + <tr> + <td + style=" + direction: ltr; + font-size: 0px; + padding: 20px 0px 20px 0px; + padding-bottom: 70px; + padding-top: 30px; + text-align: center; + " + > + <!--[if mso | IE]><table role="presentation" border="0" cellpadding="0" cellspacing="0"><tr><td class="" style="vertical-align:top;width:600px;" ><![endif]--> + <div + class="mj-column-per-100 mj-outlook-group-fix" + style=" + font-size: 0px; + text-align: left; + direction: ltr; + display: inline-block; + vertical-align: top; + width: 100%; + " + > + <table + border="0" + cellpadding="0" + cellspacing="0" + role="presentation" + style="vertical-align: top" + width="100%" + > + + #parse($mailBodyTemplate) + + </table> + </div> + <!--[if mso | IE]></td></tr></table><![endif]--> + </td> + </tr> + </tbody> + </table> + </div> + <!--[if mso | IE]></td></tr></table><table align="center" border="0" cellpadding="0" cellspacing="0" class="" role="presentation" style="width:600px;" width="600" ><tr><td style="line-height:0px;font-size:0px;mso-line-height-rule:exactly;"><![endif]--> + <div style="margin: 0px auto; max-width: 600px"> + <table + align="center" + border="0" + cellpadding="0" + cellspacing="0" + role="presentation" + style="width: 100%" + > + <tbody> + <tr> + <td + style=" + direction: ltr; + font-size: 0px; + padding: 20px 0px 20px 0px; + padding-bottom: 0px; + padding-top: 20px; + text-align: center; + " + > + <!--[if mso | IE]><table role="presentation" border="0" cellpadding="0" cellspacing="0"><tr><td class="" style="vertical-align:top;width:600px;" ><![endif]--> + <div + class="mj-column-per-100 mj-outlook-group-fix" + style=" + font-size: 0px; + text-align: left; + direction: ltr; + display: inline-block; + vertical-align: top; + width: 100%; + " + > + <table + border="0" + cellpadding="0" + cellspacing="0" + role="presentation" + style="vertical-align: top" + width="100%" + > + <tbody> + <tr> + <td + align="center" + style=" + font-size: 0px; + padding: 0px 20px 0px 20px; + padding-top: 0px; + padding-bottom: 0px; + word-break: break-word; + " + > + <div + style=" + font-family: Open Sans, Helvetica, Arial, + sans-serif; + font-size: 11px; + line-height: 22px; + text-align: center; + color: #797e82; + " + > + <p style="margin: 10px 0"> + <a + target="_blank" + rel="noopener noreferrer" + style="color: #005a8e" + href="$privacyPolicyLink" + ><span style="color: #005a8e" + >Yahoo Privacy Policy</span + ></a + ><span style="color: #797e82" + > | </span + ><a + target="_blank" + rel="noopener noreferrer" + style="color: #005a8e" + href="$termsOfServiceLink" + ><span style="color: #005a8e" + >Terms of Service</span + ></a + ><span style="color: #797e82" + > | </span + ><a + target="_blank" + rel="noopener noreferrer" + style="color: #005a8e" + href="$supportLink" + ><span style="color: #005a8e">Support</span></a + > + </p> + <p style="margin: 10px 0"> + <a + target="_blank" + rel="noopener noreferrer" + style="color: inherit; text-decoration: none" + href="$accountNotificationLink" + >Click + <span style="color: #005a8e"><u>here</u></span> + to manage your notifications setting.</a + ><br /> + </p> + </div> + </td> + </tr> + </tbody> + </table> + </div> + <!--[if mso | IE]></td></tr></table><![endif]--> + </td> + </tr> + </tbody> + </table> + </div> + <!--[if mso | IE]></td></tr></table><![endif]--> + </div> + </body> +</html> diff --git a/controller-server/src/main/resources/mail/notification-message.vm b/controller-server/src/main/resources/mail/notification-message.vm new file mode 100644 index 00000000000..29673d38420 --- /dev/null +++ b/controller-server/src/main/resources/mail/notification-message.vm @@ -0,0 +1,6 @@ +<p> + $esc.html($notificationHeader): +</p> +#foreach( $i in $notificationItems ) +<p>$i</p> +#end diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java index eb86f23fbfb..345c880eaea 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java @@ -1345,6 +1345,7 @@ public class ControllerTest { Type.testPackage, Level.warning, NotificationSource.from(app.application().id()), + "There are problems with tests for [application](https://console.tld/tenant/tenant/application/application/prod/instance)", List.of("test package has staging tests, so it should also include staging setup", "see https://docs.vespa.ai/en/testing.html for details on how to write system tests for Vespa"))), tester.controller().notificationsDb().listNotifications(NotificationSource.from(app.application().id()), true)); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/MailVerifierTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/MailVerifierTest.java index e641b332a72..4fbf39f8d8b 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/MailVerifierTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/MailVerifierTest.java @@ -9,15 +9,15 @@ import com.yahoo.vespa.hosted.controller.tenant.CloudTenant; import com.yahoo.vespa.hosted.controller.tenant.Email; import com.yahoo.vespa.hosted.controller.tenant.PendingMailVerification; import com.yahoo.vespa.hosted.controller.tenant.Tenant; +import com.yahoo.vespa.hosted.controller.tenant.TenantBilling; +import com.yahoo.vespa.hosted.controller.tenant.TenantContact; import com.yahoo.vespa.hosted.controller.tenant.TenantContacts; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import java.net.URI; import java.time.Duration; import java.util.List; -import static com.yahoo.yolean.Exceptions.uncheck; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -30,7 +30,7 @@ class MailVerifierTest { private final ControllerTester tester = new ControllerTester(SystemName.Public); private final MockMailer mailer = tester.serviceRegistry().mailer(); - private final MailVerifier mailVerifier = new MailVerifier(URI.create("https://dashboard.uri.example.com"), tester.controller().tenants(), mailer, tester.curator(), tester.clock()); + private final MailVerifier mailVerifier = new MailVerifier(tester.serviceRegistry().consoleUrls(), tester.controller().tenants(), mailer, tester.curator(), tester.clock()); private static final TenantName tenantName = TenantName.from("scoober"); private static final String mail = "unverified@bar.com"; @@ -100,4 +100,29 @@ class MailVerifierTest { assertTrue(tester.curator().getPendingMailVerification(resentVerification.get().getVerificationCode()).isPresent()); } + @Test + public void test_billing_mail_verification() { + var billingMail = "billing@foo.bar"; + tester.controller().tenants().lockOrThrow(tenantName, LockedTenant.Cloud.class, lockedTenant -> { + var tenantBilling = TenantBilling.empty().withContact(TenantContact.empty().withEmail(new Email(billingMail, false))); + lockedTenant = lockedTenant.withInfo(lockedTenant.get().info().withBilling(tenantBilling)); + tester.controller().tenants().store(lockedTenant); + }); + mailVerifier.sendMailVerification(tenantName, billingMail, PendingMailVerification.MailType.BILLING); + + // Assert written verification data + var writtenMailVerification = tester.curator().listPendingMailVerifications().get(0); + assertEquals(PendingMailVerification.MailType.BILLING, writtenMailVerification.getMailType()); + assertEquals(tenantName, writtenMailVerification.getTenantName()); + assertEquals(tester.clock().instant().plus(Duration.ofDays(7)), writtenMailVerification.getVerificationDeadline()); + assertEquals(billingMail, writtenMailVerification.getMailAddress()); + + // Assert mail is verified + mailVerifier.verifyMail(writtenMailVerification.getVerificationCode()); + assertTrue(tester.curator().listPendingMailVerifications().isEmpty()); + var tenant = tester.controller().tenants().require(tenantName, CloudTenant.class); + var expectedBillingContact = TenantContact.empty().withEmail(new Email(billingMail, true)); + assertEquals(expectedBillingContact, tenant.info().billingContact().contact()); + } + } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificatesTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificatesTest.java index 7faaee95abb..378b92d37ce 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificatesTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificatesTest.java @@ -303,8 +303,7 @@ public class EndpointCertificatesTest { @Test public void assign_certificate_from_pool() { - tester.flagSource().withBooleanFlag(Flags.RANDOMIZED_ENDPOINT_NAMES.id(), true); - tester.flagSource().withBooleanFlag(Flags.LEGACY_ENDPOINTS.id(), false); + setEndpointConfig(tester, EndpointConfig.generated); try { addCertificateToPool("bad0f00d", UnassignedCertificate.State.requested, tester); endpointCertificates.get(new DeploymentId(instance, prodZone), DeploymentSpec.empty, lock); @@ -340,7 +339,7 @@ public class EndpointCertificatesTest { @Test public void certificate_migration() { - // An application is initially deployed with legacy config + // An application is initially deployed with legacy config (the default) ZoneId zone1 = ZoneId.from(Environment.prod, RegionName.from("aws-us-east-1c")); ApplicationPackage applicationPackage = new ApplicationPackageBuilder().region(zone1.region()) .build(); @@ -408,8 +407,7 @@ public class EndpointCertificatesTest { devCert0.requestedDnsSans()); // Application switches to combined config - tester.flagSource().withBooleanFlag(Flags.RANDOMIZED_ENDPOINT_NAMES.id(), true); - tester.flagSource().withBooleanFlag(Flags.LEGACY_ENDPOINTS.id(), true); + setEndpointConfig(tester, EndpointConfig.combined); tester.clock().advance(Duration.ofHours(1)); assertEquals(certificate.withLastRequested(tester.clock().instant().getEpochSecond()), endpointCertificates.get(deployment0, applicationPackage.deploymentSpec(), lock), @@ -420,7 +418,7 @@ public class EndpointCertificatesTest { "Certificate is not assigned at application level"); // Application switches to generated config - tester.flagSource().withBooleanFlag(Flags.LEGACY_ENDPOINTS.id(), false); + setEndpointConfig(tester, EndpointConfig.generated); tester.clock().advance(Duration.ofHours(1)); assertEquals(certificate.withLastRequested(tester.clock().instant().getEpochSecond()), endpointCertificates.get(deployment0, applicationPackage.deploymentSpec(), lock), @@ -451,8 +449,7 @@ public class EndpointCertificatesTest { assertEquals(poolCertId1, prodCertificate.generatedId().get()); // Application switches back to legacy config - tester.flagSource().withBooleanFlag(Flags.RANDOMIZED_ENDPOINT_NAMES.id(), false); - tester.flagSource().withBooleanFlag(Flags.LEGACY_ENDPOINTS.id(), true); + setEndpointConfig(tester, EndpointConfig.legacy); EndpointCertificate reissuedCertificate = endpointCertificates.get(deployment0, applicationPackage.deploymentSpec(), lock); assertEquals(certificate.requestedDnsSans(), reissuedCertificate.requestedDnsSans()); assertTrue(tester.curator().readAssignedCertificate(deployment0.applicationId()).isPresent(), "Certificate is assigned at instance level again"); @@ -460,6 +457,10 @@ public class EndpointCertificatesTest { "Certificate is still assigned at application level"); // Not removed because the assumption is that the application will eventually migrate back } + private void setEndpointConfig(ControllerTester tester, EndpointConfig config) { + tester.flagSource().withStringFlag(Flags.ENDPOINT_CONFIG.id(), config.name()); + } + private void addCertificateToPool(String id, UnassignedCertificate.State state, ControllerTester tester) { EndpointCertificate cert = new EndpointCertificate(testKeyName, testCertName, diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java index c16234b3948..bc03d46a30a 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java @@ -75,7 +75,7 @@ public class DeploymentTester { tester = controllerTester; jobs = tester.controller().jobController(); cloud = (MockTesterCloud) tester.controller().jobController().cloud(); - runner = new JobRunner(tester.controller(), maintenanceInterval, JobRunnerTest.inThreadExecutor(), new InternalStepRunner(tester.controller())); + runner = new JobRunner(tester.controller(), maintenanceInterval, JobRunnerTest.inThreadInOrderExecutor(), new InternalStepRunner(tester.controller())); upgrader = new Upgrader(tester.controller(), maintenanceInterval); upgrader.setUpgradesPerMinute(1); // Anything that makes it at least one for any maintenance period is fine. readyJobsTrigger = new ReadyJobsTrigger(tester.controller(), maintenanceInterval); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java index 259e877afd9..5995b3eaac6 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java @@ -417,12 +417,11 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer applications.put(id, new Application(id.applicationId(), lastPrepareVersion, appPackage)); ClusterSpec.Id cluster = ClusterSpec.Id.from("default"); - deployment.endpoints(); // Supplier with side effects >_< if (nodeRepository().list(id.zoneId(), NodeFilter.all().applications(id.applicationId())).isEmpty()) provision(id.zoneId(), id.applicationId(), cluster); - this.containerEndpoints.put(id, deployment.endpoints().get().endpoints()); + this.containerEndpoints.put(id, deployment.endpoints().endpoints()); deployment.cloudAccount().ifPresent(account -> this.cloudAccounts.put(id, account)); if (!deferLoadBalancerProvisioning.contains(id.zoneId().environment())) { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ServiceRegistryMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ServiceRegistryMock.java index e8e3de697dc..39d867d813d 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ServiceRegistryMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ServiceRegistryMock.java @@ -10,7 +10,7 @@ import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.SystemName; import com.yahoo.test.ManualClock; import com.yahoo.vespa.hosted.controller.api.identifiers.ControllerVersion; -import com.yahoo.vespa.hosted.controller.api.integration.MockPricingController; +import com.yahoo.vespa.hosted.controller.api.integration.ConsoleUrls; import com.yahoo.vespa.hosted.controller.api.integration.ServiceRegistry; import com.yahoo.vespa.hosted.controller.api.integration.archive.ArchiveService; import com.yahoo.vespa.hosted.controller.api.integration.archive.MockArchiveService; @@ -37,7 +37,6 @@ import com.yahoo.vespa.hosted.controller.api.integration.horizon.HorizonClient; import com.yahoo.vespa.hosted.controller.api.integration.horizon.MockHorizonClient; import com.yahoo.vespa.hosted.controller.api.integration.organization.MockContactRetriever; import com.yahoo.vespa.hosted.controller.api.integration.organization.MockIssueHandler; -import com.yahoo.vespa.hosted.controller.api.integration.pricing.PricingController; import com.yahoo.vespa.hosted.controller.api.integration.resource.CostReportConsumerMock; import com.yahoo.vespa.hosted.controller.api.integration.resource.ResourceDatabaseClient; import com.yahoo.vespa.hosted.controller.api.integration.resource.ResourceDatabaseClientMock; @@ -56,6 +55,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.user.RoleMaintainer; import com.yahoo.vespa.hosted.controller.api.integration.user.RoleMaintainerMock; import com.yahoo.vespa.hosted.controller.api.integration.vcmr.MockChangeRequestClient; +import java.net.URI; import java.time.Instant; import java.util.Optional; @@ -70,6 +70,7 @@ public class ServiceRegistryMock extends AbstractComponent implements ServiceReg private final ControllerVersion controllerVersion; private final ZoneRegistryMock zoneRegistryMock; private final ConfigServerMock configServerMock; + private final ConsoleUrls consoleUrls = new ConsoleUrls(URI.create("https://console.tld")); private final MemoryNameService memoryNameService = new MemoryNameService(); private final MockVpcEndpointService vpcEndpointService = new MockVpcEndpointService(clock, memoryNameService); private final MockMailer mockMailer = new MockMailer(); @@ -89,7 +90,6 @@ public class ServiceRegistryMock extends AbstractComponent implements ServiceReg private final MockEnclaveAccessService mockAMIService = new MockEnclaveAccessService(); private final MockResourceTagger mockResourceTagger = new MockResourceTagger(); private final MockRoleService roleService = new MockRoleService(); - private final MockBillingController billingController = new MockBillingController(clock); private final ArtifactRegistryMock containerRegistry = new ArtifactRegistryMock(); private final NoopTenantSecretService tenantSecretService = new NoopTenantSecretService(); private final NoopEndpointSecretManager secretManager = new NoopEndpointSecretManager(); @@ -100,8 +100,8 @@ public class ServiceRegistryMock extends AbstractComponent implements ServiceReg private final PlanRegistry planRegistry = new PlanRegistryMock(); private final ResourceDatabaseClient resourceDb = new ResourceDatabaseClientMock(planRegistry); private final BillingDatabaseClient billingDb = new BillingDatabaseClientMock(clock, planRegistry); + private final MockBillingController billingController = new MockBillingController(clock, billingDb); private final RoleMaintainerMock roleMaintainer = new RoleMaintainerMock(); - private final MockPricingController pricingController = new MockPricingController(); public ServiceRegistryMock(SystemName system) { this.zoneRegistryMock = new ZoneRegistryMock(system); @@ -221,6 +221,11 @@ public class ServiceRegistryMock extends AbstractComponent implements ServiceReg } @Override + public ConsoleUrls consoleUrls() { + return consoleUrls; + } + + @Override public MockResourceTagger resourceTagger() { return mockResourceTagger; } @@ -320,10 +325,7 @@ public class ServiceRegistryMock extends AbstractComponent implements ServiceReg @Override public BillingReporter billingReporter() { - return new BillingReporterMock(clock()); + return new BillingReporterMock(clock(), billingDb); } - @Override - public PricingController pricingController() { return pricingController; } - } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneRegistryMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneRegistryMock.java index 70198ae35fd..c5b11fe21b0 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneRegistryMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneRegistryMock.java @@ -2,8 +2,6 @@ package com.yahoo.vespa.hosted.controller.integration; import com.yahoo.component.AbstractComponent; -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.ApplicationName; import com.yahoo.config.provision.AthenzDomain; import com.yahoo.config.provision.CloudAccount; import com.yahoo.config.provision.CloudName; @@ -21,7 +19,6 @@ import com.yahoo.text.Text; import com.yahoo.vespa.athenz.api.AthenzIdentity; import com.yahoo.vespa.athenz.api.AthenzService; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneRegistry; import java.net.URI; @@ -220,36 +217,6 @@ public class ZoneRegistryMock extends AbstractComponent implements ZoneRegistry } @Override - public URI dashboardUrl() { - return URI.create("https://dashboard.tld"); - } - - @Override - public URI dashboardUrl(ApplicationId id) { - return URI.create("https://dashboard.tld/" + id); - } - - @Override - public URI dashboardUrl(TenantName tenantName, ApplicationName applicationName) { - return URI.create("https://dashboard.tld/" + tenantName + "/" + applicationName); - } - - @Override - public URI dashboardUrl(TenantName tenantName) { - return URI.create("https://dashboard.tld/" + tenantName); - } - - @Override - public URI dashboardUrl(RunId id) { - return URI.create("https://dashboard.tld/" + id.application() + "/" + id.type().jobName() + "/" + id.number()); - } - - @Override - public URI supportUrl() { - return URI.create("https://help.tld"); - } - - @Override public URI apiUrl() { return URI.create("https://api.tld:4443/"); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/BillingReportMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/BillingReportMaintainerTest.java index 9e1aa64beae..5cb46664a75 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/BillingReportMaintainerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/BillingReportMaintainerTest.java @@ -4,12 +4,16 @@ package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.TenantName; import com.yahoo.vespa.hosted.controller.ControllerTester; +import com.yahoo.vespa.hosted.controller.api.integration.billing.BillStatus; +import com.yahoo.vespa.hosted.controller.api.integration.billing.InvoiceUpdate; import com.yahoo.vespa.hosted.controller.api.integration.billing.PlanRegistryMock; import com.yahoo.vespa.hosted.controller.tenant.BillingReference; import com.yahoo.vespa.hosted.controller.tenant.CloudTenant; import org.junit.jupiter.api.Test; import java.time.Duration; +import java.time.LocalDate; +import java.time.ZoneOffset; import java.util.Optional; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -39,8 +43,47 @@ public class BillingReportMaintainerTest { assertNotNull(b1.orElseThrow().reference()); } + @Test + void only_open_bills_with_exported_id_are_maintained() { + var t1 = tester.createTenant("t1"); + var billingController = tester.controller().serviceRegistry().billingController(); + var billingDb = tester.controller().serviceRegistry().billingDatabase(); + + var start = LocalDate.of(2020, 5, 23).atStartOfDay(ZoneOffset.UTC); + var end = start.toLocalDate().plusDays(6).atStartOfDay(ZoneOffset.UTC); + + var bill1 = billingDb.createBill(t1, start, end, "non-exported"); + var bill2 = billingDb.createBill(t1, start, end, "exported"); + var bill3 = billingDb.createBill(t1, start, end, "exported-and-frozen"); + billingDb.setStatus(bill3, "foo", BillStatus.FROZEN); + + billingController.setPlan(t1, PlanRegistryMock.paidPlan.id(), false, true); + + tester.controller().serviceRegistry().billingReporter().exportBill(billingDb.readBill(bill2).get(), "FOO", cloudTenant(t1)); + tester.controller().serviceRegistry().billingReporter().exportBill(billingDb.readBill(bill3).get(), "FOO", cloudTenant(t1)); + var updates = maintainer.maintainInvoices(); + assertEquals(new InvoiceUpdate(1, 0, 0), updates); + + assertTrue(billingDb.readBill(bill1).get().getExportedId().isEmpty()); + + var exportedBill = billingDb.readBill(bill2).get(); + assertEquals("EXT-ID-123", exportedBill.getExportedId().get()); + var lineItems = exportedBill.lineItems(); + assertEquals(1, lineItems.size()); + assertEquals("maintained", lineItems.get(0).id()); + + var frozenBill = billingDb.readBill(bill3).get(); + assertEquals("EXT-ID-123", frozenBill.getExportedId().get()); + assertEquals(0, frozenBill.lineItems().size()); + + } + + private CloudTenant cloudTenant(TenantName tenantName) { + return tester.controller().tenants().require(tenantName, CloudTenant.class); + } + private Optional<BillingReference> billingReference(TenantName tenantName) { - var t = tester.controller().tenants().require(tenantName, CloudTenant.class); - return t.billingReference(); + return cloudTenant(tenantName).billingReference(); } + } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/CloudTrialExpirerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/CloudTrialExpirerTest.java index 95cffae8728..4056459c532 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/CloudTrialExpirerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/CloudTrialExpirerTest.java @@ -3,20 +3,30 @@ package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.TenantName; +import com.yahoo.test.ManualClock; +import com.yahoo.vespa.flags.Flags; import com.yahoo.vespa.flags.InMemoryFlagSource; import com.yahoo.vespa.flags.PermanentFlags; import com.yahoo.vespa.hosted.controller.ControllerTester; import com.yahoo.vespa.hosted.controller.api.integration.billing.PlanId; +import com.yahoo.vespa.hosted.controller.api.integration.stubs.MockMailer; import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; +import com.yahoo.vespa.hosted.controller.notification.Notification; +import com.yahoo.vespa.hosted.controller.notification.NotificationSource; import com.yahoo.vespa.hosted.controller.tenant.LastLoginInfo; import com.yahoo.vespa.hosted.controller.tenant.Tenant; import org.junit.jupiter.api.Test; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.nio.file.StandardOpenOption; import java.time.Duration; import java.util.List; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; /** @@ -24,6 +34,8 @@ import static org.junit.jupiter.api.Assertions.assertTrue; */ public class CloudTrialExpirerTest { + private static final boolean OVERWRITE_TEST_FILES = false; + private final ControllerTester tester = new ControllerTester(SystemName.PublicCd); private final DeploymentTester deploymentTester = new DeploymentTester(tester); private final CloudTrialExpirer expirer = new CloudTrialExpirer(tester.controller(), Duration.ofMinutes(5)); @@ -89,6 +101,58 @@ public class CloudTrialExpirerTest { assertPlan("active", "none"); } + @Test + void queues_trial_notification_based_on_account_age() throws IOException { + var clock = (ManualClock)tester.controller().clock(); + var mailer = (MockMailer) tester.serviceRegistry().mailer(); + var tenant = TenantName.from("trial-tenant"); + ((InMemoryFlagSource) tester.controller().flagSource()) + .withBooleanFlag(Flags.CLOUD_TRIAL_NOTIFICATIONS.id(), true); + registerTenant(tenant.value(), "trial", Duration.ZERO); + assertEquals(0.0, expirer.maintain()); + var expected = "Welcome to Vespa Cloud trial! [Manage plan](https://console.tld/tenant/trial-tenant/account/billing)"; + assertEquals(expected, lastAccountLevelNotificationTitle(tenant)); + assertLastEmailEquals(mailer, "welcome.html"); + + expected = "You're halfway through the **14 day** trial period. [Manage plan](https://console.tld/tenant/trial-tenant/account/billing)"; + clock.advance(Duration.ofDays(7)); + assertEquals(0.0, expirer.maintain()); + assertEquals(expected, lastAccountLevelNotificationTitle(tenant)); + assertLastEmailEquals(mailer, "trial-reminder.html"); + + expected = "Your Vespa Cloud trial expires in **2** days. [Manage plan](https://console.tld/tenant/trial-tenant/account/billing)"; + clock.advance(Duration.ofDays(5)); + assertEquals(0.0, expirer.maintain()); + assertEquals(expected, lastAccountLevelNotificationTitle(tenant)); + assertLastEmailEquals(mailer, "trial-expiring-soon.html"); + + expected = "Your Vespa Cloud trial expires **tomorrow**. [Manage plan](https://console.tld/tenant/trial-tenant/account/billing)"; + clock.advance(Duration.ofDays(1)); + assertEquals(0.0, expirer.maintain()); + assertEquals(expected, lastAccountLevelNotificationTitle(tenant)); + assertLastEmailEquals(mailer, "trial-expiring-immediately.html"); + + expected = "Your Vespa Cloud trial has expired. [Upgrade plan](https://console.tld/tenant/trial-tenant/account/billing)"; + clock.advance(Duration.ofDays(2)); + assertEquals(0.0, expirer.maintain()); + assertEquals(expected, lastAccountLevelNotificationTitle(tenant)); + assertLastEmailEquals(mailer, "trial-expired.html"); + } + + private void assertLastEmailEquals(MockMailer mailer, String expectedContentFile) throws IOException { + var mails = mailer.inbox("dev-trial-tenant"); + assertFalse(mails.isEmpty()); + var content = mails.get(mails.size() - 1).htmlMessage().orElseThrow(); + var path = Paths.get("src/test/resources/mail/" + expectedContentFile); + if (OVERWRITE_TEST_FILES) { + Files.write(path, content.getBytes(), + StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING, StandardOpenOption.WRITE); + } else { + var expectedContent = Files.readString(path); + assertEquals(expectedContent, content); + } + } + private void registerTenant(String tenantName, String plan, Duration timeSinceLastLogin) { var name = TenantName.from(tenantName); tester.createTenant(tenantName, Tenant.Type.cloud); @@ -111,4 +175,11 @@ public class CloudTrialExpirerTest { assertEquals(planId, tester.serviceRegistry().billingController().getPlan(TenantName.from(tenant)).value()); } + private String lastAccountLevelNotificationTitle(TenantName tenant) { + return tester.controller().notificationsDb() + .listNotifications(NotificationSource.from(tenant), false).stream() + .filter(n -> n.type() == Notification.Type.account).map(Notification::title) + .findFirst().orElseThrow(); + } + } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainerTest.java index f551a99829e..fe9e9b28655 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainerTest.java @@ -26,6 +26,7 @@ import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; import com.yahoo.vespa.hosted.controller.deployment.DeploymentContext; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; import com.yahoo.vespa.hosted.controller.integration.SecretStoreMock; +import com.yahoo.vespa.hosted.controller.routing.EndpointConfig; import org.assertj.core.api.Assertions; import org.junit.jupiter.api.Test; @@ -262,9 +263,8 @@ public class EndpointCertificateMaintainerTest { } private void prepareCertificatePool(int numCertificates) { - ((InMemoryFlagSource)tester.controller().flagSource()).withIntFlag(PermanentFlags.CERT_POOL_SIZE.id(), numCertificates); - ((InMemoryFlagSource)tester.controller().flagSource()).withBooleanFlag(Flags.RANDOMIZED_ENDPOINT_NAMES.id(), true); - ((InMemoryFlagSource)tester.controller().flagSource()).withBooleanFlag(Flags.LEGACY_ENDPOINTS.id(), false); + ((InMemoryFlagSource) tester.controller().flagSource()).withIntFlag(PermanentFlags.CERT_POOL_SIZE.id(), numCertificates); + ((InMemoryFlagSource) tester.controller().flagSource()).withStringFlag(Flags.ENDPOINT_CONFIG.id(), EndpointConfig.generated.name()); // Provision certificates for (int i = 0; i < numCertificates; i++) { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java index 20717be598f..d96de8df6fd 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java @@ -34,18 +34,24 @@ import java.util.EnumMap; import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.Queue; import java.util.Set; import java.util.concurrent.AbstractExecutorService; import java.util.concurrent.BrokenBarrierException; +import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.CyclicBarrier; +import java.util.concurrent.Executor; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.Phaser; +import java.util.concurrent.RejectedExecutionException; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Consumer; import java.util.function.Supplier; import java.util.stream.Collectors; @@ -70,10 +76,12 @@ import static com.yahoo.vespa.hosted.controller.deployment.Step.installReal; import static com.yahoo.vespa.hosted.controller.deployment.Step.installTester; import static com.yahoo.vespa.hosted.controller.deployment.Step.report; import static com.yahoo.vespa.hosted.controller.deployment.Step.startTests; +import static java.util.Objects.requireNonNull; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotSame; import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assertions.fail; @@ -175,7 +183,7 @@ public class JobRunnerTest { DeploymentTester tester = new DeploymentTester(); JobController jobs = tester.controller().jobController(); Map<Step, RunStatus> outcomes = new EnumMap<>(Step.class); - JobRunner runner = new JobRunner(tester.controller(), Duration.ofDays(1), inThreadExecutor(), mappedRunner(outcomes)); + JobRunner runner = new JobRunner(tester.controller(), Duration.ofDays(1), inThreadInOrderExecutor(), mappedRunner(outcomes)); TenantAndApplicationId appId = tester.createApplication("tenant", "real", "default").id(); ApplicationId id = appId.defaultInstance(); @@ -322,7 +330,7 @@ public class JobRunnerTest { void historyPruning() { DeploymentTester tester = new DeploymentTester(); JobController jobs = tester.controller().jobController(); - JobRunner runner = new JobRunner(tester.controller(), Duration.ofDays(1), inThreadExecutor(), (id, step) -> Optional.of(running)); + JobRunner runner = new JobRunner(tester.controller(), Duration.ofDays(1), inThreadInOrderExecutor(), (id, step) -> Optional.of(running)); TenantAndApplicationId appId = tester.createApplication("tenant", "real", "default").id(); ApplicationId instanceId = appId.defaultInstance(); @@ -347,7 +355,7 @@ public class JobRunnerTest { assertFalse(jobs.details(new RunId(instanceId, systemTest, 1)).isPresent()); assertTrue(jobs.details(new RunId(instanceId, systemTest, 65)).isPresent()); - JobRunner failureRunner = new JobRunner(tester.controller(), Duration.ofDays(1), inThreadExecutor(), (id, step) -> Optional.of(error)); + JobRunner failureRunner = new JobRunner(tester.controller(), Duration.ofDays(1), inThreadInOrderExecutor(), (id, step) -> Optional.of(error)); // Make all but the oldest of the 54 jobs a failure. for (int i = 0; i < jobs.historyLength() - 1; i++) { @@ -419,7 +427,7 @@ public class JobRunnerTest { DeploymentTester tester = new DeploymentTester(); JobController jobs = tester.controller().jobController(); Map<Step, RunStatus> outcomes = new EnumMap<>(Step.class); - JobRunner runner = new JobRunner(tester.controller(), Duration.ofDays(1), inThreadExecutor(), mappedRunner(outcomes)); + JobRunner runner = new JobRunner(tester.controller(), Duration.ofDays(1), inThreadInOrderExecutor(), mappedRunner(outcomes)); TenantAndApplicationId appId = tester.createApplication("tenant", "real", "default").id(); ApplicationId id = appId.defaultInstance(); @@ -437,7 +445,7 @@ public class JobRunnerTest { DeploymentTester tester = new DeploymentTester(); JobController jobs = tester.controller().jobController(); Map<Step, RunStatus> outcomes = new EnumMap<>(Step.class); - JobRunner runner = new JobRunner(tester.controller(), Duration.ofDays(1), inThreadExecutor(), mappedRunner(outcomes)); + JobRunner runner = new JobRunner(tester.controller(), Duration.ofDays(1), inThreadInOrderExecutor(), mappedRunner(outcomes)); TenantAndApplicationId appId = tester.createApplication("tenant", "real", "default").id(); ApplicationId id = appId.defaultInstance(); @@ -473,19 +481,58 @@ public class JobRunnerTest { assertEquals(1, metric.getMetric(context::equals, JobMetrics.noTests).get().intValue()); } + @Test + void testInThreadExecutor() throws InterruptedException { + ExecutorService executor = inThreadInOrderExecutor(); + AtomicInteger c = new AtomicInteger(0), d = new AtomicInteger(0); + Consumer<AtomicInteger> task = i -> executor.execute(() -> { + executor.execute(() -> { + i.set(2); + executor.execute(() -> i.set(4)); + }); + executor.execute(() -> i.set(3)); + i.set(1); + }); + Thread s = new Thread(() -> task.accept(d)); + s.start(); + task.accept(c); + s.join(); + assertEquals(4, c.get()); + assertEquals(4, d.get()); + assertEquals("executor is shut down", + assertThrows(RejectedExecutionException.class, + () -> executor.execute(() -> { + executor.execute(() -> executor.execute(() -> { c.set(6); })); + executor.shutdown(); + c.set(5); + })).getMessage()); + assertEquals(5, c.get()); + } + private void start(JobController jobs, ApplicationId id, JobType type) { jobs.start(id, type, versions, false, Reason.empty()); } - public static ExecutorService inThreadExecutor() { + /** Dummy test executor for unit tests. Runs tasks BFS rather than DFS, like a simple {@code Runnable::run} would do. No real shutdown logic. */ + public static ExecutorService inThreadInOrderExecutor() { return new AbstractExecutorService() { - final AtomicBoolean shutDown = new AtomicBoolean(false); + private final ThreadLocal<Boolean> inExecute = ThreadLocal.withInitial(() -> false); + private final ThreadLocal<Queue<Runnable>> tasks = ThreadLocal.withInitial(ConcurrentLinkedQueue::new); + private final AtomicBoolean shutDown = new AtomicBoolean(false); + @Override + public void execute(Runnable command) { + if (isShutdown()) throw new RejectedExecutionException("executor is shut down"); + tasks.get().add(requireNonNull(command)); + if (inExecute.get()) return; + inExecute.set(true); + try { Runnable task; while (null != (task = tasks.get().poll())) task.run(); } + finally { inExecute.set(false); } + } @Override public void shutdown() { shutDown.set(true); } @Override public List<Runnable> shutdownNow() { shutDown.set(true); return Collections.emptyList(); } @Override public boolean isShutdown() { return shutDown.get(); } @Override public boolean isTerminated() { return shutDown.get(); } @Override public boolean awaitTermination(long timeout, TimeUnit unit) { return true; } - @Override public void execute(Runnable command) { command.run(); } }; } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/notification/NotificationFormatterTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/notification/NotificationFormatterTest.java index 751d0123e40..875487144d9 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/notification/NotificationFormatterTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/notification/NotificationFormatterTest.java @@ -6,16 +6,16 @@ import com.yahoo.config.provision.ApplicationName; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.InstanceName; import com.yahoo.config.provision.RegionName; -import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; +import com.yahoo.vespa.hosted.controller.api.integration.ConsoleUrls; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; -import com.yahoo.vespa.hosted.controller.integration.ZoneRegistryMock; import org.junit.jupiter.api.Test; +import java.net.URI; import java.time.Instant; import java.util.List; @@ -31,9 +31,8 @@ public class NotificationFormatterTest { private final ApplicationId applicationId = ApplicationId.from(tenant, application, instance); private final DeploymentId deploymentId = new DeploymentId(applicationId, ZoneId.defaultId()); private final ClusterSpec.Id cluster = new ClusterSpec.Id("content"); - private final ZoneRegistryMock zoneRegistry = new ZoneRegistryMock(SystemName.Public); - private final NotificationFormatter formatter = new NotificationFormatter(zoneRegistry); + private final NotificationFormatter formatter = new NotificationFormatter(new ConsoleUrls(URI.create("https://console.tld"))); @Test void applicationPackage() { @@ -41,7 +40,7 @@ public class NotificationFormatterTest { var content = formatter.format(notification); assertEquals("Application package", content.prettyType()); assertEquals("Application package for myapp.beta has 2 warnings", content.messagePrefix()); - assertEquals("https://dashboard.tld/scoober.myapp.beta", content.uri().toString()); + assertEquals("https://console.tld/tenant/scoober/application/myapp/prod/instance/beta", content.uri()); } @Test @@ -51,7 +50,7 @@ public class NotificationFormatterTest { var content = formatter.format(notification); assertEquals("Deployment", content.prettyType()); assertEquals("production-default #1001 for myapp.beta has a warning", content.messagePrefix()); - assertEquals("https://dashboard.tld/scoober.myapp.beta/production-default/1001", content.uri().toString()); + assertEquals("https://console.tld/tenant/scoober/application/myapp/prod/instance/beta/job/production-default/run/1001", content.uri()); } @Test @@ -61,7 +60,7 @@ public class NotificationFormatterTest { var content = formatter.format(notification); assertEquals("Deployment", content.prettyType()); assertEquals("production-default #1001 for myapp.beta has failed", content.messagePrefix()); - assertEquals("https://dashboard.tld/scoober.myapp.beta/production-default/1001", content.uri().toString()); + assertEquals("https://console.tld/tenant/scoober/application/myapp/prod/instance/beta/job/production-default/run/1001", content.uri()); } @Test @@ -70,7 +69,7 @@ public class NotificationFormatterTest { var content = formatter.format(notification); assertEquals("Test package", content.prettyType()); assertEquals("There is a problem with tests for myapp", content.messagePrefix()); - assertEquals("https://dashboard.tld/scoober/myapp", content.uri().toString()); + assertEquals("https://console.tld/tenant/scoober/application/myapp/prod/instance", content.uri()); } @Test @@ -79,7 +78,7 @@ public class NotificationFormatterTest { var content = formatter.format(notification); assertEquals("Reindex", content.prettyType()); assertEquals("Cluster content in prod.default for myapp.beta is reindexing", content.messagePrefix()); - assertEquals("https://dashboard.tld/scoober.myapp.beta?beta.prod.default=clusters%2Ccontent%3Dstatus", content.uri().toString()); + assertEquals("https://console.tld/tenant/scoober/application/myapp/prod/instance/beta?beta.prod.default=clusters%2Ccontent%3Dreindexing", content.uri()); } @Test @@ -88,6 +87,6 @@ public class NotificationFormatterTest { var content = formatter.format(notification); assertEquals("Nearly feed blocked", content.prettyType()); assertEquals("Cluster content in prod.default for myapp.beta is nearly feed blocked", content.messagePrefix()); - assertEquals("https://dashboard.tld/scoober.myapp.beta?beta.prod.default=clusters%2Ccontent", content.uri().toString()); + assertEquals("https://console.tld/tenant/scoober/application/myapp/prod/instance/beta?beta.prod.default=clusters%2Ccontent", content.uri()); } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/notification/NotificationsDbTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/notification/NotificationsDbTest.java index 003a7f59eef..e41be11c846 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/notification/NotificationsDbTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/notification/NotificationsDbTest.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.hosted.controller.notification; import com.google.common.collect.ImmutableBiMap; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.CloudName; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.TenantName; @@ -14,8 +15,10 @@ import com.yahoo.vespa.flags.InMemoryFlagSource; import com.yahoo.vespa.flags.PermanentFlags; import com.yahoo.vespa.hosted.controller.api.application.v4.model.ClusterMetrics; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; +import com.yahoo.vespa.hosted.controller.api.integration.ConsoleUrls; import com.yahoo.vespa.hosted.controller.api.integration.billing.PlanId; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ApplicationReindexing; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; import com.yahoo.vespa.hosted.controller.api.integration.stubs.MockMailer; import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; @@ -31,6 +34,7 @@ import com.yahoo.vespa.hosted.controller.tenant.TenantInfo; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import java.net.URI; import java.time.Instant; import java.util.ArrayList; import java.util.Arrays; @@ -84,7 +88,8 @@ public class NotificationsDbTest { private final MockCuratorDb curatorDb = new MockCuratorDb(SystemName.Public); private final MockMailer mailer = new MockMailer(); private final FlagSource flagSource = new InMemoryFlagSource().withBooleanFlag(PermanentFlags.NOTIFICATION_DISPATCH_FLAG.id(), true); - private final NotificationsDb notificationsDb = new NotificationsDb(clock, curatorDb, new Notifier(curatorDb, new ZoneRegistryMock(SystemName.cd), mailer, flagSource)); + private final ConsoleUrls consoleUrls = new ConsoleUrls(URI.create("https://console.tld")); + private final NotificationsDb notificationsDb = new NotificationsDb(clock, curatorDb, new Notifier(curatorDb, consoleUrls, mailer, flagSource), consoleUrls); @Test void list_test() { @@ -102,10 +107,10 @@ public class NotificationsDbTest { Notification notification2 = notification(12345, Type.deployment, Level.error, NotificationSource.from(ApplicationId.from(tenant.value(), "app3", "instance2")), "instance msg #3"); // Replace the 3rd notification - notificationsDb.setNotification(notification1.source(), notification1.type(), notification1.level(), notification1.messages()); + setNotification(notification1); // Notification for a new app, add without replacement - notificationsDb.setNotification(notification2.source(), notification2.type(), notification2.level(), notification2.messages()); + setNotification(notification2); List<Notification> expected = notificationIndices(0, 1, 3, 4, 5); expected.addAll(List.of(notification1, notification2)); @@ -119,19 +124,19 @@ public class NotificationsDbTest { Notification notification3 = notification(12345, Type.reindex, Level.warning, NotificationSource.from(new DeploymentId(ApplicationId.from(tenant.value(), "app2", "instance2"), ZoneId.defaultId()), new ClusterSpec.Id("content")), "instance msg #2"); ; var a = notifications.get(0); - notificationsDb.setNotification(a.source(), a.type(), a.level(), a.messages()); + setNotification(a); assertEquals(0, mailer.inbox(email.getEmailAddress()).size()); // Replace the 3rd notification. but don't change source or type - notificationsDb.setNotification(notification1.source(), notification1.type(), notification1.level(), notification1.messages()); + setNotification(notification1); assertEquals(0, mailer.inbox(email.getEmailAddress()).size()); // Notification for a new app, add without replacement - notificationsDb.setNotification(notification2.source(), notification2.type(), notification2.level(), notification2.messages()); + setNotification(notification2); assertEquals(1, mailer.inbox(email.getEmailAddress()).size()); // Notification for new type on existing app - notificationsDb.setNotification(notification3.source(), notification3.type(), notification3.level(), notification3.messages()); + setNotification(notification3); assertEquals(2, mailer.inbox(email.getEmailAddress()).size()); } @@ -199,17 +204,19 @@ public class NotificationsDbTest { // One resource is at warning notificationsDb.setDeploymentMetricsNotifications(deploymentId, List.of(clusterMetrics("cluster1", 0.88, 0.9, 0.3, 0.5)), emptyReindexing); - expected.add(notification(12345, Type.feedBlock, Level.warning, sourceCluster1, "disk (usage: 88.0%, feed block limit: 90.0%)")); + expected.add(notification(12345, Type.feedBlock, Level.warning, "Cluster [cluster1](https://console.tld/tenant/tenant1/application/app1/prod/instance/instance1?instance1.prod.us-south-3=clusters%2Ccluster1) in **prod.us-south-3** for **app1.instance1** is nearly feed blocked", + sourceCluster1, "disk (usage: 88.0%, feed block limit: 90.0%)")); assertEquals(expected, curatorDb.readNotifications(tenant)); // Both resources over the limit notificationsDb.setDeploymentMetricsNotifications(deploymentId, List.of(clusterMetrics("cluster1", 0.95, 0.9, 0.3, 0.5)), emptyReindexing); - expected.set(6, notification(12345, Type.feedBlock, Level.error, sourceCluster1, "disk (usage: 95.0%, feed block limit: 90.0%)")); + expected.set(6, notification(12345, Type.feedBlock, Level.error, "Cluster [cluster1](https://console.tld/tenant/tenant1/application/app1/prod/instance/instance1?instance1.prod.us-south-3=clusters%2Ccluster1) in **prod.us-south-3** for **app1.instance1** is feed blocked", + sourceCluster1, "disk (usage: 95.0%, feed block limit: 90.0%)")); assertEquals(expected, curatorDb.readNotifications(tenant)); // One resource at warning, one at error: Only show error message notificationsDb.setDeploymentMetricsNotifications(deploymentId, List.of(clusterMetrics("cluster1", 0.95, 0.9, 0.7, 0.5)), emptyReindexing); - expected.set(6, notification(12345, Type.feedBlock, Level.error, sourceCluster1, + expected.set(6, notification(12345, Type.feedBlock, Level.error, "Cluster [cluster1](https://console.tld/tenant/tenant1/application/app1/prod/instance/instance1?instance1.prod.us-south-3=clusters%2Ccluster1) in **prod.us-south-3** for **app1.instance1** is feed blocked", sourceCluster1, "memory (usage: 70.0%, feed block limit: 50.0%)", "disk (usage: 95.0%, feed block limit: 90.0%)")); assertEquals(expected, curatorDb.readNotifications(tenant)); } @@ -229,9 +236,9 @@ public class NotificationsDbTest { "build", reindexingStatus(null, 0.50))))); notificationsDb.setDeploymentMetricsNotifications(deploymentId, List.of( clusterMetrics("cluster1", 0.88, 0.9, 0.3, 0.5), clusterMetrics("cluster2", 0.6, 0.8, 0.9, 0.75), clusterMetrics("cluster3", 0.1, 0.8, 0.2, 0.9)), applicationReindexing1); - expected.add(notification(12345, Type.feedBlock, Level.warning, sourceCluster1, "disk (usage: 88.0%, feed block limit: 90.0%)")); - expected.add(notification(12345, Type.feedBlock, Level.error, sourceCluster2, "memory (usage: 90.0%, feed block limit: 75.0%)")); - expected.add(notification(12345, Type.reindex, Level.info, sourceCluster3, "document type 'announcements' reindexing due to a schema change (75.0% done)", "document type 'build' (50.0% done)")); + expected.add(notification(12345, Type.feedBlock, Level.warning, "Cluster [cluster1](https://console.tld/tenant/tenant1/application/app1/prod/instance/instance1?instance1.prod.us-south-3=clusters%2Ccluster1) in **prod.us-south-3** for **app1.instance1** is nearly feed blocked", sourceCluster1, "disk (usage: 88.0%, feed block limit: 90.0%)")); + expected.add(notification(12345, Type.feedBlock, Level.error, "Cluster [cluster2](https://console.tld/tenant/tenant1/application/app1/prod/instance/instance1?instance1.prod.us-south-3=clusters%2Ccluster2) in **prod.us-south-3** for **app1.instance1** is feed blocked", sourceCluster2, "memory (usage: 90.0%, feed block limit: 75.0%)")); + expected.add(notification(12345, Type.reindex, Level.info, "Cluster [cluster3](https://console.tld/tenant/tenant1/application/app1/prod/instance/instance1?instance1.prod.us-south-3=clusters%2Ccluster3%3Dreindexing) in **prod.us-south-3** for **app1.instance1** is [reindexing](https://docs.vespa.ai/en/operations/reindexing.html)", sourceCluster3, "document type 'announcements' reindexing due to a schema change (75.0% done)", "document type 'build' (50.0% done)")); assertEquals(expected, curatorDb.readNotifications(tenant)); // Cluster1 improves, while cluster3 starts having feed block issues and finishes reindexing 'build' documents @@ -241,12 +248,41 @@ public class NotificationsDbTest { "build", reindexingStatus(null, null))))); notificationsDb.setDeploymentMetricsNotifications(deploymentId, List.of( clusterMetrics("cluster1", 0.15, 0.9, 0.3, 0.5), clusterMetrics("cluster2", 0.6, 0.8, 0.9, 0.75), clusterMetrics("cluster3", 0.78, 0.8, 0.2, 0.9)), applicationReindexing2); - expected.set(6, notification(12345, Type.feedBlock, Level.error, sourceCluster2, "memory (usage: 90.0%, feed block limit: 75.0%)")); - expected.set(7, notification(12345, Type.feedBlock, Level.warning, sourceCluster3, "disk (usage: 78.0%, feed block limit: 80.0%)")); - expected.set(8, notification(12345, Type.reindex, Level.info, sourceCluster3, "document type 'announcements' reindexing due to a schema change (90.0% done)")); + expected.set(6, notification(12345, Type.feedBlock, Level.error, "Cluster [cluster2](https://console.tld/tenant/tenant1/application/app1/prod/instance/instance1?instance1.prod.us-south-3=clusters%2Ccluster2) in **prod.us-south-3** for **app1.instance1** is feed blocked", sourceCluster2, "memory (usage: 90.0%, feed block limit: 75.0%)")); + expected.set(7, notification(12345, Type.feedBlock, Level.warning, "Cluster [cluster3](https://console.tld/tenant/tenant1/application/app1/prod/instance/instance1?instance1.prod.us-south-3=clusters%2Ccluster3) in **prod.us-south-3** for **app1.instance1** is nearly feed blocked", sourceCluster3, "disk (usage: 78.0%, feed block limit: 80.0%)")); + expected.set(8, notification(12345, Type.reindex, Level.info, "Cluster [cluster3](https://console.tld/tenant/tenant1/application/app1/prod/instance/instance1?instance1.prod.us-south-3=clusters%2Ccluster3%3Dreindexing) in **prod.us-south-3** for **app1.instance1** is [reindexing](https://docs.vespa.ai/en/operations/reindexing.html)", sourceCluster3, "document type 'announcements' reindexing due to a schema change (90.0% done)")); assertEquals(expected, curatorDb.readNotifications(tenant)); } + @Test + void title_test() { + curatorDb.deleteNotifications(tenant); + TenantAndApplicationId tenantApp = TenantAndApplicationId.from(tenant.value(), "app1"); + ApplicationId app = tenantApp.instance("instance1"); + ZoneRegistryMock zoneRegistry = new ZoneRegistryMock(SystemName.Public); + + notificationsDb.setApplicationPackageNotification(NotificationSource.from(tenantApp), List.of()); + notificationsDb.setApplicationPackageNotification(NotificationSource.from(new DeploymentId(app, ZoneId.from("dev.us-east-3"))), List.of()); + notificationsDb.setSubmissionNotification(tenantApp, "msg"); + notificationsDb.setTestPackageNotification(tenantApp, List.of()); + notificationsDb.setDeploymentNotification(new RunId(app, JobType.prod("us-east-3"), 123), "msg"); + notificationsDb.setDeploymentNotification(new RunId(app, JobType.productionTestOf(ZoneId.from("prod.us-east-3")), 123), "msg"); + notificationsDb.setDeploymentNotification(new RunId(app, JobType.systemTest(zoneRegistry, CloudName.AWS), 123), "msg"); + notificationsDb.setDeploymentNotification(new RunId(app, JobType.stagingTest(zoneRegistry, CloudName.AWS), 123), "msg"); + notificationsDb.setDeploymentNotification(new RunId(app, JobType.dev("us-east-3"), 123), "msg"); + assertEquals(List.of( + "Application package for [app1](https://console.tld/tenant/tenant1/application/app1/prod/instance) has warnings", + "Application package for [app1.instance1](https://console.tld/tenant/tenant1/application/app1/dev/instance/instance1) has warnings", + "Application package for [app1](https://console.tld/tenant/tenant1/application/app1/prod/instance) has a warning", + "There are problems with tests for [app1](https://console.tld/tenant/tenant1/application/app1/prod/instance)", + "Deployment job [#123 to us-east-3](https://console.tld/tenant/tenant1/application/app1/prod/instance/instance1/job/production-us-east-3/run/123) for application **app1.instance1** has failed", + "Test job [#123 to us-east-3](https://console.tld/tenant/tenant1/application/app1/prod/instance/instance1/job/test-us-east-3/run/123) for application **app1.instance1** has failed", + "[System test #123](https://console.tld/tenant/tenant1/application/app1/prod/instance/instance1/job/system-test/run/123) for application **app1.instance1** has failed", + "[Staging test #123](https://console.tld/tenant/tenant1/application/app1/prod/instance/instance1/job/staging-test/run/123) for application **app1.instance1** has failed", + "Deployment job [#123 to dev.us-east-3](https://console.tld/tenant/tenant1/application/app1/dev/instance/instance1/job/dev-us-east-3/run/123) for application **app1.instance1** has failed" + ), notificationsDb.listNotifications(NotificationSource.from(tenant), false).stream().map(Notification::title).toList()); + } + @BeforeEach public void init() { curatorDb.writeNotifications(tenant, notifications); @@ -254,12 +290,20 @@ public class NotificationsDbTest { mailer.reset(); } + private void setNotification(Notification notification) { + notificationsDb.setNotification(notification.source(), notification.type(), notification.level(), "", notification.messages(), Optional.empty()); + } + private static List<Notification> notificationIndices(int... indices) { return Arrays.stream(indices).mapToObj(notifications::get).collect(Collectors.toCollection(ArrayList::new)); } private static Notification notification(long secondsSinceEpoch, Type type, Level level, NotificationSource source, String... messages) { - return new Notification(Instant.ofEpochSecond(secondsSinceEpoch), type, level, source, List.of(messages)); + return notification(secondsSinceEpoch, type, level, "", source, messages); + } + + private static Notification notification(long secondsSinceEpoch, Type type, Level level, String title, NotificationSource source, String... messages) { + return new Notification(Instant.ofEpochSecond(secondsSinceEpoch), type, level, source, title, List.of(messages)); } private static ClusterMetrics clusterMetrics(String clusterId, diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/notification/NotifierTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/notification/NotifierTest.java index 1251963f01c..55531dff72d 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/notification/NotifierTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/notification/NotifierTest.java @@ -9,9 +9,9 @@ import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.TenantName; import com.yahoo.vespa.flags.InMemoryFlagSource; import com.yahoo.vespa.flags.PermanentFlags; +import com.yahoo.vespa.hosted.controller.api.integration.ConsoleUrls; import com.yahoo.vespa.hosted.controller.api.integration.billing.PlanId; import com.yahoo.vespa.hosted.controller.api.integration.stubs.MockMailer; -import com.yahoo.vespa.hosted.controller.integration.ZoneRegistryMock; import com.yahoo.vespa.hosted.controller.persistence.MockCuratorDb; import com.yahoo.vespa.hosted.controller.tenant.ArchiveAccess; import com.yahoo.vespa.hosted.controller.tenant.CloudTenant; @@ -23,6 +23,7 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import java.io.IOException; +import java.net.URI; import java.time.Instant; import java.util.List; import java.util.Map; @@ -64,7 +65,7 @@ public class NotifierTest { void dispatch() throws IOException { var mailer = new MockMailer(); var flagSource = new InMemoryFlagSource().withBooleanFlag(PermanentFlags.NOTIFICATION_DISPATCH_FLAG.id(), true); - var notifier = new Notifier(curatorDb, new ZoneRegistryMock(SystemName.cd), mailer, flagSource); + var notifier = new Notifier(curatorDb, new ConsoleUrls(URI.create("https://console.tld")), mailer, flagSource); var notification = new Notification(Instant.now(), Notification.Type.testPackage, Notification.Level.warning, NotificationSource.from(ApplicationId.from(tenant, ApplicationName.defaultName(), InstanceName.defaultName())), @@ -75,7 +76,7 @@ public class NotifierTest { var mail = mailer.inbox(email.getEmailAddress()).get(0); assertEquals("[WARNING] Test package Vespa Notification for tenant1.default.default", mail.subject()); - assertEquals(new String(NotifierTest.class.getResourceAsStream("/mail/notification.txt").readAllBytes()), mail.htmlMessage().get()); + assertEquals(new String(NotifierTest.class.getResourceAsStream("/mail/notification.html").readAllBytes()), mail.htmlMessage().get()); } @Test diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/NotificationsSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/NotificationsSerializerTest.java index 63aa45a5a34..65da43a3ec4 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/NotificationsSerializerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/NotificationsSerializerTest.java @@ -8,6 +8,7 @@ import com.yahoo.slime.SlimeUtils; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; import com.yahoo.vespa.hosted.controller.deployment.DeploymentContext; +import com.yahoo.vespa.hosted.controller.notification.MailTemplating; import com.yahoo.vespa.hosted.controller.notification.Notification; import com.yahoo.vespa.hosted.controller.notification.NotificationSource; import org.junit.jupiter.api.Test; @@ -15,6 +16,7 @@ import org.junit.jupiter.api.Test; import java.io.IOException; import java.time.Instant; import java.util.List; +import java.util.Optional; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -27,6 +29,8 @@ public class NotificationsSerializerTest { void serialization_test() throws IOException { NotificationsSerializer serializer = new NotificationsSerializer(); TenantName tenantName = TenantName.from("tenant1"); + var mail = Notification.MailContent.fromTemplate(MailTemplating.Template.DEFAULT_MAIL_CONTENT).subject("My mail subject") + .with("string-param", "string-value").with("list-param", List.of("elem1", "elem2")).build(); List<Notification> notifications = List.of( new Notification(Instant.ofEpochSecond(1234), Notification.Type.applicationPackage, @@ -37,7 +41,8 @@ public class NotificationsSerializerTest { Notification.Type.deployment, Notification.Level.error, NotificationSource.from(new RunId(ApplicationId.from(tenantName.value(), "app1", "instance1"), DeploymentContext.systemTest, 12)), - List.of("Failed to deploy: Node allocation failure"))); + "Failed to deploy", List.of("Node allocation failure"), + Optional.of(mail))); Slime serialized = serializer.toSlime(notifications); assertEquals("{\"notifications\":[" + @@ -45,17 +50,22 @@ public class NotificationsSerializerTest { "\"at\":1234000," + "\"type\":\"applicationPackage\"," + "\"level\":\"warning\"," + + "\"title\":\"\"," + "\"messages\":[\"Something something deprecated...\"]," + "\"application\":\"app1\"" + "},{" + "\"at\":2345000," + "\"type\":\"deployment\"," + "\"level\":\"error\"," + - "\"messages\":[\"Failed to deploy: Node allocation failure\"]," + + "\"title\":\"Failed to deploy\"," + + "\"messages\":[\"Node allocation failure\"]," + "\"application\":\"app1\"," + "\"instance\":\"instance1\"," + "\"jobId\":\"test.us-east-1\"," + - "\"runNumber\":12" + + "\"runNumber\":12," + + "\"mail-template\":\"default-mail-content\"," + + "\"mail-subject\":\"My mail subject\"," + + "\"mail-params\":{\"list-param\":[\"elem1\",\"elem2\"],\"string-param\":\"string-value\"}" + "}]}", new String(SlimeUtils.toJsonBytes(serialized))); List<Notification> deserialized = serializer.fromSlime(tenantName, serialized); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializerTest.java index 8f114a7255c..cfc3320b083 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializerTest.java @@ -24,6 +24,8 @@ import com.yahoo.vespa.hosted.controller.tenant.CloudTenant; import com.yahoo.vespa.hosted.controller.tenant.DeletedTenant; import com.yahoo.vespa.hosted.controller.tenant.Email; import com.yahoo.vespa.hosted.controller.tenant.LastLoginInfo; +import com.yahoo.vespa.hosted.controller.tenant.PurchaseOrder; +import com.yahoo.vespa.hosted.controller.tenant.TaxId; import com.yahoo.vespa.hosted.controller.tenant.TenantAddress; import com.yahoo.vespa.hosted.controller.tenant.TenantBilling; import com.yahoo.vespa.hosted.controller.tenant.TenantContact; @@ -212,7 +214,7 @@ public class TenantSerializerTest { Slime slime = new Slime(); Cursor parentObject = slime.setObject(); serializer.toSlime(partialInfo, parentObject); - assertEquals("{\"info\":{\"name\":\"\",\"email\":\"\",\"website\":\"\",\"contactName\":\"\",\"contactEmail\":\"\",\"contactEmailVerified\":true,\"address\":{\"addressLines\":\"\",\"postalCodeOrZip\":\"\",\"city\":\"Hønefoss\",\"stateRegionProvince\":\"\",\"country\":\"\"}}}", slime.toString()); + assertEquals("{\"info\":{\"name\":\"\",\"email\":\"\",\"website\":\"\",\"contactName\":\"\",\"contactEmail\":\"\",\"contactEmailVerified\":false,\"address\":{\"addressLines\":\"\",\"postalCodeOrZip\":\"\",\"city\":\"Hønefoss\",\"stateRegionProvince\":\"\",\"country\":\"\"}}}", slime.toString()); } @Test @@ -229,12 +231,16 @@ public class TenantSerializerTest { .withCode("3510") .withRegion("Viken")) .withBilling(TenantBilling.empty() - .withContact(TenantContact.from("Thomas The Tank Engine", new Email("ceo@mycomp.any", true), "NA")) + .withContact(TenantContact.from("Thomas The Tank Engine", new Email("ceo@mycomp.any", false), "NA")) .withAddress(TenantAddress.empty() .withCity("Suddery") .withCountry("Sodor") .withAddress("Central Station") - .withRegion("Irish Sea"))); + .withRegion("Irish Sea")) + .withPurchaseOrder(new PurchaseOrder("PO42")) + .withTaxId(new TaxId("no_vat", "123456789MVA")) + .withInvoiceEmail(new Email("billing@mycomp.any", false)) + ); Slime slime = new Slime(); Cursor parentCursor = slime.setObject(); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java index 6103b715744..3ada598f4f8 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java @@ -77,7 +77,6 @@ public class ControllerContainerTest { <component id='com.yahoo.vespa.hosted.controller.Controller'/> <component id='com.yahoo.vespa.hosted.controller.integration.ConfigServerProxyMock'/> <component id='com.yahoo.vespa.hosted.controller.maintenance.ControllerMaintenance'/> - <component id='com.yahoo.vespa.hosted.controller.api.integration.MockPricingController'/> <component id='com.yahoo.vespa.hosted.controller.api.integration.stubs.MockMavenRepository'/> <component id='com.yahoo.vespa.hosted.controller.api.integration.stubs.MockUserManagement'/> <component id='com.yahoo.vespa.hosted.controller.integration.SecretStoreMock'/> @@ -118,9 +117,6 @@ public class ControllerContainerTest { <handler id='com.yahoo.vespa.hosted.controller.restapi.changemanagement.ChangeManagementApiHandler'> <binding>http://localhost/changemanagement/v1/*</binding> </handler> - <handler id='com.yahoo.vespa.hosted.controller.restapi.pricing.PricingApiHandler'> - <binding>http://localhost/pricing/v1/*</binding> - </handler> %s </container> """.formatted(system().value(), variablePartXml()); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiCloudTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiCloudTest.java index e89c913ab7d..1ad88675169 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiCloudTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiCloudTest.java @@ -49,7 +49,6 @@ import static com.yahoo.application.container.handler.Request.Method.DELETE; import static com.yahoo.application.container.handler.Request.Method.GET; import static com.yahoo.application.container.handler.Request.Method.POST; import static com.yahoo.application.container.handler.Request.Method.PUT; -import static java.nio.charset.StandardCharsets.UTF_8; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -79,7 +78,7 @@ public class ApplicationApiCloudTest extends ControllerContainerCloudTest { void tenant_info_profile() { var request = request("/application/v4/tenant/scoober/info/profile", GET) .roles(Set.of(Role.reader(tenantName))); - tester.assertResponse(request, "{}", 200); + tester.assertResponse(request, "{\"contact\":{\"name\":\"\",\"email\":\"\",\"emailVerified\":false},\"tenant\":{\"company\":\"\",\"website\":\"\"}}", 200); var updateRequest = request("/application/v4/tenant/scoober/info/profile", PUT) .data("{\"contact\":{\"name\":\"Some Name\",\"email\":\"foo@example.com\"},\"tenant\":{\"company\":\"Scoober, Inc.\",\"website\":\"https://example.com/\"}}") @@ -99,34 +98,109 @@ public class ApplicationApiCloudTest extends ControllerContainerCloudTest { @Test void tenant_info_billing() { + var expectedResponse = """ + { + "contact": { + "name":"", + "email":"", + "emailVerified":false, + "phone":"" + }, + "taxId": { + "type": "", + "code": "" + }, + "purchaseOrder":"", + "invoiceEmail":"" + } + """; var request = request("/application/v4/tenant/scoober/info/billing", GET) .roles(Set.of(Role.reader(tenantName))); - tester.assertResponse(request, "{}", 200); - - var fullAddress = "{\"addressLines\":\"addressLines\",\"postalCodeOrZip\":\"postalCodeOrZip\",\"city\":\"city\",\"stateRegionProvince\":\"stateRegionProvince\",\"country\":\"country\"}"; - var fullBillingContact = "{\"contact\":{\"name\":\"name\",\"email\":\"foo@example\",\"phone\":\"phone\"},\"address\":" + fullAddress + "}"; - + tester.assertJsonResponse(request, expectedResponse, 200); + + var fullBillingContact = """ + { + "contact": { + "name":"name", + "email":"foo@example", + "phone":"phone" + }, + "taxId":{"type": "no_vat", "code": "123456789MVA"}, + "purchaseOrder":"PO9001", + "invoiceEmail":"billing@mycomp.any", + "address": { + "addressLines":"addressLines", + "postalCodeOrZip":"postalCodeOrZip", + "city":"city", + "stateRegionProvince":"stateRegionProvince", + "country":"country" + } + } + """; var updateRequest = request("/application/v4/tenant/scoober/info/billing", PUT) .data(fullBillingContact) .roles(Set.of(Role.administrator(tenantName))); tester.assertResponse(updateRequest, "{\"message\":\"Tenant info updated\"}", 200); - tester.assertResponse(request, "{\"contact\":{\"name\":\"name\",\"email\":\"foo@example\",\"phone\":\"phone\"},\"address\":{\"addressLines\":\"addressLines\",\"postalCodeOrZip\":\"postalCodeOrZip\",\"city\":\"city\",\"stateRegionProvince\":\"stateRegionProvince\",\"country\":\"country\"}}", 200); + expectedResponse = """ + { + "contact": { + "name":"name", + "email":"foo@example", + "emailVerified": false, + "phone":"phone" + }, + "taxId": { + "type": "no_vat", + "code": "123456789MVA" + }, + "purchaseOrder":"PO9001", + "invoiceEmail":"billing@mycomp.any", + "address": { + "addressLines":"addressLines", + "postalCodeOrZip":"postalCodeOrZip", + "city":"city", + "stateRegionProvince":"stateRegionProvince", + "country":"country" + } + } + """; + tester.assertJsonResponse(request, expectedResponse, 200); } @Test void tenant_info_contacts() { var request = request("/application/v4/tenant/scoober/info/contacts", GET) .roles(Set.of(Role.reader(tenantName))); - tester.assertResponse(request, "{\"contacts\":[]}", 200); - - - var fullContacts = "{\"contacts\":[{\"audiences\":[\"tenant\"],\"email\":\"contact1@example.com\",\"emailVerified\":false},{\"audiences\":[\"notifications\"],\"email\":\"contact2@example.com\",\"emailVerified\":false},{\"audiences\":[\"tenant\",\"notifications\"],\"email\":\"contact3@example.com\",\"emailVerified\":false}]}"; + tester.assertResponse(request, "{\"contacts\":[{\"audiences\":[\"tenant\",\"notifications\"],\"email\":\"developer@scoober\",\"emailVerified\":true}]}", 200); + + + var fullContacts = """ + { + "contacts":[ + { + "audiences":["tenant"] + ,"email":"contact1@example.com", + "emailVerified":false + }, + { + "audiences":["notifications"], + "email":"contact2@example.com", + "emailVerified":false + }, + { + "audiences":["tenant","notifications"], + "email":"contact3@example.com", + "emailVerified":false + } + ] + } + """; var updateRequest = request("/application/v4/tenant/scoober/info/contacts", PUT) .data(fullContacts) .roles(Set.of(Role.administrator(tenantName))); tester.assertResponse(updateRequest, "{\"message\":\"Tenant info updated\"}", 200); - tester.assertResponse(request, fullContacts, 200); + tester.assertJsonResponse(request, fullContacts, 200); } @Test @@ -134,7 +208,7 @@ public class ApplicationApiCloudTest extends ControllerContainerCloudTest { var infoRequest = request("/application/v4/tenant/scoober/info", GET) .roles(Set.of(Role.reader(tenantName))); - tester.assertResponse(infoRequest, "{}", 200); + tester.assertResponse(infoRequest, "{\"name\":\"\",\"email\":\"\",\"website\":\"\",\"contactName\":\"\",\"contactEmail\":\"\",\"contactEmailVerified\":false,\"contacts\":[{\"audiences\":[\"tenant\",\"notifications\"],\"email\":\"developer@scoober\",\"emailVerified\":true}]}", 200); String partialInfo = "{\"contactName\":\"newName\", \"contactEmail\": \"foo@example.com\", \"billingContact\":{\"name\":\"billingName\"}}"; var postPartial = @@ -151,13 +225,85 @@ public class ApplicationApiCloudTest extends ControllerContainerCloudTest { tester.assertResponse(postPartialContacts, "{\"message\":\"Tenant info updated\"}", 200); // Read back the updated info - tester.assertResponse(infoRequest, "{\"name\":\"\",\"email\":\"\",\"website\":\"\",\"contactName\":\"newName\",\"contactEmail\":\"foo@example.com\",\"contactEmailVerified\":false,\"billingContact\":{\"name\":\"billingName\",\"email\":\"\",\"phone\":\"\"},\"contacts\":[{\"audiences\":[\"tenant\"],\"email\":\"contact1@example.com\",\"emailVerified\":false}]}", 200); - - String fullAddress = "{\"addressLines\":\"addressLines\",\"postalCodeOrZip\":\"postalCodeOrZip\",\"city\":\"city\",\"stateRegionProvince\":\"stateRegionProvince\",\"country\":\"country\"}"; - String fullBillingContact = "{\"name\":\"name\",\"email\":\"foo@example\",\"phone\":\"phone\",\"address\":" + fullAddress + "}"; - String fullContacts = "[{\"audiences\":[\"tenant\"],\"email\":\"contact1@example.com\",\"emailVerified\":false},{\"audiences\":[\"notifications\"],\"email\":\"contact2@example.com\",\"emailVerified\":false},{\"audiences\":[\"tenant\",\"notifications\"],\"email\":\"contact3@example.com\",\"emailVerified\":false}]"; - String fullInfo = "{\"name\":\"name\",\"email\":\"foo@example\",\"website\":\"https://yahoo.com\",\"contactName\":\"contactName\",\"contactEmail\":\"contact@example.com\",\"contactEmailVerified\":false,\"address\":" + fullAddress + ",\"billingContact\":" + fullBillingContact + ",\"contacts\":" + fullContacts + "}"; - + var expectedResponse = """ + { + "name":"", + "email":"", + "website":"", + "contactName":"newName", + "contactEmail":"foo@example.com", + "contactEmailVerified":false, + "billingContact": { + "name":"billingName", + "email":"","emailVerified":false, + "phone":"", + "taxId": { + "type": "", + "code": "" + }, + "purchaseOrder":"", + "invoiceEmail":"" + }, + "contacts": [ + {"audiences":["tenant"],"email":"contact1@example.com","emailVerified":false} + ] + } + """; + tester.assertJsonResponse(infoRequest, expectedResponse, 200); + + var fullInfo = """ + { + "name":"name", + "email":"foo@example", + "website":"https://yahoo.com", + "contactName":"contactName", + "contactEmail":"contact@example.com", + "contactEmailVerified":false, + "address": { + "addressLines":"addressLines", + "postalCodeOrZip":"postalCodeOrZip", + "city":"city", + "stateRegionProvince":"stateRegionProvince", + "country":"country" + }, + "billingContact": { + "name":"name", + "email":"foo@example", + "emailVerified":false, + "phone":"phone", + "taxId": { + "type": "", + "code": "" + }, + "purchaseOrder":"", + "invoiceEmail":"", + "address": { + "addressLines":"addressLines", + "postalCodeOrZip":"postalCodeOrZip", + "city":"city", + "stateRegionProvince":"stateRegionProvince", + "country":"country" + } + }, + "contacts": [ + { + "audiences":["tenant"], + "email":"contact1@example.com", + "emailVerified":false + }, + { + "audiences":["notifications"], + "email":"contact2@example.com", + "emailVerified":false + }, + { + "audiences":["tenant","notifications"] + ,"email":"contact3@example.com", + "emailVerified":false + } + ] + } + """; // Now set all fields var postFull = request("/application/v4/tenant/scoober/info", PUT) @@ -166,7 +312,7 @@ public class ApplicationApiCloudTest extends ControllerContainerCloudTest { tester.assertResponse(postFull, "{\"message\":\"Tenant info updated\"}", 200); // Now compare the updated info with the full info we sent - tester.assertResponse(infoRequest, fullInfo, 200); + tester.assertJsonResponse(infoRequest, fullInfo, 200); var invalidBody = "{\"mail\":\"contact1@example.com\", \"mailType\":\"blurb\"}"; var resendMailRequest = @@ -189,7 +335,7 @@ public class ApplicationApiCloudTest extends ControllerContainerCloudTest { var infoRequest = request("/application/v4/tenant/scoober/info", GET) .roles(Set.of(Role.reader(tenantName))); - tester.assertResponse(infoRequest, "{}", 200); + tester.assertResponse(infoRequest, "{\"name\":\"\",\"email\":\"\",\"website\":\"\",\"contactName\":\"\",\"contactEmail\":\"\",\"contactEmailVerified\":false,\"contacts\":[{\"audiences\":[\"tenant\",\"notifications\"],\"email\":\"developer@scoober\",\"emailVerified\":true}]}", 200); // name needs to be present and not blank var partialInfoMissingName = "{\"contactName\": \" \"}"; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java index cc336bfb35b..66fb17410fd 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java @@ -61,7 +61,6 @@ import com.yahoo.vespa.hosted.controller.integration.ConfigServerMock; import com.yahoo.vespa.hosted.controller.integration.NodeRepositoryMock; import com.yahoo.vespa.hosted.controller.integration.ZoneApiMock; import com.yahoo.vespa.hosted.controller.metric.ApplicationMetrics; -import com.yahoo.vespa.hosted.controller.notification.Notification; import com.yahoo.vespa.hosted.controller.notification.NotificationSource; import com.yahoo.vespa.hosted.controller.restapi.ContainerTester; import com.yahoo.vespa.hosted.controller.restapi.ControllerContainerTest; @@ -1979,15 +1978,11 @@ public class ApplicationApiTest extends ControllerContainerTest { } private void addNotifications(TenantName tenantName) { - tester.controller().notificationsDb().setNotification( + tester.controller().notificationsDb().setApplicationPackageNotification( NotificationSource.from(TenantAndApplicationId.from(tenantName.value(), "app1")), - Notification.Type.applicationPackage, - Notification.Level.warning, - "Something something deprecated..."); - tester.controller().notificationsDb().setNotification( - NotificationSource.from(new RunId(ApplicationId.from(tenantName.value(), "app2", "instance1"), DeploymentContext.systemTest, 12)), - Notification.Type.deployment, - Notification.Level.error, + List.of("Something something deprecated...")); + tester.controller().notificationsDb().setDeploymentNotification( + new RunId(ApplicationId.from(tenantName.value(), "app2", "instance1"), DeploymentContext.systemTest, 12), "Failed to deploy: Node allocation failure"); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/notifications-tenant1-app2.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/notifications-tenant1-app2.json index 556440c40d5..6206e3b277a 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/notifications-tenant1-app2.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/notifications-tenant1-app2.json @@ -4,6 +4,7 @@ "at": 1600000000000, "level": "error", "type": "deployment", + "title": "[System test #12](https://console.tld/tenant/tenant1/application/app2/prod/instance/instance1/job/system-test/run/12) for application **app2.instance1** has failed", "messages": [ "Failed to deploy: Node allocation failure" ], diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/notifications-tenant1.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/notifications-tenant1.json index 1a731dfe4a9..78deea65008 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/notifications-tenant1.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/notifications-tenant1.json @@ -4,6 +4,7 @@ "at": 1600000000000, "level": "warning", "type": "applicationPackage", + "title": "Application package for [app1](https://console.tld/tenant/tenant1/application/app1/prod/instance) has a warning", "messages": [ "Something something deprecated..." ], @@ -13,6 +14,7 @@ "at": 1600000000000, "level": "error", "type": "deployment", + "title": "[System test #12](https://console.tld/tenant/tenant1/application/app2/prod/instance/instance1/job/system-test/run/12) for application **app2.instance1** has failed", "messages": [ "Failed to deploy: Node allocation failure" ], diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/overview-enclave.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/overview-enclave.json index ef9c8a608ab..1d2cd8eaabb 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/overview-enclave.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/overview-enclave.json @@ -81,6 +81,9 @@ "commit": "commit1" } }, + "enclave": { + "cloudAccount": "aws:123456789012" + }, "steps": [ { "name": "deployTester", @@ -177,6 +180,9 @@ "commit": "commit1" } }, + "enclave": { + "cloudAccount": "aws:123456789012" + }, "steps": [ { "name": "deployTester", @@ -264,6 +270,9 @@ "commit": "commit1" } }, + "enclave": { + "cloudAccount": "aws:123456789012" + }, "steps": [ { "name": "deployReal", diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandlerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandlerTest.java deleted file mode 100644 index 7f68fbf6909..00000000000 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandlerTest.java +++ /dev/null @@ -1,236 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.restapi.billing; - -import com.yahoo.config.provision.SystemName; -import com.yahoo.config.provision.TenantName; -import com.yahoo.vespa.hosted.controller.api.integration.billing.Bill; -import com.yahoo.vespa.hosted.controller.api.integration.billing.CollectionMethod; -import com.yahoo.vespa.hosted.controller.api.integration.billing.MockBillingController; -import com.yahoo.vespa.hosted.controller.api.integration.billing.PlanId; -import com.yahoo.vespa.hosted.controller.api.role.Role; -import com.yahoo.vespa.hosted.controller.restapi.ContainerTester; -import com.yahoo.vespa.hosted.controller.restapi.ControllerContainerCloudTest; -import com.yahoo.vespa.hosted.controller.security.Auth0Credentials; -import com.yahoo.vespa.hosted.controller.security.CloudTenantSpec; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - -import java.io.File; -import java.math.BigDecimal; -import java.time.LocalDate; -import java.time.ZoneOffset; -import java.time.ZonedDateTime; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.Set; -import java.util.TreeMap; - -import static com.yahoo.application.container.handler.Request.Method.DELETE; -import static com.yahoo.application.container.handler.Request.Method.GET; -import static com.yahoo.application.container.handler.Request.Method.PATCH; -import static com.yahoo.application.container.handler.Request.Method.POST; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; - -/** - * @author olaa - */ -public class BillingApiHandlerTest extends ControllerContainerCloudTest { - - private static final String responseFiles = "src/test/java/com/yahoo/vespa/hosted/controller/restapi/billing/responses/"; - private static final TenantName tenant = TenantName.from("tenant1"); - private static final TenantName tenant2 = TenantName.from("tenant2"); - private static final Set<Role> tenantRole = Set.of(Role.administrator(tenant)); - private static final Set<Role> financeAdmin = Set.of(Role.hostedAccountant()); - private MockBillingController billingController; - - private ContainerTester tester; - - @BeforeEach - public void setup() { - tester = new ContainerTester(container, responseFiles); - billingController = (MockBillingController) tester.serviceRegistry().billingController(); - } - - @Override - protected SystemName system() { - return SystemName.PublicCd; - } - - @Override - protected String variablePartXml() { - return " <component id='com.yahoo.vespa.hosted.controller.security.CloudAccessControlRequests'/>\n" + - " <component id='com.yahoo.vespa.hosted.controller.security.CloudAccessControl'/>\n" + - - " <handler id='com.yahoo.vespa.hosted.controller.restapi.billing.BillingApiHandler'>\n" + - " <binding>http://*/billing/v1/*</binding>\n" + - " </handler>\n" + - - " <http>\n" + - " <server id='default' port='8080' />\n" + - " <filtering>\n" + - " <request-chain id='default'>\n" + - " <filter id='com.yahoo.vespa.hosted.controller.restapi.filter.ControllerAuthorizationFilter'/>\n" + - " <binding>http://*/*</binding>\n" + - " </request-chain>\n" + - " </filtering>\n" + - " </http>\n"; - } - - @Test - void list_plans() { - var listPlansRequest = request("/billing/v1/plans", GET) - .roles(Role.hostedAccountant()); - tester.assertResponse(listPlansRequest, "{\"plans\":[{\"id\":\"trial\",\"name\":\"Free Trial - for testing purposes\"},{\"id\":\"paid\",\"name\":\"Paid Plan - for testing purposes\"},{\"id\":\"none\",\"name\":\"None Plan - for testing purposes\"}]}"); - } - - @Test - void response_list_bills() { - var bill = createBill(); - - billingController.addBill(tenant, bill, true); - billingController.addBill(tenant, bill, false); - billingController.setPlan(tenant, PlanId.from("some-plan"), true, false); - - var request = request("/billing/v1/tenant/tenant1/billing?until=2020-05-28").roles(tenantRole); - tester.assertResponse(request, new File("tenant-billing-view.json")); - - } - - @Test - void test_bill_creation() { - var bills = billingController.getBillsForTenant(tenant); - assertEquals(0, bills.size()); - - String requestBody = "{\"tenant\":\"tenant1\", \"startTime\":\"2020-04-20\", \"endTime\":\"2020-05-20\"}"; - var request = request("/billing/v1/invoice", POST) - .data(requestBody) - .roles(tenantRole); - - tester.assertResponse(request, accessDenied, 403); - request.roles(financeAdmin); - tester.assertResponse(request, new File("invoice-creation-response.json")); - - bills = billingController.getBillsForTenant(tenant); - assertEquals(1, bills.size()); - Bill bill = bills.get(0); - assertEquals("2020-04-20T00:00Z", bill.getStartTime().toString()); - assertEquals("2020-05-21T00:00Z", bill.getEndTime().toString()); - - assertEquals("2020-04-20", bill.getStartDate().toString()); - assertEquals("2020-05-20", bill.getEndDate().toString()); - } - - @Test - void adding_and_listing_line_item() { - - var requestBody = "{" + - "\"description\":\"some description\"," + - "\"amount\":\"123.45\" " + - "}"; - - var request = request("/billing/v1/invoice/tenant/tenant1/line-item", POST) - .data(requestBody) - .roles(financeAdmin); - - tester.assertResponse(request, "{\"message\":\"Added line item for tenant tenant1\"}"); - - var lineItems = billingController.getUnusedLineItems(tenant); - assertEquals(1, lineItems.size()); - Bill.LineItem lineItem = lineItems.get(0); - assertEquals("some description", lineItem.description()); - assertEquals(new BigDecimal("123.45"), lineItem.amount()); - - request = request("/billing/v1/invoice/tenant/tenant1/line-item") - .roles(financeAdmin); - - tester.assertResponse(request, new File("line-item-list.json")); - } - - @Test - void adding_new_status() { - billingController.addBill(tenant, createBill(), true); - - var requestBody = "{\"status\":\"DONE\"}"; - var request = request("/billing/v1/invoice/id-1/status", POST) - .data(requestBody) - .roles(financeAdmin); - tester.assertResponse(request, "{\"message\":\"Updated status of invoice id-1\"}"); - - var bill = billingController.getBillsForTenant(tenant).get(0); - assertEquals("DONE", bill.status()); - } - - @Test - void list_all_unbilled_items() { - tester.controller().tenants().create(new CloudTenantSpec(tenant, ""), new Auth0Credentials(() -> "foo", Set.of(Role.hostedOperator()))); - tester.controller().tenants().create(new CloudTenantSpec(tenant2, ""), new Auth0Credentials(() -> "foo", Set.of(Role.hostedOperator()))); - - var bill = createBill(); - billingController.setPlan(tenant, PlanId.from("some-plan"), true, false); - billingController.setPlan(tenant2, PlanId.from("some-plan"), true, false); - billingController.addBill(tenant, bill, false); - billingController.addLineItem(tenant, "support", new BigDecimal("42"), Optional.empty(), "Smith"); - billingController.addBill(tenant2, bill, false); - - var request = request("/billing/v1/billing?until=2020-05-28").roles(financeAdmin); - - tester.assertResponse(request, new File("billing-all-tenants.json")); - } - - @Test - void csv_export() { - var bill = createBill(); - billingController.addBill(tenant, bill, true); - var csvRequest = request("/billing/v1/invoice/export", GET).roles(financeAdmin); - tester.assertResponse(csvRequest.get(), new File("billing-all-invoices"), 200, false); - } - - @Test - void patch_collection_method() { - test_patch_collection_with_field_name("collectionMethod"); - test_patch_collection_with_field_name("collection"); - } - - private void test_patch_collection_with_field_name(String fieldName) { - var planRequest = request("/billing/v1/tenant/tenant1/collection", PATCH) - .data("{\"" + fieldName + "\": \"invoice\"}") - .roles(financeAdmin); - tester.assertResponse(planRequest, "Collection method updated to INVOICE"); - assertEquals(CollectionMethod.INVOICE, billingController.getCollectionMethod(tenant)); - - // Test that not event tenant administrators can do this - planRequest = request("/billing/v1/tenant/tenant1/collection", PATCH) - .data("{\"collectionMethod\": \"epay\"}") - .roles(tenantRole); - tester.assertResponse(planRequest, accessDenied, 403); - assertEquals(CollectionMethod.INVOICE, billingController.getCollectionMethod(tenant)); - } - - static Bill createBill() { - var start = LocalDate.of(2020, 5, 23).atStartOfDay(ZoneOffset.UTC); - var end = start.toLocalDate().plusDays(6).atStartOfDay(ZoneOffset.UTC); - var statusHistory = new Bill.StatusHistory(new TreeMap<>(Map.of(start, "OPEN"))); - return new Bill( - Bill.Id.of("id-1"), - TenantName.defaultName(), - statusHistory, - List.of(createLineItem(start)), - start, - end - ); - } - - static Bill.LineItem createLineItem(ZonedDateTime addedAt) { - return new Bill.LineItem( - "some-id", - "description", - new BigDecimal("123.00"), - "paid", - "Smith", - addedAt - ); - } - -} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandlerV2Test.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandlerV2Test.java index 78bbb006467..356076a8d00 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandlerV2Test.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandlerV2Test.java @@ -4,7 +4,10 @@ package com.yahoo.vespa.hosted.controller.restapi.billing; import com.yahoo.application.container.handler.Request; import com.yahoo.config.provision.TenantName; import com.yahoo.test.ManualClock; +import com.yahoo.vespa.hosted.controller.api.integration.billing.Bill; +import com.yahoo.vespa.hosted.controller.api.integration.billing.BillStatus; import com.yahoo.vespa.hosted.controller.api.integration.billing.MockBillingController; +import com.yahoo.vespa.hosted.controller.api.integration.billing.StatusHistory; import com.yahoo.vespa.hosted.controller.api.role.Role; import com.yahoo.vespa.hosted.controller.restapi.ContainerTester; import com.yahoo.vespa.hosted.controller.restapi.ControllerContainerCloudTest; @@ -13,8 +16,16 @@ import com.yahoo.vespa.hosted.controller.security.CloudTenantSpec; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import java.math.BigDecimal; +import java.io.File; import java.time.Instant; +import java.time.LocalDate; +import java.time.ZoneOffset; +import java.time.ZonedDateTime; +import java.util.List; +import java.util.Map; import java.util.Set; +import java.util.TreeMap; /** * @author ogronnesby @@ -29,11 +40,6 @@ public class BillingApiHandlerV2Test extends ControllerContainerCloudTest { private static final Set<Role> tenantAdmin = Set.of(Role.administrator(tenant)); private static final Set<Role> financeAdmin = Set.of(Role.hostedAccountant()); - private static final String ACCESS_DENIED = "{\n" + - " \"code\" : 403,\n" + - " \"message\" : \"Access denied\"\n" + - "}"; - private MockBillingController billingController; private ContainerTester tester; @@ -44,7 +50,7 @@ public class BillingApiHandlerV2Test extends ControllerContainerCloudTest { var clock = (ManualClock) tester.controller().serviceRegistry().clock(); clock.setInstant(Instant.parse("2021-04-13T00:00:00Z")); billingController = (MockBillingController) tester.serviceRegistry().billingController(); - billingController.addBill(tenant, BillingApiHandlerTest.createBill(), true); + billingController.addBill(tenant, createBill(), true); } @Override @@ -103,7 +109,7 @@ public class BillingApiHandlerV2Test extends ControllerContainerCloudTest { var singleRequest = request("/billing/v2/tenant/" + tenant + "/bill/id-1").roles(tenantReader); tester.assertResponse(singleRequest, """ - {"id":"id-1","from":"2020-05-23","to":"2020-05-28","total":"123.00","status":"OPEN","statusHistory":[{"at":"2020-05-23T00:00:00Z","status":"OPEN"}],"items":[{"id":"some-id","description":"description","amount":"123.00","plan":{"id":"paid","name":"Paid Plan - for testing purposes"},"majorVersion":0,"cpu":{},"memory":{},"disk":{}}]}"""); + {"id":"id-1","from":"2020-05-23","to":"2020-05-28","total":"123.00","status":"OPEN","statusHistory":[{"at":"2020-05-23T00:00:00Z","status":"OPEN"}],"items":[{"id":"some-id","description":"description","amount":"123.00","plan":{"id":"paid","name":"Paid Plan - for testing purposes"},"majorVersion":0,"cpu":{},"memory":{},"disk":{},"gpu":{}}]}"""); } @Test @@ -122,7 +128,7 @@ public class BillingApiHandlerV2Test extends ControllerContainerCloudTest { @Test void require_accountant_preview() { var accountantRequest = request("/billing/v2/accountant/preview").roles(Role.hostedAccountant()); - billingController.uncommittedBills.put(tenant, BillingApiHandlerTest.createBill()); + billingController.uncommittedBills.put(tenant, createBill()); tester.assertResponse(accountantRequest, """ {"tenants":[{"tenant":"tenant1","plan":{"id":"trial","name":"Free Trial - for testing purposes"},"quota":{"budget":-1.0},"collection":"AUTO","lastBill":"2020-05-23","unbilled":"123.00"}]}"""); @@ -130,13 +136,13 @@ public class BillingApiHandlerV2Test extends ControllerContainerCloudTest { @Test void require_accountant_tenant_preview() { - var accountantRequest = request("/billing/v2/accountant/preview/tenant/tenant1").roles(Role.hostedAccountant()); + var accountantRequest = request("/billing/v2/accountant/tenant/tenant1/preview").roles(Role.hostedAccountant()); tester.assertResponse(accountantRequest, "{\"id\":\"empty\",\"from\":\"2021-04-13\",\"to\":\"2021-04-12\",\"total\":\"0.00\",\"status\":\"OPEN\",\"statusHistory\":[{\"at\":\"2021-04-13T00:00:00Z\",\"status\":\"OPEN\"}],\"items\":[]}"); } @Test void require_accountant_tenant_bill() { - var accountantRequest = request("/billing/v2/accountant/preview/tenant/tenant1", Request.Method.POST) + var accountantRequest = request("/billing/v2/accountant/tenant/tenant1/preview", Request.Method.POST) .roles(Role.hostedAccountant()) .data("{\"from\": \"2020-05-01\",\"to\": \"2020-06-01\"}"); tester.assertResponse(accountantRequest, "{\"message\":\"Created bill id-123\"}"); @@ -148,4 +154,132 @@ public class BillingApiHandlerV2Test extends ControllerContainerCloudTest { .roles(Role.hostedAccountant()); tester.assertResponse(accountantRequest, "{\"plans\":[{\"id\":\"trial\",\"name\":\"Free Trial - for testing purposes\"},{\"id\":\"paid\",\"name\":\"Paid Plan - for testing purposes\"},{\"id\":\"none\",\"name\":\"None Plan - for testing purposes\"}]}"); } + + @Test + void require_additional_items_empty() { + var accountantRequest = request("/billing/v2/accountant/tenant/tenant1/items") + .roles(Role.hostedAccountant()); + tester.assertResponse(accountantRequest, """ + {"items":[]}"""); + } + + @Test + void require_additional_items_with_content() { + { + var accountantRequest = request("/billing/v2/accountant/tenant/tenant1/items", Request.Method.POST) + .roles(Role.hostedAccountant()) + .data(""" + { + "description": "Additional support costs", + "amount": "123.45" + }"""); + tester.assertResponse(accountantRequest, """ + {"message":"Added line item for tenant tenant1"}"""); + } + + { + var accountantRequest = request("/billing/v2/accountant/tenant/tenant1/items") + .roles(Role.hostedAccountant()); + tester.assertResponse(accountantRequest, """ + {"items":[{"id":"line-item-id","description":"Additional support costs","amount":"123.45","plan":{"id":"paid","name":"Paid Plan - for testing purposes"},"majorVersion":0,"cpu":{},"memory":{},"disk":{},"gpu":{}}]}"""); + } + + { + var accountantRequest = request("/billing/v2/accountant/tenant/tenant1/item/line-item-id", Request.Method.DELETE) + .roles(Role.hostedAccountant()); + tester.assertResponse(accountantRequest, """ + {"message":"Successfully deleted line item line-item-id"}"""); + } + } + + @Test + void require_current_plan() { + { + var accountantRequest = request("/billing/v2/accountant/tenant/tenant1/plan") + .roles(Role.hostedAccountant()); + tester.assertResponse(accountantRequest, """ + {"id":"trial","name":"Free Trial - for testing purposes"}"""); + } + + { + var accountantRequest = request("/billing/v2/accountant/tenant/tenant1/plan", Request.Method.POST) + .roles(Role.hostedAccountant()) + .data(""" + {"id": "paid"}"""); + tester.assertResponse(accountantRequest, """ + {"message":"Plan: paid"}"""); + } + + { + var accountantRequest = request("/billing/v2/accountant/tenant/tenant1/plan") + .roles(Role.hostedAccountant()); + tester.assertResponse(accountantRequest, """ + {"id":"paid","name":"Paid Plan - for testing purposes"}"""); + } + } + + @Test + void require_current_collection() { + { + var accountantRequest = request("/billing/v2/accountant/tenant/tenant1/collection") + .roles(Role.hostedAccountant()); + tester.assertResponse(accountantRequest, """ + {"collection":"AUTO"}"""); + } + + { + var accountantRequest = request("/billing/v2/accountant/tenant/tenant1/collection", Request.Method.POST) + .roles(Role.hostedAccountant()) + .data(""" + {"collection": "INVOICE"}"""); + tester.assertResponse(accountantRequest, """ + {"message":"Collection: INVOICE"}"""); + } + + { + var accountantRequest = request("/billing/v2/accountant/tenant/tenant1/collection") + .roles(Role.hostedAccountant()); + tester.assertResponse(accountantRequest, """ + {"collection":"INVOICE"}"""); + } + } + + @Test + void require_accountant_tenant() { + var accountantRequest = request("/billing/v2/accountant/tenant/tenant1") + .roles(Role.hostedAccountant()); + tester.assertResponse(accountantRequest, """ + {"tenant":"tenant1","plan":{"id":"trial","name":"Free Trial - for testing purposes","billed":false,"supported":false},"billing":{},"collection":"AUTO"}"""); + } + + @Test + void lists_accepted_countries() { + var req = request("/billing/v2/countries").roles(tenantReader); + tester.assertJsonResponse(req, new File("accepted-countries.json")); + } + + private static Bill createBill() { + var start = LocalDate.of(2020, 5, 23).atStartOfDay(ZoneOffset.UTC); + var end = start.toLocalDate().plusDays(6).atStartOfDay(ZoneOffset.UTC); + var statusHistory = new StatusHistory(new TreeMap<>(Map.of(start, BillStatus.OPEN))); + return new Bill( + Bill.Id.of("id-1"), + TenantName.defaultName(), + statusHistory, + List.of(createLineItem(start)), + start, + end + ); + } + + static Bill.LineItem createLineItem(ZonedDateTime addedAt) { + return new Bill.LineItem( + "some-id", + "description", + new BigDecimal("123.00"), + "paid", + "Smith", + addedAt + ); + } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/billing/responses/accepted-countries.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/billing/responses/accepted-countries.json new file mode 100644 index 00000000000..2714de1e64d --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/billing/responses/accepted-countries.json @@ -0,0 +1,34 @@ +{ + "countries": [ + { + "code": "NO", + "name": "Norway", + "taxTypes": [ + { + "id": "no_vat", + "description": "Norwegian VAT number", + "pattern": "[0-9]{9}MVA", + "example": "123456789MVA" + } + ] + }, + { + "code": "CA", + "name": "Canada", + "taxTypes": [ + { + "id": "ca_gst_hst", + "description": "Canadian GST/HST number", + "pattern": "([0-9]{9}) ?RT ?([0-9]{4})", + "example": "123456789RT0002" + }, + { + "id": "ca_pst_bc", + "description": "Canadian PST number (British Columbia)", + "pattern": "PST-?([0-9]{4})-?([0-9]{4})", + "example": "PST-1234-5678" + } + ] + } + ] +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/billing/responses/billing-all-invoices b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/billing/responses/billing-all-invoices deleted file mode 100644 index 957ed858951..00000000000 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/billing/responses/billing-all-invoices +++ /dev/null @@ -1,2 +0,0 @@ -ID,Tenant,From,To,CpuHours,MemoryHours,DiskHours,Cpu,Memory,Disk,Additional -id-1,default,2020-05-23,2020-05-28,0.00,0.00,0.00,0.00,0.00,0.00,123.00 diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/billing/responses/billing-all-tenants.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/billing/responses/billing-all-tenants.json deleted file mode 100644 index d761439667a..00000000000 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/billing/responses/billing-all-tenants.json +++ /dev/null @@ -1,62 +0,0 @@ -{ - "until": "2020-05-28", - "tenants": [ - { - "tenant": "tenant1", - "plan": "some-plan", - "planName": "Plan with id: some-plan", - "collection": "AUTO", - "current": { - "amount": "123.00", - "status": "accrued", - "from": "2020-05-23", - "items": [ - { - "id": "some-id", - "description": "description", - "amount": "123.00", - "plan": "paid", - "planName": "Plan with id: paid", - "majorVersion": 0 - } - ] - }, - "additional": { - "items": [ - { - "id": "line-item-id", - "description": "support", - "amount": "42.00", - "plan": "some-plan", - "planName": "Plan with id: some-plan", - "majorVersion": 0 - } - ] - } - }, - { - "tenant": "tenant2", - "plan": "some-plan", - "planName": "Plan with id: some-plan", - "collection": "AUTO", - "current": { - "amount": "123.00", - "status": "accrued", - "from": "2020-05-23", - "items": [ - { - "id": "some-id", - "description": "description", - "amount": "123.00", - "plan": "paid", - "planName": "Plan with id: paid", - "majorVersion": 0 - } - ] - }, - "additional": { - "items": [ ] - } - } - ] -} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/billing/responses/invoice-creation-response.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/billing/responses/invoice-creation-response.json deleted file mode 100644 index 49fde010c58..00000000000 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/billing/responses/invoice-creation-response.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "message": "Created invoice with ID id-123", - "id": "id-123" -} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/billing/responses/line-item-list.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/billing/responses/line-item-list.json deleted file mode 100644 index fbfc5ce09ee..00000000000 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/billing/responses/line-item-list.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "lineItems": [ - { - "id": "line-item-id", - "description": "some description", - "amount": "123.45", - "plan": "some-plan", - "planName": "Plan with id: some-plan", - "majorVersion": 0 - } - ] -} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/billing/responses/tenant-billing-view.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/billing/responses/tenant-billing-view.json deleted file mode 100644 index 4e255205e19..00000000000 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/billing/responses/tenant-billing-view.json +++ /dev/null @@ -1,49 +0,0 @@ -{ - "until": "2020-05-28", - "plan": "some-plan", - "planName": "Plan with id: some-plan", - "current": { - "amount": "123.00", - "status": "accrued", - "from": "2020-05-23", - "items": [ - { - "id": "some-id", - "description": "description", - "amount": "123.00", - "plan": "paid", - "planName": "Plan with id: paid", - "majorVersion": 0 - } - ] - }, - "additional": { - "items": [ ] - }, - "bills": [ - { - "id": "id-1", - "from": "2020-05-23", - "to": "2020-05-28", - "amount": "123.00", - "status": "OPEN", - "statusHistory": [ - { - "at": "2020-05-23", - "status": "OPEN" - } - ], - "items": [ - { - "id": "some-id", - "description": "description", - "amount": "123.00", - "plan": "paid", - "planName": "Plan with id: paid", - "majorVersion": 0 - } - ] - } - ], - "collection": "AUTO" -} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/filter/SignatureFilterTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/filter/SignatureFilterTest.java index d37097b8068..9477e71af33 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/filter/SignatureFilterTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/filter/SignatureFilterTest.java @@ -71,7 +71,7 @@ public class SignatureFilterTest { filter = new SignatureFilter(tester.controller()); signer = new RequestSigner(privateKey, id.serializedForm(), tester.clock()); - tester.curator().writeTenant(CloudTenant.create(appId.tenant(), Instant.EPOCH, null)); + tester.curator().writeTenant(CloudTenant.create(appId.tenant(), Instant.EPOCH, new SimplePrincipal("owner@my-tenant.my-app"))); tester.curator().writeApplication(new Application(appId, tester.clock().instant())); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/pricing/PricingApiHandlerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/pricing/PricingApiHandlerTest.java deleted file mode 100644 index 9733ccc4dd8..00000000000 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/pricing/PricingApiHandlerTest.java +++ /dev/null @@ -1,101 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.restapi.pricing; - -import com.yahoo.config.provision.SystemName; -import com.yahoo.vespa.hosted.controller.api.integration.pricing.PricingInfo; -import com.yahoo.vespa.hosted.controller.restapi.ContainerTester; -import com.yahoo.vespa.hosted.controller.restapi.ControllerContainerCloudTest; -import org.junit.jupiter.api.Test; - -import java.net.URLEncoder; - -import static com.yahoo.vespa.hosted.controller.api.integration.pricing.PricingInfo.SupportLevel.BASIC; -import static com.yahoo.vespa.hosted.controller.api.integration.pricing.PricingInfo.SupportLevel.COMMERCIAL; -import static java.nio.charset.StandardCharsets.UTF_8; -import static org.junit.jupiter.api.Assertions.assertEquals; - -/** - * @author hmusum - */ -public class PricingApiHandlerTest extends ControllerContainerCloudTest { - - private static final String responseFiles = "src/test/java/com/yahoo/vespa/hosted/controller/restapi/pricing/responses/"; - - @Test - void testPricingInfoBasic() { - ContainerTester tester = new ContainerTester(container, responseFiles); - assertEquals(SystemName.Public, tester.controller().system()); - - var request = request("/pricing/v1/pricing?" + urlEncodedPriceInformation(BASIC, false)); - tester.assertJsonResponse(request, """ - { - "priceInfo": [ - {"description": "Basic support unit price", "amount": "2240.00"}, - {"description": "Volume discount", "amount": "-5.64"}, - {"description": "Committed spend", "amount": "-1.23"} - ], - "totalAmount": "2233.13" - } - """, - 200); - } - - @Test - void testPricingInfoCommercialEnclave() { - ContainerTester tester = new ContainerTester(container, responseFiles); - assertEquals(SystemName.Public, tester.controller().system()); - - var request = request("/pricing/v1/pricing?" + urlEncodedPriceInformation(COMMERCIAL, true)); - tester.assertJsonResponse(request, """ - { - "priceInfo": [ - {"description": "Commercial support unit price", "amount": "3200.00"}, - {"description": "Enclave discount", "amount": "-15.12"}, - {"description": "Volume discount", "amount": "-5.64"}, - {"description": "Committed spend", "amount": "-1.23"} - ], - "totalAmount": "3178.00" - } - """, - 200); - } - - @Test - void testInvalidRequests() { - ContainerTester tester = new ContainerTester(container, responseFiles); - assertEquals(SystemName.Public, tester.controller().system()); - - tester.assertJsonResponse(request("/pricing/v1/pricing"), - "{\"error-code\":\"BAD_REQUEST\",\"message\":\"No price information found in query\"}", - 400); - tester.assertJsonResponse(request("/pricing/v1/pricing?"), - "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Error in query parameter, expected '=' between key and value: ''\"}", - 400); - tester.assertJsonResponse(request("/pricing/v1/pricing?supportLevel=basic&committedSpend=0&enclave=false"), - "{\"error-code\":\"BAD_REQUEST\",\"message\":\"No cluster resources found in query\"}", - 400); - tester.assertJsonResponse(request("/pricing/v1/pricing?supportLevel=basic&committedSpend=0&enclave=false&resources"), - "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Error in query parameter, expected '=' between key and value: 'resources'\"}", - 400); - tester.assertJsonResponse(request("/pricing/v1/pricing?supportLevel=basic&committedSpend=0&enclave=false&resources="), - "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Error in query parameter, expected '=' between key and value: 'resources='\"}", - 400); - tester.assertJsonResponse(request("/pricing/v1/pricing?supportLevel=basic&committedSpend=0&enclave=false&key=value"), - "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Unknown query parameter 'key'\"}", - 400); - tester.assertJsonResponse(request("/pricing/v1/pricing?supportLevel=basic&committedSpend=0&enclave=false&resources=key%3Dvalue"), - "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Unknown resource type 'key'\"}", - 400); - } - - /** - * 2 clusters, with each having 1 node, with 1 vcpu, 1 Gb memory, 10 Gb disk and no GPU - * price will be 20000 + 2000 + 200 - */ - String urlEncodedPriceInformation(PricingInfo.SupportLevel supportLevel, boolean enclave) { - String resources = URLEncoder.encode("nodes=1,vcpu=1,memoryGb=1,diskGb=10,gpuMemoryGb=0", UTF_8); - return "supportLevel=" + supportLevel.name().toLowerCase() + "&committedSpend=100&enclave=" + enclave + - "&resources=" + resources + - "&resources=" + resources; - } -} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/tenant-info-after-created.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/tenant-info-after-created.json index 6702eff8dde..1926dcc9f82 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/tenant-info-after-created.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/tenant-info-after-created.json @@ -5,5 +5,14 @@ "contactName": "administrator", "contactEmail": "administrator@tenant", "contactEmailVerified": true, - "contacts": [ ] + "contacts": [ + { + "audiences": [ + "tenant", + "notifications" + ], + "email": "administrator@tenant", + "emailVerified": true + } + ] } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java index a10bfd46b0c..a671f567895 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java @@ -46,7 +46,6 @@ import com.yahoo.vespa.hosted.controller.dns.NameServiceQueue; import com.yahoo.vespa.hosted.controller.dns.RemoveRecords; import com.yahoo.vespa.hosted.controller.integration.ZoneApiMock; import com.yahoo.vespa.hosted.rotation.config.RotationsConfig; -import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Timeout; @@ -113,6 +112,11 @@ public class RoutingPoliciesTest { private static final ZoneId zone5 = zoneApi5.getId(); private static final ZoneId zone6 = zoneApi6.getId(); + private static final ZoneId testZonePublic = ZoneId.from("test", "aws-us-east-2c"); + private static final ZoneId stagingZonePublic = ZoneId.from("staging", "aws-us-east-3c"); + private static final ZoneId testZoneMain = ZoneId.from("test", "us-east-1"); + private static final ZoneId stagingZoneMain = ZoneId.from("staging", "us-east-3"); + private static final ApplicationPackage applicationPackage = applicationPackageBuilder().region(zone1.region()) .region(zone2.region()) .build(); @@ -400,20 +404,24 @@ public class RoutingPoliciesTest { } @Test - @Disabled // TODO(mpolden): Enable this test when we start creating generated endpoints for shared routing - void zone_routing_policies_with_shared_routing_and_generated_endpoint() { + void zone_routing_policies_with_shared_routing_and_generated_endpoint_config_and_token() { var tester = new RoutingPoliciesTester(new DeploymentTester(), false); var context = tester.newDeploymentContext("tenant1", "app1", "default"); - tester.provisionLoadBalancers(1, context.instanceId(), true, zone1, zone2); - tester.controllerTester().flagSource().withBooleanFlag(Flags.RANDOMIZED_ENDPOINT_NAMES.id(), true); + tester.setEndpointConfig(EndpointConfig.generated); addCertificateToPool("cafed00d", UnassignedCertificate.State.ready, tester); ApplicationPackage applicationPackage = applicationPackageBuilder().region(zone1.region()) .region(zone2.region()) .container("c0", AuthMethod.mtls, AuthMethod.token) .build(); + tester.provisionLoadBalancers(1, context.instanceId(), true, + testZoneMain, stagingZoneMain, zone1, zone2); context.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.prod).deploy(); - assertEquals(List.of("c0a25b7c.cafed00d.z.vespa.oath.cloud", - "dc5e383c.cafed00d.z.vespa.oath.cloud"), + // This only creates wildcard endpoint names in DNS because legacy names in shared routing-mode use a static + // wildcard DNS record pointing to the routing layer + assertEquals(List.of("a9c8c045.cafed00d.z.vespa.oath.cloud", + "dc5e383c.cafed00d.z.vespa.oath.cloud", + "ebd395b6.cafed00d.z.vespa.oath.cloud", + "ee82b867.cafed00d.z.vespa.oath.cloud"), tester.recordNames()); } @@ -1215,9 +1223,7 @@ public class RoutingPoliciesTest { .container("c0", AuthMethod.mtls) .endpoint("foo", "c0") .build(); - tester.provisionLoadBalancers(1, context.instanceId(), ZoneId.from("test", "aws-us-east-2c")); - tester.provisionLoadBalancers(1, context.instanceId(), ZoneId.from("staging", "aws-us-east-3c")); - tester.provisionLoadBalancers(1, context.instanceId(), zone1); + tester.provisionLoadBalancers(1, context.instanceId(), testZonePublic, stagingZonePublic, zone1); context.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.test, Environment.staging, Environment.prod).deploy(); assertEquals(List.of("a9c8c045.cafed00d.g.vespa-app.cloud", "ebd395b6.cafed00d.z.vespa-app.cloud", @@ -1229,8 +1235,7 @@ public class RoutingPoliciesTest { .container("c0", AuthMethod.mtls, AuthMethod.token) .endpoint("foo", "c0") .build(); - tester.provisionLoadBalancers(1, context.instanceId(), ZoneId.from("test", "aws-us-east-2c")); - tester.provisionLoadBalancers(1, context.instanceId(), ZoneId.from("staging", "aws-us-east-3c")); + tester.provisionLoadBalancers(1, context.instanceId(), testZonePublic, stagingZonePublic); context.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.test, Environment.staging, Environment.prod).deploy(); // Additional zone- and global-scoped endpoints are added (token) assertEquals(List.of("a9c8c045.cafed00d.g.vespa-app.cloud", @@ -1246,8 +1251,7 @@ public class RoutingPoliciesTest { .endpoint("foo", "c0") .endpoint("bar", "c0") .build(); - tester.provisionLoadBalancers(1, context.instanceId(), ZoneId.from("test", "aws-us-east-2c")); - tester.provisionLoadBalancers(1, context.instanceId(), ZoneId.from("staging", "aws-us-east-3c")); + tester.provisionLoadBalancers(1, context.instanceId(), testZonePublic, stagingZonePublic); context.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.test, Environment.staging, Environment.prod).deploy(); List<String> expectedRecords = List.of("a9c8c045.cafed00d.g.vespa-app.cloud", "aa7591aa.cafed00d.g.vespa-app.cloud", @@ -1259,8 +1263,7 @@ public class RoutingPoliciesTest { assertEquals(expectedRecords, tester.recordNames()); // No change on redeployment - tester.provisionLoadBalancers(1, context.instanceId(), ZoneId.from("test", "aws-us-east-2c")); - tester.provisionLoadBalancers(1, context.instanceId(), ZoneId.from("staging", "aws-us-east-3c")); + tester.provisionLoadBalancers(1, context.instanceId(), testZonePublic, stagingZonePublic); context.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.test, Environment.staging, Environment.prod).deploy(); assertEquals(expectedRecords, tester.recordNames()); } @@ -1281,8 +1284,7 @@ public class RoutingPoliciesTest { tester.provisionLoadBalancers(1, context.instanceId(), zone1); // ConfigServerMock provisions a load balancer for the "default" cluster, but in this scenario we need full // control over the load balancer name because "default" has no special treatment when using generated endpoints - tester.provisionLoadBalancers(1, context.instanceId(), ZoneId.from("test", "aws-us-east-2c")); - tester.provisionLoadBalancers(1, context.instanceId(), ZoneId.from("staging", "aws-us-east-3c")); + tester.provisionLoadBalancers(1, context.instanceId(), testZonePublic, stagingZonePublic); context.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.test, Environment.staging, Environment.prod).deploy(); tester.assertTargets(context.instance().id(), EndpointId.of("foo"), ClusterSpec.Id.from("c0"), 0, Map.of(zone1, 1L), true); @@ -1297,8 +1299,7 @@ public class RoutingPoliciesTest { .container("c0", AuthMethod.mtls) .endpoint("foo", "c0") .build(); - tester.provisionLoadBalancers(1, context.instanceId(), ZoneId.from("test", "aws-us-east-2c")); - tester.provisionLoadBalancers(1, context.instanceId(), ZoneId.from("staging", "aws-us-east-3c")); + tester.provisionLoadBalancers(1, context.instanceId(), testZonePublic, stagingZonePublic); tester.provisionLoadBalancers(1, context.instanceId(), zone2); context.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.test, Environment.staging, Environment.prod).deploy(); assertEquals(List.of("a6414896.cafed00d.aws-eu-west-1.w.vespa-app.cloud", @@ -1516,8 +1517,7 @@ public class RoutingPoliciesTest { } public RoutingPoliciesTester setEndpointConfig(EndpointConfig config) { - tester.controllerTester().flagSource().withBooleanFlag(Flags.LEGACY_ENDPOINTS.id(), config.supportsLegacy()); - tester.controllerTester().flagSource().withBooleanFlag(Flags.RANDOMIZED_ENDPOINT_NAMES.id(), config.supportsGenerated()); + tester.controllerTester().flagSource().withStringFlag(Flags.ENDPOINT_CONFIG.id(), config.name()); return this; } diff --git a/controller-server/src/main/resources/mail/mail-notification.tmpl b/controller-server/src/test/resources/mail/notification.html index 5bf5530b433..2a0edeea7e1 100644 --- a/controller-server/src/main/resources/mail/mail-notification.tmpl +++ b/controller-server/src/test/resources/mail/notification.html @@ -383,11 +383,12 @@ style="vertical-align: top" width="100%" > - <tbody> - <tr> - <td - align="left" - style=" + +<tbody> +<tr> + <td + align="left" + style=" font-size: 0px; padding: 0px 25px 0px 25px; padding-top: 0px; @@ -396,9 +397,9 @@ padding-left: 50px; word-break: break-word; " - > - <div - style=" + > + <div + style=" font-family: Open Sans, Helvetica, Arial, sans-serif; font-size: 13px; @@ -406,23 +407,23 @@ text-align: left; color: #797e82; " - > - <h1 - style=" + > + <h1 + style=" text-align: center; color: #000000; line-height: 32px; " - > - Vespa Cloud Notifications - </h1> - </div> - </td> - </tr> - <tr> - <td - align="left" - style=" + > + Vespa Cloud Notifications + </h1> + </div> + </td> +</tr> +<tr> + <td + align="left" + style=" font-size: 0px; padding: 0px 25px 0px 25px; padding-top: 0px; @@ -431,9 +432,9 @@ padding-left: 50px; word-break: break-word; " - > - <div - style=" + > + <div + style=" font-family: Open Sans, Helvetica, Arial, sans-serif; font-size: 13px; @@ -441,51 +442,54 @@ text-align: left; color: #797e82; " - > - <p> - [[NOTIFICATION_HEADER]]: - </p> - [[NOTIFICATION_ITEMS]] - </div> - </td> - </tr> - <tr> - <td - align="center" - vertical-align="middle" - style=" + > + +<p> + There are problems with tests for default.default: +</p> +<p>Test package has production tests, but no production tests are declared in deployment.xml</p> +<p>See <a href="https://docs.vespa.ai/en/testing.html">https://docs.vespa.ai/en/testing.html</a> for details on how to write system tests for Vespa</p> + + </div> + </td> +</tr> +<tr> + <td + align="center" + vertical-align="middle" + style=" font-size: 0px; padding: 10px 25px; padding-top: 20px; padding-bottom: 20px; word-break: break-word; " - > - <table - border="0" - cellpadding="0" - cellspacing="0" - role="presentation" - style="border-collapse: separate; line-height: 100%" - > - <tbody> - <tr> - <td - align="center" - bgcolor="#005A8E" - role="presentation" - style=" + > + <table + border="0" + cellpadding="0" + cellspacing="0" + role="presentation" + style="border-collapse: separate; line-height: 100%" + > + <tbody> + <tr> + <td + align="center" + bgcolor="#005A8E" + role="presentation" + style=" border: none; border-radius: 100px; cursor: auto; mso-padding-alt: 15px 25px 15px 25px; background: #005a8e; " - valign="middle" - > - <a - href="[[LINK_TO_NOTIFICATION]]" - style=" + valign="middle" + > + <a + href="https://console.tld/tenant/tenant1/application/default/prod/instance/default" + style=" display: inline-block; background: #005a8e; color: #ffffff; @@ -501,20 +505,20 @@ mso-padding-alt: 0px; border-radius: 100px; " - target="_blank" - ><b style="font-weight: 700" - ><b style="font-weight: 700" - >Go to Console</b - ></b - ></a - > - </td> - </tr> - </tbody> - </table> - </td> - </tr> - </tbody> + target="_blank" + ><b style="font-weight: 700" + ><b style="font-weight: 700" + >Go to Console</b + ></b + ></a + > + </td> + </tr> + </tbody> + </table> + </td> +</tr> +</tbody> </table> </div> <!--[if mso | IE]></td></tr></table><![endif]--> @@ -592,7 +596,7 @@ target="_blank" rel="noopener noreferrer" style="color: #005a8e" - href="[[LINK_TO_PRIVACY_POLICY]]" + href="https://legal.yahoo.com/xw/en/yahoo/privacy/topic/b2bprivacypolicy/index.html" ><span style="color: #005a8e" >Yahoo Privacy Policy</span ></a @@ -602,7 +606,7 @@ target="_blank" rel="noopener noreferrer" style="color: #005a8e" - href="[[LINK_TO_TERMS_OF_SERVICE]]" + href="https://console.tld/terms-of-service-trial.html" ><span style="color: #005a8e" >Terms of Service</span ></a @@ -612,7 +616,7 @@ target="_blank" rel="noopener noreferrer" style="color: #005a8e" - href="[[LINK_TO_SUPPORT]]" + href="https://console.tld/support" ><span style="color: #005a8e">Support</span></a > </p> @@ -621,7 +625,7 @@ target="_blank" rel="noopener noreferrer" style="color: inherit; text-decoration: none" - href="[[LINK_TO_ACCOUNT_NOTIFICATIONS]]" + href="https://console.tld/tenant/tenant1/account/notifications" >Click <span style="color: #005a8e"><u>here</u></span> to manage your notifications setting.</a diff --git a/controller-server/src/test/resources/mail/notification.txt b/controller-server/src/test/resources/mail/trial-expired.html index 35db37fbc12..bdeafe8c7d3 100644 --- a/controller-server/src/test/resources/mail/notification.txt +++ b/controller-server/src/test/resources/mail/trial-expired.html @@ -383,11 +383,12 @@ style="vertical-align: top" width="100%" > - <tbody> - <tr> - <td - align="left" - style=" + +<tbody> +<tr> + <td + align="left" + style=" font-size: 0px; padding: 0px 25px 0px 25px; padding-top: 0px; @@ -396,9 +397,9 @@ padding-left: 50px; word-break: break-word; " - > - <div - style=" + > + <div + style=" font-family: Open Sans, Helvetica, Arial, sans-serif; font-size: 13px; @@ -406,23 +407,23 @@ text-align: left; color: #797e82; " - > - <h1 - style=" + > + <h1 + style=" text-align: center; color: #000000; line-height: 32px; " - > - Vespa Cloud Notifications - </h1> - </div> - </td> - </tr> - <tr> - <td - align="left" - style=" + > + Your Vespa Cloud trial has expired + </h1> + </div> + </td> +</tr> +<tr> + <td + align="left" + style=" font-size: 0px; padding: 0px 25px 0px 25px; padding-top: 0px; @@ -431,9 +432,9 @@ padding-left: 50px; word-break: break-word; " - > - <div - style=" + > + <div + style=" font-family: Open Sans, Helvetica, Arial, sans-serif; font-size: 13px; @@ -441,51 +442,51 @@ text-align: left; color: #797e82; " - > - <p> - There are problems with tests for default.default: - </p> - <p>Test package has production tests, but no production tests are declared in deployment.xml</p><p>See <a href="https://docs.vespa.ai/en/testing.html">https://docs.vespa.ai/en/testing.html</a> for details on how to write system tests for Vespa</p> - </div> - </td> - </tr> - <tr> - <td - align="center" - vertical-align="middle" - style=" + > + +<p> + Your Vespa Cloud trial has expired. Please reach out to us if you have any questions or feedback. +</p> + </div> + </td> +</tr> +<tr> + <td + align="center" + vertical-align="middle" + style=" font-size: 0px; padding: 10px 25px; padding-top: 20px; padding-bottom: 20px; word-break: break-word; " - > - <table - border="0" - cellpadding="0" - cellspacing="0" - role="presentation" - style="border-collapse: separate; line-height: 100%" - > - <tbody> - <tr> - <td - align="center" - bgcolor="#005A8E" - role="presentation" - style=" + > + <table + border="0" + cellpadding="0" + cellspacing="0" + role="presentation" + style="border-collapse: separate; line-height: 100%" + > + <tbody> + <tr> + <td + align="center" + bgcolor="#005A8E" + role="presentation" + style=" border: none; border-radius: 100px; cursor: auto; mso-padding-alt: 15px 25px 15px 25px; background: #005a8e; " - valign="middle" - > - <a - href="https://dashboard.tld/tenant/tenant1/application/default/prod/instance" - style=" + valign="middle" + > + <a + href="https://console.tld/tenant/trial-tenant" + style=" display: inline-block; background: #005a8e; color: #ffffff; @@ -501,20 +502,20 @@ mso-padding-alt: 0px; border-radius: 100px; " - target="_blank" - ><b style="font-weight: 700" - ><b style="font-weight: 700" - >Go to Console</b - ></b - ></a - > - </td> - </tr> - </tbody> - </table> - </td> - </tr> - </tbody> + target="_blank" + ><b style="font-weight: 700" + ><b style="font-weight: 700" + >Go to Console</b + ></b + ></a + > + </td> + </tr> + </tbody> + </table> + </td> +</tr> +</tbody> </table> </div> <!--[if mso | IE]></td></tr></table><![endif]--> @@ -602,7 +603,7 @@ target="_blank" rel="noopener noreferrer" style="color: #005a8e" - href="https://dashboard.tld/terms-of-service-trial.html" + href="https://console.tld/terms-of-service-trial.html" ><span style="color: #005a8e" >Terms of Service</span ></a @@ -612,7 +613,7 @@ target="_blank" rel="noopener noreferrer" style="color: #005a8e" - href="https://dashboard.tld/support" + href="https://console.tld/support" ><span style="color: #005a8e">Support</span></a > </p> @@ -621,7 +622,7 @@ target="_blank" rel="noopener noreferrer" style="color: inherit; text-decoration: none" - href="https://dashboard.tld/tenant/tenant1/account/notifications" + href="https://console.tld/tenant/trial-tenant/account/notifications" >Click <span style="color: #005a8e"><u>here</u></span> to manage your notifications setting.</a diff --git a/controller-server/src/test/resources/mail/trial-expiring-immediately.html b/controller-server/src/test/resources/mail/trial-expiring-immediately.html new file mode 100644 index 00000000000..db89eca195a --- /dev/null +++ b/controller-server/src/test/resources/mail/trial-expiring-immediately.html @@ -0,0 +1,646 @@ +<!DOCTYPE html> +<html + xmlns="http://www.w3.org/1999/xhtml" + xmlns:v="urn:schemas-microsoft-com:vml" + xmlns:o="urn:schemas-microsoft-com:office:office" +> + <head> + <title></title> + <!--[if !mso]><!--> + <meta http-equiv="X-UA-Compatible" content="IE=edge" /> + <!--<![endif]--> + <meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /> + <meta name="viewport" content="width=device-width,initial-scale=1" /> + <style type="text/css"> + #outlook a { + padding: 0; + } + + body { + margin: 0; + padding: 0; + -webkit-text-size-adjust: 100%; + -ms-text-size-adjust: 100%; + } + + table, + td { + border-collapse: collapse; + mso-table-lspace: 0pt; + mso-table-rspace: 0pt; + } + + img { + border: 0; + height: auto; + line-height: 100%; + outline: none; + text-decoration: none; + -ms-interpolation-mode: bicubic; + } + + p { + display: block; + margin: 13px 0; + } + </style> + <!--[if mso]> + <noscript> + <xml> + <o:OfficeDocumentSettings> + <o:AllowPNG /> + <o:PixelsPerInch>96</o:PixelsPerInch> + </o:OfficeDocumentSettings> + </xml> + </noscript> + <![endif]--> + <!--[if lte mso 11]> + <style type="text/css"> + .mj-outlook-group-fix { + width: 100% !important; + } + </style> + <![endif]--> + <!--[if !mso]><!--> + <link + href="https://fonts.googleapis.com/css?family=Open Sans" + rel="stylesheet" + type="text/css" + /> + <style type="text/css"> + @import url(https://fonts.googleapis.com/css?family=Open Sans); + </style> + <!--<![endif]--> + <style type="text/css"> + @media only screen and (min-width: 480px) { + .mj-column-per-100 { + width: 100% !important; + max-width: 100%; + } + } + </style> + <style media="screen and (min-width:480px)"> + .moz-text-html .mj-column-per-100 { + width: 100% !important; + max-width: 100%; + } + </style> + <style type="text/css"> + [owa] .mj-column-per-100 { + width: 100% !important; + max-width: 100%; + } + </style> + <style type="text/css"> + @media only screen and (max-width: 480px) { + table.mj-full-width-mobile { + width: 100% !important; + } + + td.mj-full-width-mobile { + width: auto !important; + } + } + </style> + </head> + + <body style="word-spacing: normal; background-color: #f2f7fa"> + <div style="background-color: #f2f7fa"> + <!--[if mso | IE]><table align="center" border="0" cellpadding="0" cellspacing="0" class="" role="presentation" style="width:600px;" width="600" ><tr><td style="line-height:0px;font-size:0px;mso-line-height-rule:exactly;"><![endif]--> + <div style="margin: 0px auto; max-width: 600px"> + <table + align="center" + border="0" + cellpadding="0" + cellspacing="0" + role="presentation" + style="width: 100%" + > + <tbody> + <tr> + <td + style=" + direction: ltr; + font-size: 0px; + padding: 20px 0px 20px 0px; + padding-bottom: 0px; + padding-top: 0px; + text-align: center; + " + > + <!--[if mso | IE]><table role="presentation" border="0" cellpadding="0" cellspacing="0"><tr><td class="" style="vertical-align:top;width:600px;" ><![endif]--> + <div + class="mj-column-per-100 mj-outlook-group-fix" + style=" + font-size: 0px; + text-align: left; + direction: ltr; + display: inline-block; + vertical-align: top; + width: 100%; + " + > + <table + border="0" + cellpadding="0" + cellspacing="0" + role="presentation" + style="vertical-align: top" + width="100%" + > + <tbody> + <tr> + <td + align="left" + style=" + font-size: 0px; + padding: 0px 0px 0px 25px; + padding-top: 0px; + padding-bottom: 0px; + word-break: break-word; + " + > + <div + style=" + font-family: Open Sans, Helvetica, Arial, + sans-serif; + font-size: 11px; + line-height: 22px; + text-align: left; + color: #797e82; + " + > + <br /> + </div> + </td> + </tr> + </tbody> + </table> + </div> + <!--[if mso | IE]></td></tr></table><![endif]--> + </td> + </tr> + </tbody> + </table> + </div> + <!--[if mso | IE]></td></tr></table><table align="center" border="0" cellpadding="0" cellspacing="0" class="" role="presentation" style="width:600px;" width="600" bgcolor="#ffffff" ><tr><td style="line-height:0px;font-size:0px;mso-line-height-rule:exactly;"><![endif]--> + <div + style=" + background: #ffffff; + background-color: #ffffff; + margin: 0px auto; + max-width: 600px; + " + > + <table + align="center" + border="0" + cellpadding="0" + cellspacing="0" + role="presentation" + style="background: #ffffff; background-color: #ffffff; width: 100%" + > + <tbody> + <tr> + <td + style=" + direction: ltr; + font-size: 0px; + padding: 20px 0; + padding-bottom: 0px; + padding-left: 0px; + padding-right: 0px; + padding-top: 0px; + text-align: center; + " + > + <!--[if mso | IE]><table role="presentation" border="0" cellpadding="0" cellspacing="0"><tr><td class="" style="vertical-align:top;width:600px;" ><![endif]--> + <div + class="mj-column-per-100 mj-outlook-group-fix" + style=" + font-size: 0px; + text-align: left; + direction: ltr; + display: inline-block; + vertical-align: top; + width: 100%; + " + > + <table + border="0" + cellpadding="0" + cellspacing="0" + role="presentation" + style="vertical-align: top" + width="100%" + > + <tbody> + <tr> + <td + align="center" + style=" + font-size: 0px; + padding: 10px 25px; + padding-top: 0px; + padding-right: 0px; + padding-bottom: 40px; + padding-left: 0px; + word-break: break-word; + " + > + <p + style=" + border-top: solid 8px #005a8e; + font-size: 1px; + margin: 0px auto; + width: 100%; + " + ></p> + <!--[if mso | IE + ]><table + align="center" + border="0" + cellpadding="0" + cellspacing="0" + style=" + border-top: solid 8px #005a8e; + font-size: 1px; + margin: 0px auto; + width: 600px; + " + role="presentation" + width="600px" + > + <tr> + <td style="height: 0; line-height: 0"> + + </td> + </tr> + </table><! + [endif]--> + </td> + </tr> + <tr> + <td + align="center" + style=" + font-size: 0px; + padding: 10px 25px; + padding-top: 0px; + padding-bottom: 0px; + word-break: break-word; + " + > + <table + border="0" + cellpadding="0" + cellspacing="0" + role="presentation" + style=" + border-collapse: collapse; + border-spacing: 0px; + " + > + <tbody> + <tr> + <td style="width: 121px"> + <img + alt="" + height="auto" + src="https://data.vespa.oath.cloud/assets/vespa-cloud-logo.png" + style=" + border: none; + display: block; + outline: none; + text-decoration: none; + height: auto; + width: 100%; + font-size: 13px; + " + width="121" + /> + </td> + </tr> + </tbody> + </table> + </td> + </tr> + </tbody> + </table> + </div> + <!--[if mso | IE]></td></tr></table><![endif]--> + </td> + </tr> + </tbody> + </table> + </div> + <!--[if mso | IE]></td></tr></table><table align="center" border="0" cellpadding="0" cellspacing="0" class="" role="presentation" style="width:600px;" width="600" bgcolor="#ffffff" ><tr><td style="line-height:0px;font-size:0px;mso-line-height-rule:exactly;"><![endif]--> + <div + style=" + background: #ffffff; + background-color: #ffffff; + margin: 0px auto; + max-width: 600px; + " + > + <table + align="center" + border="0" + cellpadding="0" + cellspacing="0" + role="presentation" + style="background: #ffffff; background-color: #ffffff; width: 100%" + > + <tbody> + <tr> + <td + style=" + direction: ltr; + font-size: 0px; + padding: 20px 0px 20px 0px; + padding-bottom: 70px; + padding-top: 30px; + text-align: center; + " + > + <!--[if mso | IE]><table role="presentation" border="0" cellpadding="0" cellspacing="0"><tr><td class="" style="vertical-align:top;width:600px;" ><![endif]--> + <div + class="mj-column-per-100 mj-outlook-group-fix" + style=" + font-size: 0px; + text-align: left; + direction: ltr; + display: inline-block; + vertical-align: top; + width: 100%; + " + > + <table + border="0" + cellpadding="0" + cellspacing="0" + role="presentation" + style="vertical-align: top" + width="100%" + > + +<tbody> +<tr> + <td + align="left" + style=" + font-size: 0px; + padding: 0px 25px 0px 25px; + padding-top: 0px; + padding-right: 50px; + padding-bottom: 0px; + padding-left: 50px; + word-break: break-word; + " + > + <div + style=" + font-family: Open Sans, Helvetica, Arial, + sans-serif; + font-size: 13px; + line-height: 22px; + text-align: left; + color: #797e82; + " + > + <h1 + style=" + text-align: center; + color: #000000; + line-height: 32px; + " + > + Your Vespa Cloud trial expires tomorrow + </h1> + </div> + </td> +</tr> +<tr> + <td + align="left" + style=" + font-size: 0px; + padding: 0px 25px 0px 25px; + padding-top: 0px; + padding-right: 50px; + padding-bottom: 0px; + padding-left: 50px; + word-break: break-word; + " + > + <div + style=" + font-family: Open Sans, Helvetica, Arial, + sans-serif; + font-size: 13px; + line-height: 22px; + text-align: left; + color: #797e82; + " + > + +<p> + Your Vespa Cloud trial expires tomorrow. Please reach out to us if you have any questions or feedback. +</p> + </div> + </td> +</tr> +<tr> + <td + align="center" + vertical-align="middle" + style=" + font-size: 0px; + padding: 10px 25px; + padding-top: 20px; + padding-bottom: 20px; + word-break: break-word; + " + > + <table + border="0" + cellpadding="0" + cellspacing="0" + role="presentation" + style="border-collapse: separate; line-height: 100%" + > + <tbody> + <tr> + <td + align="center" + bgcolor="#005A8E" + role="presentation" + style=" + border: none; + border-radius: 100px; + cursor: auto; + mso-padding-alt: 15px 25px 15px 25px; + background: #005a8e; + " + valign="middle" + > + <a + href="https://console.tld/tenant/trial-tenant" + style=" + display: inline-block; + background: #005a8e; + color: #ffffff; + font-family: Open Sans, Helvetica, Arial, + sans-serif; + font-size: 13px; + font-weight: normal; + line-height: 120%; + margin: 0; + text-decoration: none; + text-transform: none; + padding: 15px 25px 15px 25px; + mso-padding-alt: 0px; + border-radius: 100px; + " + target="_blank" + ><b style="font-weight: 700" + ><b style="font-weight: 700" + >Go to Console</b + ></b + ></a + > + </td> + </tr> + </tbody> + </table> + </td> +</tr> +</tbody> + </table> + </div> + <!--[if mso | IE]></td></tr></table><![endif]--> + </td> + </tr> + </tbody> + </table> + </div> + <!--[if mso | IE]></td></tr></table><table align="center" border="0" cellpadding="0" cellspacing="0" class="" role="presentation" style="width:600px;" width="600" ><tr><td style="line-height:0px;font-size:0px;mso-line-height-rule:exactly;"><![endif]--> + <div style="margin: 0px auto; max-width: 600px"> + <table + align="center" + border="0" + cellpadding="0" + cellspacing="0" + role="presentation" + style="width: 100%" + > + <tbody> + <tr> + <td + style=" + direction: ltr; + font-size: 0px; + padding: 20px 0px 20px 0px; + padding-bottom: 0px; + padding-top: 20px; + text-align: center; + " + > + <!--[if mso | IE]><table role="presentation" border="0" cellpadding="0" cellspacing="0"><tr><td class="" style="vertical-align:top;width:600px;" ><![endif]--> + <div + class="mj-column-per-100 mj-outlook-group-fix" + style=" + font-size: 0px; + text-align: left; + direction: ltr; + display: inline-block; + vertical-align: top; + width: 100%; + " + > + <table + border="0" + cellpadding="0" + cellspacing="0" + role="presentation" + style="vertical-align: top" + width="100%" + > + <tbody> + <tr> + <td + align="center" + style=" + font-size: 0px; + padding: 0px 20px 0px 20px; + padding-top: 0px; + padding-bottom: 0px; + word-break: break-word; + " + > + <div + style=" + font-family: Open Sans, Helvetica, Arial, + sans-serif; + font-size: 11px; + line-height: 22px; + text-align: center; + color: #797e82; + " + > + <p style="margin: 10px 0"> + <a + target="_blank" + rel="noopener noreferrer" + style="color: #005a8e" + href="https://legal.yahoo.com/xw/en/yahoo/privacy/topic/b2bprivacypolicy/index.html" + ><span style="color: #005a8e" + >Yahoo Privacy Policy</span + ></a + ><span style="color: #797e82" + > | </span + ><a + target="_blank" + rel="noopener noreferrer" + style="color: #005a8e" + href="https://console.tld/terms-of-service-trial.html" + ><span style="color: #005a8e" + >Terms of Service</span + ></a + ><span style="color: #797e82" + > | </span + ><a + target="_blank" + rel="noopener noreferrer" + style="color: #005a8e" + href="https://console.tld/support" + ><span style="color: #005a8e">Support</span></a + > + </p> + <p style="margin: 10px 0"> + <a + target="_blank" + rel="noopener noreferrer" + style="color: inherit; text-decoration: none" + href="https://console.tld/tenant/trial-tenant/account/notifications" + >Click + <span style="color: #005a8e"><u>here</u></span> + to manage your notifications setting.</a + ><br /> + </p> + </div> + </td> + </tr> + </tbody> + </table> + </div> + <!--[if mso | IE]></td></tr></table><![endif]--> + </td> + </tr> + </tbody> + </table> + </div> + <!--[if mso | IE]></td></tr></table><![endif]--> + </div> + </body> +</html> diff --git a/controller-server/src/test/resources/mail/trial-expiring-soon.html b/controller-server/src/test/resources/mail/trial-expiring-soon.html new file mode 100644 index 00000000000..17c59240cc4 --- /dev/null +++ b/controller-server/src/test/resources/mail/trial-expiring-soon.html @@ -0,0 +1,646 @@ +<!DOCTYPE html> +<html + xmlns="http://www.w3.org/1999/xhtml" + xmlns:v="urn:schemas-microsoft-com:vml" + xmlns:o="urn:schemas-microsoft-com:office:office" +> + <head> + <title></title> + <!--[if !mso]><!--> + <meta http-equiv="X-UA-Compatible" content="IE=edge" /> + <!--<![endif]--> + <meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /> + <meta name="viewport" content="width=device-width,initial-scale=1" /> + <style type="text/css"> + #outlook a { + padding: 0; + } + + body { + margin: 0; + padding: 0; + -webkit-text-size-adjust: 100%; + -ms-text-size-adjust: 100%; + } + + table, + td { + border-collapse: collapse; + mso-table-lspace: 0pt; + mso-table-rspace: 0pt; + } + + img { + border: 0; + height: auto; + line-height: 100%; + outline: none; + text-decoration: none; + -ms-interpolation-mode: bicubic; + } + + p { + display: block; + margin: 13px 0; + } + </style> + <!--[if mso]> + <noscript> + <xml> + <o:OfficeDocumentSettings> + <o:AllowPNG /> + <o:PixelsPerInch>96</o:PixelsPerInch> + </o:OfficeDocumentSettings> + </xml> + </noscript> + <![endif]--> + <!--[if lte mso 11]> + <style type="text/css"> + .mj-outlook-group-fix { + width: 100% !important; + } + </style> + <![endif]--> + <!--[if !mso]><!--> + <link + href="https://fonts.googleapis.com/css?family=Open Sans" + rel="stylesheet" + type="text/css" + /> + <style type="text/css"> + @import url(https://fonts.googleapis.com/css?family=Open Sans); + </style> + <!--<![endif]--> + <style type="text/css"> + @media only screen and (min-width: 480px) { + .mj-column-per-100 { + width: 100% !important; + max-width: 100%; + } + } + </style> + <style media="screen and (min-width:480px)"> + .moz-text-html .mj-column-per-100 { + width: 100% !important; + max-width: 100%; + } + </style> + <style type="text/css"> + [owa] .mj-column-per-100 { + width: 100% !important; + max-width: 100%; + } + </style> + <style type="text/css"> + @media only screen and (max-width: 480px) { + table.mj-full-width-mobile { + width: 100% !important; + } + + td.mj-full-width-mobile { + width: auto !important; + } + } + </style> + </head> + + <body style="word-spacing: normal; background-color: #f2f7fa"> + <div style="background-color: #f2f7fa"> + <!--[if mso | IE]><table align="center" border="0" cellpadding="0" cellspacing="0" class="" role="presentation" style="width:600px;" width="600" ><tr><td style="line-height:0px;font-size:0px;mso-line-height-rule:exactly;"><![endif]--> + <div style="margin: 0px auto; max-width: 600px"> + <table + align="center" + border="0" + cellpadding="0" + cellspacing="0" + role="presentation" + style="width: 100%" + > + <tbody> + <tr> + <td + style=" + direction: ltr; + font-size: 0px; + padding: 20px 0px 20px 0px; + padding-bottom: 0px; + padding-top: 0px; + text-align: center; + " + > + <!--[if mso | IE]><table role="presentation" border="0" cellpadding="0" cellspacing="0"><tr><td class="" style="vertical-align:top;width:600px;" ><![endif]--> + <div + class="mj-column-per-100 mj-outlook-group-fix" + style=" + font-size: 0px; + text-align: left; + direction: ltr; + display: inline-block; + vertical-align: top; + width: 100%; + " + > + <table + border="0" + cellpadding="0" + cellspacing="0" + role="presentation" + style="vertical-align: top" + width="100%" + > + <tbody> + <tr> + <td + align="left" + style=" + font-size: 0px; + padding: 0px 0px 0px 25px; + padding-top: 0px; + padding-bottom: 0px; + word-break: break-word; + " + > + <div + style=" + font-family: Open Sans, Helvetica, Arial, + sans-serif; + font-size: 11px; + line-height: 22px; + text-align: left; + color: #797e82; + " + > + <br /> + </div> + </td> + </tr> + </tbody> + </table> + </div> + <!--[if mso | IE]></td></tr></table><![endif]--> + </td> + </tr> + </tbody> + </table> + </div> + <!--[if mso | IE]></td></tr></table><table align="center" border="0" cellpadding="0" cellspacing="0" class="" role="presentation" style="width:600px;" width="600" bgcolor="#ffffff" ><tr><td style="line-height:0px;font-size:0px;mso-line-height-rule:exactly;"><![endif]--> + <div + style=" + background: #ffffff; + background-color: #ffffff; + margin: 0px auto; + max-width: 600px; + " + > + <table + align="center" + border="0" + cellpadding="0" + cellspacing="0" + role="presentation" + style="background: #ffffff; background-color: #ffffff; width: 100%" + > + <tbody> + <tr> + <td + style=" + direction: ltr; + font-size: 0px; + padding: 20px 0; + padding-bottom: 0px; + padding-left: 0px; + padding-right: 0px; + padding-top: 0px; + text-align: center; + " + > + <!--[if mso | IE]><table role="presentation" border="0" cellpadding="0" cellspacing="0"><tr><td class="" style="vertical-align:top;width:600px;" ><![endif]--> + <div + class="mj-column-per-100 mj-outlook-group-fix" + style=" + font-size: 0px; + text-align: left; + direction: ltr; + display: inline-block; + vertical-align: top; + width: 100%; + " + > + <table + border="0" + cellpadding="0" + cellspacing="0" + role="presentation" + style="vertical-align: top" + width="100%" + > + <tbody> + <tr> + <td + align="center" + style=" + font-size: 0px; + padding: 10px 25px; + padding-top: 0px; + padding-right: 0px; + padding-bottom: 40px; + padding-left: 0px; + word-break: break-word; + " + > + <p + style=" + border-top: solid 8px #005a8e; + font-size: 1px; + margin: 0px auto; + width: 100%; + " + ></p> + <!--[if mso | IE + ]><table + align="center" + border="0" + cellpadding="0" + cellspacing="0" + style=" + border-top: solid 8px #005a8e; + font-size: 1px; + margin: 0px auto; + width: 600px; + " + role="presentation" + width="600px" + > + <tr> + <td style="height: 0; line-height: 0"> + + </td> + </tr> + </table><! + [endif]--> + </td> + </tr> + <tr> + <td + align="center" + style=" + font-size: 0px; + padding: 10px 25px; + padding-top: 0px; + padding-bottom: 0px; + word-break: break-word; + " + > + <table + border="0" + cellpadding="0" + cellspacing="0" + role="presentation" + style=" + border-collapse: collapse; + border-spacing: 0px; + " + > + <tbody> + <tr> + <td style="width: 121px"> + <img + alt="" + height="auto" + src="https://data.vespa.oath.cloud/assets/vespa-cloud-logo.png" + style=" + border: none; + display: block; + outline: none; + text-decoration: none; + height: auto; + width: 100%; + font-size: 13px; + " + width="121" + /> + </td> + </tr> + </tbody> + </table> + </td> + </tr> + </tbody> + </table> + </div> + <!--[if mso | IE]></td></tr></table><![endif]--> + </td> + </tr> + </tbody> + </table> + </div> + <!--[if mso | IE]></td></tr></table><table align="center" border="0" cellpadding="0" cellspacing="0" class="" role="presentation" style="width:600px;" width="600" bgcolor="#ffffff" ><tr><td style="line-height:0px;font-size:0px;mso-line-height-rule:exactly;"><![endif]--> + <div + style=" + background: #ffffff; + background-color: #ffffff; + margin: 0px auto; + max-width: 600px; + " + > + <table + align="center" + border="0" + cellpadding="0" + cellspacing="0" + role="presentation" + style="background: #ffffff; background-color: #ffffff; width: 100%" + > + <tbody> + <tr> + <td + style=" + direction: ltr; + font-size: 0px; + padding: 20px 0px 20px 0px; + padding-bottom: 70px; + padding-top: 30px; + text-align: center; + " + > + <!--[if mso | IE]><table role="presentation" border="0" cellpadding="0" cellspacing="0"><tr><td class="" style="vertical-align:top;width:600px;" ><![endif]--> + <div + class="mj-column-per-100 mj-outlook-group-fix" + style=" + font-size: 0px; + text-align: left; + direction: ltr; + display: inline-block; + vertical-align: top; + width: 100%; + " + > + <table + border="0" + cellpadding="0" + cellspacing="0" + role="presentation" + style="vertical-align: top" + width="100%" + > + +<tbody> +<tr> + <td + align="left" + style=" + font-size: 0px; + padding: 0px 25px 0px 25px; + padding-top: 0px; + padding-right: 50px; + padding-bottom: 0px; + padding-left: 50px; + word-break: break-word; + " + > + <div + style=" + font-family: Open Sans, Helvetica, Arial, + sans-serif; + font-size: 13px; + line-height: 22px; + text-align: left; + color: #797e82; + " + > + <h1 + style=" + text-align: center; + color: #000000; + line-height: 32px; + " + > + Your Vespa Cloud trial expires in 2 days + </h1> + </div> + </td> +</tr> +<tr> + <td + align="left" + style=" + font-size: 0px; + padding: 0px 25px 0px 25px; + padding-top: 0px; + padding-right: 50px; + padding-bottom: 0px; + padding-left: 50px; + word-break: break-word; + " + > + <div + style=" + font-family: Open Sans, Helvetica, Arial, + sans-serif; + font-size: 13px; + line-height: 22px; + text-align: left; + color: #797e82; + " + > + +<p> + Your Vespa Cloud trial expires in 2 days. Please reach out to us if you have any questions or feedback. +</p> + </div> + </td> +</tr> +<tr> + <td + align="center" + vertical-align="middle" + style=" + font-size: 0px; + padding: 10px 25px; + padding-top: 20px; + padding-bottom: 20px; + word-break: break-word; + " + > + <table + border="0" + cellpadding="0" + cellspacing="0" + role="presentation" + style="border-collapse: separate; line-height: 100%" + > + <tbody> + <tr> + <td + align="center" + bgcolor="#005A8E" + role="presentation" + style=" + border: none; + border-radius: 100px; + cursor: auto; + mso-padding-alt: 15px 25px 15px 25px; + background: #005a8e; + " + valign="middle" + > + <a + href="https://console.tld/tenant/trial-tenant" + style=" + display: inline-block; + background: #005a8e; + color: #ffffff; + font-family: Open Sans, Helvetica, Arial, + sans-serif; + font-size: 13px; + font-weight: normal; + line-height: 120%; + margin: 0; + text-decoration: none; + text-transform: none; + padding: 15px 25px 15px 25px; + mso-padding-alt: 0px; + border-radius: 100px; + " + target="_blank" + ><b style="font-weight: 700" + ><b style="font-weight: 700" + >Go to Console</b + ></b + ></a + > + </td> + </tr> + </tbody> + </table> + </td> +</tr> +</tbody> + </table> + </div> + <!--[if mso | IE]></td></tr></table><![endif]--> + </td> + </tr> + </tbody> + </table> + </div> + <!--[if mso | IE]></td></tr></table><table align="center" border="0" cellpadding="0" cellspacing="0" class="" role="presentation" style="width:600px;" width="600" ><tr><td style="line-height:0px;font-size:0px;mso-line-height-rule:exactly;"><![endif]--> + <div style="margin: 0px auto; max-width: 600px"> + <table + align="center" + border="0" + cellpadding="0" + cellspacing="0" + role="presentation" + style="width: 100%" + > + <tbody> + <tr> + <td + style=" + direction: ltr; + font-size: 0px; + padding: 20px 0px 20px 0px; + padding-bottom: 0px; + padding-top: 20px; + text-align: center; + " + > + <!--[if mso | IE]><table role="presentation" border="0" cellpadding="0" cellspacing="0"><tr><td class="" style="vertical-align:top;width:600px;" ><![endif]--> + <div + class="mj-column-per-100 mj-outlook-group-fix" + style=" + font-size: 0px; + text-align: left; + direction: ltr; + display: inline-block; + vertical-align: top; + width: 100%; + " + > + <table + border="0" + cellpadding="0" + cellspacing="0" + role="presentation" + style="vertical-align: top" + width="100%" + > + <tbody> + <tr> + <td + align="center" + style=" + font-size: 0px; + padding: 0px 20px 0px 20px; + padding-top: 0px; + padding-bottom: 0px; + word-break: break-word; + " + > + <div + style=" + font-family: Open Sans, Helvetica, Arial, + sans-serif; + font-size: 11px; + line-height: 22px; + text-align: center; + color: #797e82; + " + > + <p style="margin: 10px 0"> + <a + target="_blank" + rel="noopener noreferrer" + style="color: #005a8e" + href="https://legal.yahoo.com/xw/en/yahoo/privacy/topic/b2bprivacypolicy/index.html" + ><span style="color: #005a8e" + >Yahoo Privacy Policy</span + ></a + ><span style="color: #797e82" + > | </span + ><a + target="_blank" + rel="noopener noreferrer" + style="color: #005a8e" + href="https://console.tld/terms-of-service-trial.html" + ><span style="color: #005a8e" + >Terms of Service</span + ></a + ><span style="color: #797e82" + > | </span + ><a + target="_blank" + rel="noopener noreferrer" + style="color: #005a8e" + href="https://console.tld/support" + ><span style="color: #005a8e">Support</span></a + > + </p> + <p style="margin: 10px 0"> + <a + target="_blank" + rel="noopener noreferrer" + style="color: inherit; text-decoration: none" + href="https://console.tld/tenant/trial-tenant/account/notifications" + >Click + <span style="color: #005a8e"><u>here</u></span> + to manage your notifications setting.</a + ><br /> + </p> + </div> + </td> + </tr> + </tbody> + </table> + </div> + <!--[if mso | IE]></td></tr></table><![endif]--> + </td> + </tr> + </tbody> + </table> + </div> + <!--[if mso | IE]></td></tr></table><![endif]--> + </div> + </body> +</html> diff --git a/controller-server/src/test/resources/mail/trial-reminder.html b/controller-server/src/test/resources/mail/trial-reminder.html new file mode 100644 index 00000000000..fbe0d573538 --- /dev/null +++ b/controller-server/src/test/resources/mail/trial-reminder.html @@ -0,0 +1,646 @@ +<!DOCTYPE html> +<html + xmlns="http://www.w3.org/1999/xhtml" + xmlns:v="urn:schemas-microsoft-com:vml" + xmlns:o="urn:schemas-microsoft-com:office:office" +> + <head> + <title></title> + <!--[if !mso]><!--> + <meta http-equiv="X-UA-Compatible" content="IE=edge" /> + <!--<![endif]--> + <meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /> + <meta name="viewport" content="width=device-width,initial-scale=1" /> + <style type="text/css"> + #outlook a { + padding: 0; + } + + body { + margin: 0; + padding: 0; + -webkit-text-size-adjust: 100%; + -ms-text-size-adjust: 100%; + } + + table, + td { + border-collapse: collapse; + mso-table-lspace: 0pt; + mso-table-rspace: 0pt; + } + + img { + border: 0; + height: auto; + line-height: 100%; + outline: none; + text-decoration: none; + -ms-interpolation-mode: bicubic; + } + + p { + display: block; + margin: 13px 0; + } + </style> + <!--[if mso]> + <noscript> + <xml> + <o:OfficeDocumentSettings> + <o:AllowPNG /> + <o:PixelsPerInch>96</o:PixelsPerInch> + </o:OfficeDocumentSettings> + </xml> + </noscript> + <![endif]--> + <!--[if lte mso 11]> + <style type="text/css"> + .mj-outlook-group-fix { + width: 100% !important; + } + </style> + <![endif]--> + <!--[if !mso]><!--> + <link + href="https://fonts.googleapis.com/css?family=Open Sans" + rel="stylesheet" + type="text/css" + /> + <style type="text/css"> + @import url(https://fonts.googleapis.com/css?family=Open Sans); + </style> + <!--<![endif]--> + <style type="text/css"> + @media only screen and (min-width: 480px) { + .mj-column-per-100 { + width: 100% !important; + max-width: 100%; + } + } + </style> + <style media="screen and (min-width:480px)"> + .moz-text-html .mj-column-per-100 { + width: 100% !important; + max-width: 100%; + } + </style> + <style type="text/css"> + [owa] .mj-column-per-100 { + width: 100% !important; + max-width: 100%; + } + </style> + <style type="text/css"> + @media only screen and (max-width: 480px) { + table.mj-full-width-mobile { + width: 100% !important; + } + + td.mj-full-width-mobile { + width: auto !important; + } + } + </style> + </head> + + <body style="word-spacing: normal; background-color: #f2f7fa"> + <div style="background-color: #f2f7fa"> + <!--[if mso | IE]><table align="center" border="0" cellpadding="0" cellspacing="0" class="" role="presentation" style="width:600px;" width="600" ><tr><td style="line-height:0px;font-size:0px;mso-line-height-rule:exactly;"><![endif]--> + <div style="margin: 0px auto; max-width: 600px"> + <table + align="center" + border="0" + cellpadding="0" + cellspacing="0" + role="presentation" + style="width: 100%" + > + <tbody> + <tr> + <td + style=" + direction: ltr; + font-size: 0px; + padding: 20px 0px 20px 0px; + padding-bottom: 0px; + padding-top: 0px; + text-align: center; + " + > + <!--[if mso | IE]><table role="presentation" border="0" cellpadding="0" cellspacing="0"><tr><td class="" style="vertical-align:top;width:600px;" ><![endif]--> + <div + class="mj-column-per-100 mj-outlook-group-fix" + style=" + font-size: 0px; + text-align: left; + direction: ltr; + display: inline-block; + vertical-align: top; + width: 100%; + " + > + <table + border="0" + cellpadding="0" + cellspacing="0" + role="presentation" + style="vertical-align: top" + width="100%" + > + <tbody> + <tr> + <td + align="left" + style=" + font-size: 0px; + padding: 0px 0px 0px 25px; + padding-top: 0px; + padding-bottom: 0px; + word-break: break-word; + " + > + <div + style=" + font-family: Open Sans, Helvetica, Arial, + sans-serif; + font-size: 11px; + line-height: 22px; + text-align: left; + color: #797e82; + " + > + <br /> + </div> + </td> + </tr> + </tbody> + </table> + </div> + <!--[if mso | IE]></td></tr></table><![endif]--> + </td> + </tr> + </tbody> + </table> + </div> + <!--[if mso | IE]></td></tr></table><table align="center" border="0" cellpadding="0" cellspacing="0" class="" role="presentation" style="width:600px;" width="600" bgcolor="#ffffff" ><tr><td style="line-height:0px;font-size:0px;mso-line-height-rule:exactly;"><![endif]--> + <div + style=" + background: #ffffff; + background-color: #ffffff; + margin: 0px auto; + max-width: 600px; + " + > + <table + align="center" + border="0" + cellpadding="0" + cellspacing="0" + role="presentation" + style="background: #ffffff; background-color: #ffffff; width: 100%" + > + <tbody> + <tr> + <td + style=" + direction: ltr; + font-size: 0px; + padding: 20px 0; + padding-bottom: 0px; + padding-left: 0px; + padding-right: 0px; + padding-top: 0px; + text-align: center; + " + > + <!--[if mso | IE]><table role="presentation" border="0" cellpadding="0" cellspacing="0"><tr><td class="" style="vertical-align:top;width:600px;" ><![endif]--> + <div + class="mj-column-per-100 mj-outlook-group-fix" + style=" + font-size: 0px; + text-align: left; + direction: ltr; + display: inline-block; + vertical-align: top; + width: 100%; + " + > + <table + border="0" + cellpadding="0" + cellspacing="0" + role="presentation" + style="vertical-align: top" + width="100%" + > + <tbody> + <tr> + <td + align="center" + style=" + font-size: 0px; + padding: 10px 25px; + padding-top: 0px; + padding-right: 0px; + padding-bottom: 40px; + padding-left: 0px; + word-break: break-word; + " + > + <p + style=" + border-top: solid 8px #005a8e; + font-size: 1px; + margin: 0px auto; + width: 100%; + " + ></p> + <!--[if mso | IE + ]><table + align="center" + border="0" + cellpadding="0" + cellspacing="0" + style=" + border-top: solid 8px #005a8e; + font-size: 1px; + margin: 0px auto; + width: 600px; + " + role="presentation" + width="600px" + > + <tr> + <td style="height: 0; line-height: 0"> + + </td> + </tr> + </table><! + [endif]--> + </td> + </tr> + <tr> + <td + align="center" + style=" + font-size: 0px; + padding: 10px 25px; + padding-top: 0px; + padding-bottom: 0px; + word-break: break-word; + " + > + <table + border="0" + cellpadding="0" + cellspacing="0" + role="presentation" + style=" + border-collapse: collapse; + border-spacing: 0px; + " + > + <tbody> + <tr> + <td style="width: 121px"> + <img + alt="" + height="auto" + src="https://data.vespa.oath.cloud/assets/vespa-cloud-logo.png" + style=" + border: none; + display: block; + outline: none; + text-decoration: none; + height: auto; + width: 100%; + font-size: 13px; + " + width="121" + /> + </td> + </tr> + </tbody> + </table> + </td> + </tr> + </tbody> + </table> + </div> + <!--[if mso | IE]></td></tr></table><![endif]--> + </td> + </tr> + </tbody> + </table> + </div> + <!--[if mso | IE]></td></tr></table><table align="center" border="0" cellpadding="0" cellspacing="0" class="" role="presentation" style="width:600px;" width="600" bgcolor="#ffffff" ><tr><td style="line-height:0px;font-size:0px;mso-line-height-rule:exactly;"><![endif]--> + <div + style=" + background: #ffffff; + background-color: #ffffff; + margin: 0px auto; + max-width: 600px; + " + > + <table + align="center" + border="0" + cellpadding="0" + cellspacing="0" + role="presentation" + style="background: #ffffff; background-color: #ffffff; width: 100%" + > + <tbody> + <tr> + <td + style=" + direction: ltr; + font-size: 0px; + padding: 20px 0px 20px 0px; + padding-bottom: 70px; + padding-top: 30px; + text-align: center; + " + > + <!--[if mso | IE]><table role="presentation" border="0" cellpadding="0" cellspacing="0"><tr><td class="" style="vertical-align:top;width:600px;" ><![endif]--> + <div + class="mj-column-per-100 mj-outlook-group-fix" + style=" + font-size: 0px; + text-align: left; + direction: ltr; + display: inline-block; + vertical-align: top; + width: 100%; + " + > + <table + border="0" + cellpadding="0" + cellspacing="0" + role="presentation" + style="vertical-align: top" + width="100%" + > + +<tbody> +<tr> + <td + align="left" + style=" + font-size: 0px; + padding: 0px 25px 0px 25px; + padding-top: 0px; + padding-right: 50px; + padding-bottom: 0px; + padding-left: 50px; + word-break: break-word; + " + > + <div + style=" + font-family: Open Sans, Helvetica, Arial, + sans-serif; + font-size: 13px; + line-height: 22px; + text-align: left; + color: #797e82; + " + > + <h1 + style=" + text-align: center; + color: #000000; + line-height: 32px; + " + > + How is your Vespa Cloud trial going? + </h1> + </div> + </td> +</tr> +<tr> + <td + align="left" + style=" + font-size: 0px; + padding: 0px 25px 0px 25px; + padding-top: 0px; + padding-right: 50px; + padding-bottom: 0px; + padding-left: 50px; + word-break: break-word; + " + > + <div + style=" + font-family: Open Sans, Helvetica, Arial, + sans-serif; + font-size: 13px; + line-height: 22px; + text-align: left; + color: #797e82; + " + > + +<p> + How is your Vespa Cloud trial going? Please reach out to us if you have any questions or feedback. +</p> + </div> + </td> +</tr> +<tr> + <td + align="center" + vertical-align="middle" + style=" + font-size: 0px; + padding: 10px 25px; + padding-top: 20px; + padding-bottom: 20px; + word-break: break-word; + " + > + <table + border="0" + cellpadding="0" + cellspacing="0" + role="presentation" + style="border-collapse: separate; line-height: 100%" + > + <tbody> + <tr> + <td + align="center" + bgcolor="#005A8E" + role="presentation" + style=" + border: none; + border-radius: 100px; + cursor: auto; + mso-padding-alt: 15px 25px 15px 25px; + background: #005a8e; + " + valign="middle" + > + <a + href="https://console.tld/tenant/trial-tenant" + style=" + display: inline-block; + background: #005a8e; + color: #ffffff; + font-family: Open Sans, Helvetica, Arial, + sans-serif; + font-size: 13px; + font-weight: normal; + line-height: 120%; + margin: 0; + text-decoration: none; + text-transform: none; + padding: 15px 25px 15px 25px; + mso-padding-alt: 0px; + border-radius: 100px; + " + target="_blank" + ><b style="font-weight: 700" + ><b style="font-weight: 700" + >Go to Console</b + ></b + ></a + > + </td> + </tr> + </tbody> + </table> + </td> +</tr> +</tbody> + </table> + </div> + <!--[if mso | IE]></td></tr></table><![endif]--> + </td> + </tr> + </tbody> + </table> + </div> + <!--[if mso | IE]></td></tr></table><table align="center" border="0" cellpadding="0" cellspacing="0" class="" role="presentation" style="width:600px;" width="600" ><tr><td style="line-height:0px;font-size:0px;mso-line-height-rule:exactly;"><![endif]--> + <div style="margin: 0px auto; max-width: 600px"> + <table + align="center" + border="0" + cellpadding="0" + cellspacing="0" + role="presentation" + style="width: 100%" + > + <tbody> + <tr> + <td + style=" + direction: ltr; + font-size: 0px; + padding: 20px 0px 20px 0px; + padding-bottom: 0px; + padding-top: 20px; + text-align: center; + " + > + <!--[if mso | IE]><table role="presentation" border="0" cellpadding="0" cellspacing="0"><tr><td class="" style="vertical-align:top;width:600px;" ><![endif]--> + <div + class="mj-column-per-100 mj-outlook-group-fix" + style=" + font-size: 0px; + text-align: left; + direction: ltr; + display: inline-block; + vertical-align: top; + width: 100%; + " + > + <table + border="0" + cellpadding="0" + cellspacing="0" + role="presentation" + style="vertical-align: top" + width="100%" + > + <tbody> + <tr> + <td + align="center" + style=" + font-size: 0px; + padding: 0px 20px 0px 20px; + padding-top: 0px; + padding-bottom: 0px; + word-break: break-word; + " + > + <div + style=" + font-family: Open Sans, Helvetica, Arial, + sans-serif; + font-size: 11px; + line-height: 22px; + text-align: center; + color: #797e82; + " + > + <p style="margin: 10px 0"> + <a + target="_blank" + rel="noopener noreferrer" + style="color: #005a8e" + href="https://legal.yahoo.com/xw/en/yahoo/privacy/topic/b2bprivacypolicy/index.html" + ><span style="color: #005a8e" + >Yahoo Privacy Policy</span + ></a + ><span style="color: #797e82" + > | </span + ><a + target="_blank" + rel="noopener noreferrer" + style="color: #005a8e" + href="https://console.tld/terms-of-service-trial.html" + ><span style="color: #005a8e" + >Terms of Service</span + ></a + ><span style="color: #797e82" + > | </span + ><a + target="_blank" + rel="noopener noreferrer" + style="color: #005a8e" + href="https://console.tld/support" + ><span style="color: #005a8e">Support</span></a + > + </p> + <p style="margin: 10px 0"> + <a + target="_blank" + rel="noopener noreferrer" + style="color: inherit; text-decoration: none" + href="https://console.tld/tenant/trial-tenant/account/notifications" + >Click + <span style="color: #005a8e"><u>here</u></span> + to manage your notifications setting.</a + ><br /> + </p> + </div> + </td> + </tr> + </tbody> + </table> + </div> + <!--[if mso | IE]></td></tr></table><![endif]--> + </td> + </tr> + </tbody> + </table> + </div> + <!--[if mso | IE]></td></tr></table><![endif]--> + </div> + </body> +</html> diff --git a/controller-server/src/test/resources/mail/welcome.html b/controller-server/src/test/resources/mail/welcome.html new file mode 100644 index 00000000000..2e652532db8 --- /dev/null +++ b/controller-server/src/test/resources/mail/welcome.html @@ -0,0 +1,646 @@ +<!DOCTYPE html> +<html + xmlns="http://www.w3.org/1999/xhtml" + xmlns:v="urn:schemas-microsoft-com:vml" + xmlns:o="urn:schemas-microsoft-com:office:office" +> + <head> + <title></title> + <!--[if !mso]><!--> + <meta http-equiv="X-UA-Compatible" content="IE=edge" /> + <!--<![endif]--> + <meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /> + <meta name="viewport" content="width=device-width,initial-scale=1" /> + <style type="text/css"> + #outlook a { + padding: 0; + } + + body { + margin: 0; + padding: 0; + -webkit-text-size-adjust: 100%; + -ms-text-size-adjust: 100%; + } + + table, + td { + border-collapse: collapse; + mso-table-lspace: 0pt; + mso-table-rspace: 0pt; + } + + img { + border: 0; + height: auto; + line-height: 100%; + outline: none; + text-decoration: none; + -ms-interpolation-mode: bicubic; + } + + p { + display: block; + margin: 13px 0; + } + </style> + <!--[if mso]> + <noscript> + <xml> + <o:OfficeDocumentSettings> + <o:AllowPNG /> + <o:PixelsPerInch>96</o:PixelsPerInch> + </o:OfficeDocumentSettings> + </xml> + </noscript> + <![endif]--> + <!--[if lte mso 11]> + <style type="text/css"> + .mj-outlook-group-fix { + width: 100% !important; + } + </style> + <![endif]--> + <!--[if !mso]><!--> + <link + href="https://fonts.googleapis.com/css?family=Open Sans" + rel="stylesheet" + type="text/css" + /> + <style type="text/css"> + @import url(https://fonts.googleapis.com/css?family=Open Sans); + </style> + <!--<![endif]--> + <style type="text/css"> + @media only screen and (min-width: 480px) { + .mj-column-per-100 { + width: 100% !important; + max-width: 100%; + } + } + </style> + <style media="screen and (min-width:480px)"> + .moz-text-html .mj-column-per-100 { + width: 100% !important; + max-width: 100%; + } + </style> + <style type="text/css"> + [owa] .mj-column-per-100 { + width: 100% !important; + max-width: 100%; + } + </style> + <style type="text/css"> + @media only screen and (max-width: 480px) { + table.mj-full-width-mobile { + width: 100% !important; + } + + td.mj-full-width-mobile { + width: auto !important; + } + } + </style> + </head> + + <body style="word-spacing: normal; background-color: #f2f7fa"> + <div style="background-color: #f2f7fa"> + <!--[if mso | IE]><table align="center" border="0" cellpadding="0" cellspacing="0" class="" role="presentation" style="width:600px;" width="600" ><tr><td style="line-height:0px;font-size:0px;mso-line-height-rule:exactly;"><![endif]--> + <div style="margin: 0px auto; max-width: 600px"> + <table + align="center" + border="0" + cellpadding="0" + cellspacing="0" + role="presentation" + style="width: 100%" + > + <tbody> + <tr> + <td + style=" + direction: ltr; + font-size: 0px; + padding: 20px 0px 20px 0px; + padding-bottom: 0px; + padding-top: 0px; + text-align: center; + " + > + <!--[if mso | IE]><table role="presentation" border="0" cellpadding="0" cellspacing="0"><tr><td class="" style="vertical-align:top;width:600px;" ><![endif]--> + <div + class="mj-column-per-100 mj-outlook-group-fix" + style=" + font-size: 0px; + text-align: left; + direction: ltr; + display: inline-block; + vertical-align: top; + width: 100%; + " + > + <table + border="0" + cellpadding="0" + cellspacing="0" + role="presentation" + style="vertical-align: top" + width="100%" + > + <tbody> + <tr> + <td + align="left" + style=" + font-size: 0px; + padding: 0px 0px 0px 25px; + padding-top: 0px; + padding-bottom: 0px; + word-break: break-word; + " + > + <div + style=" + font-family: Open Sans, Helvetica, Arial, + sans-serif; + font-size: 11px; + line-height: 22px; + text-align: left; + color: #797e82; + " + > + <br /> + </div> + </td> + </tr> + </tbody> + </table> + </div> + <!--[if mso | IE]></td></tr></table><![endif]--> + </td> + </tr> + </tbody> + </table> + </div> + <!--[if mso | IE]></td></tr></table><table align="center" border="0" cellpadding="0" cellspacing="0" class="" role="presentation" style="width:600px;" width="600" bgcolor="#ffffff" ><tr><td style="line-height:0px;font-size:0px;mso-line-height-rule:exactly;"><![endif]--> + <div + style=" + background: #ffffff; + background-color: #ffffff; + margin: 0px auto; + max-width: 600px; + " + > + <table + align="center" + border="0" + cellpadding="0" + cellspacing="0" + role="presentation" + style="background: #ffffff; background-color: #ffffff; width: 100%" + > + <tbody> + <tr> + <td + style=" + direction: ltr; + font-size: 0px; + padding: 20px 0; + padding-bottom: 0px; + padding-left: 0px; + padding-right: 0px; + padding-top: 0px; + text-align: center; + " + > + <!--[if mso | IE]><table role="presentation" border="0" cellpadding="0" cellspacing="0"><tr><td class="" style="vertical-align:top;width:600px;" ><![endif]--> + <div + class="mj-column-per-100 mj-outlook-group-fix" + style=" + font-size: 0px; + text-align: left; + direction: ltr; + display: inline-block; + vertical-align: top; + width: 100%; + " + > + <table + border="0" + cellpadding="0" + cellspacing="0" + role="presentation" + style="vertical-align: top" + width="100%" + > + <tbody> + <tr> + <td + align="center" + style=" + font-size: 0px; + padding: 10px 25px; + padding-top: 0px; + padding-right: 0px; + padding-bottom: 40px; + padding-left: 0px; + word-break: break-word; + " + > + <p + style=" + border-top: solid 8px #005a8e; + font-size: 1px; + margin: 0px auto; + width: 100%; + " + ></p> + <!--[if mso | IE + ]><table + align="center" + border="0" + cellpadding="0" + cellspacing="0" + style=" + border-top: solid 8px #005a8e; + font-size: 1px; + margin: 0px auto; + width: 600px; + " + role="presentation" + width="600px" + > + <tr> + <td style="height: 0; line-height: 0"> + + </td> + </tr> + </table><! + [endif]--> + </td> + </tr> + <tr> + <td + align="center" + style=" + font-size: 0px; + padding: 10px 25px; + padding-top: 0px; + padding-bottom: 0px; + word-break: break-word; + " + > + <table + border="0" + cellpadding="0" + cellspacing="0" + role="presentation" + style=" + border-collapse: collapse; + border-spacing: 0px; + " + > + <tbody> + <tr> + <td style="width: 121px"> + <img + alt="" + height="auto" + src="https://data.vespa.oath.cloud/assets/vespa-cloud-logo.png" + style=" + border: none; + display: block; + outline: none; + text-decoration: none; + height: auto; + width: 100%; + font-size: 13px; + " + width="121" + /> + </td> + </tr> + </tbody> + </table> + </td> + </tr> + </tbody> + </table> + </div> + <!--[if mso | IE]></td></tr></table><![endif]--> + </td> + </tr> + </tbody> + </table> + </div> + <!--[if mso | IE]></td></tr></table><table align="center" border="0" cellpadding="0" cellspacing="0" class="" role="presentation" style="width:600px;" width="600" bgcolor="#ffffff" ><tr><td style="line-height:0px;font-size:0px;mso-line-height-rule:exactly;"><![endif]--> + <div + style=" + background: #ffffff; + background-color: #ffffff; + margin: 0px auto; + max-width: 600px; + " + > + <table + align="center" + border="0" + cellpadding="0" + cellspacing="0" + role="presentation" + style="background: #ffffff; background-color: #ffffff; width: 100%" + > + <tbody> + <tr> + <td + style=" + direction: ltr; + font-size: 0px; + padding: 20px 0px 20px 0px; + padding-bottom: 70px; + padding-top: 30px; + text-align: center; + " + > + <!--[if mso | IE]><table role="presentation" border="0" cellpadding="0" cellspacing="0"><tr><td class="" style="vertical-align:top;width:600px;" ><![endif]--> + <div + class="mj-column-per-100 mj-outlook-group-fix" + style=" + font-size: 0px; + text-align: left; + direction: ltr; + display: inline-block; + vertical-align: top; + width: 100%; + " + > + <table + border="0" + cellpadding="0" + cellspacing="0" + role="presentation" + style="vertical-align: top" + width="100%" + > + +<tbody> +<tr> + <td + align="left" + style=" + font-size: 0px; + padding: 0px 25px 0px 25px; + padding-top: 0px; + padding-right: 50px; + padding-bottom: 0px; + padding-left: 50px; + word-break: break-word; + " + > + <div + style=" + font-family: Open Sans, Helvetica, Arial, + sans-serif; + font-size: 13px; + line-height: 22px; + text-align: left; + color: #797e82; + " + > + <h1 + style=" + text-align: center; + color: #000000; + line-height: 32px; + " + > + Welcome to Vespa Cloud + </h1> + </div> + </td> +</tr> +<tr> + <td + align="left" + style=" + font-size: 0px; + padding: 0px 25px 0px 25px; + padding-top: 0px; + padding-right: 50px; + padding-bottom: 0px; + padding-left: 50px; + word-break: break-word; + " + > + <div + style=" + font-family: Open Sans, Helvetica, Arial, + sans-serif; + font-size: 13px; + line-height: 22px; + text-align: left; + color: #797e82; + " + > + +<p> + Welcome to Vespa Cloud! We hope you will enjoy your trial. Please reach out to us if you have any questions or feedback. +</p> + </div> + </td> +</tr> +<tr> + <td + align="center" + vertical-align="middle" + style=" + font-size: 0px; + padding: 10px 25px; + padding-top: 20px; + padding-bottom: 20px; + word-break: break-word; + " + > + <table + border="0" + cellpadding="0" + cellspacing="0" + role="presentation" + style="border-collapse: separate; line-height: 100%" + > + <tbody> + <tr> + <td + align="center" + bgcolor="#005A8E" + role="presentation" + style=" + border: none; + border-radius: 100px; + cursor: auto; + mso-padding-alt: 15px 25px 15px 25px; + background: #005a8e; + " + valign="middle" + > + <a + href="https://console.tld/tenant/trial-tenant" + style=" + display: inline-block; + background: #005a8e; + color: #ffffff; + font-family: Open Sans, Helvetica, Arial, + sans-serif; + font-size: 13px; + font-weight: normal; + line-height: 120%; + margin: 0; + text-decoration: none; + text-transform: none; + padding: 15px 25px 15px 25px; + mso-padding-alt: 0px; + border-radius: 100px; + " + target="_blank" + ><b style="font-weight: 700" + ><b style="font-weight: 700" + >Go to Console</b + ></b + ></a + > + </td> + </tr> + </tbody> + </table> + </td> +</tr> +</tbody> + </table> + </div> + <!--[if mso | IE]></td></tr></table><![endif]--> + </td> + </tr> + </tbody> + </table> + </div> + <!--[if mso | IE]></td></tr></table><table align="center" border="0" cellpadding="0" cellspacing="0" class="" role="presentation" style="width:600px;" width="600" ><tr><td style="line-height:0px;font-size:0px;mso-line-height-rule:exactly;"><![endif]--> + <div style="margin: 0px auto; max-width: 600px"> + <table + align="center" + border="0" + cellpadding="0" + cellspacing="0" + role="presentation" + style="width: 100%" + > + <tbody> + <tr> + <td + style=" + direction: ltr; + font-size: 0px; + padding: 20px 0px 20px 0px; + padding-bottom: 0px; + padding-top: 20px; + text-align: center; + " + > + <!--[if mso | IE]><table role="presentation" border="0" cellpadding="0" cellspacing="0"><tr><td class="" style="vertical-align:top;width:600px;" ><![endif]--> + <div + class="mj-column-per-100 mj-outlook-group-fix" + style=" + font-size: 0px; + text-align: left; + direction: ltr; + display: inline-block; + vertical-align: top; + width: 100%; + " + > + <table + border="0" + cellpadding="0" + cellspacing="0" + role="presentation" + style="vertical-align: top" + width="100%" + > + <tbody> + <tr> + <td + align="center" + style=" + font-size: 0px; + padding: 0px 20px 0px 20px; + padding-top: 0px; + padding-bottom: 0px; + word-break: break-word; + " + > + <div + style=" + font-family: Open Sans, Helvetica, Arial, + sans-serif; + font-size: 11px; + line-height: 22px; + text-align: center; + color: #797e82; + " + > + <p style="margin: 10px 0"> + <a + target="_blank" + rel="noopener noreferrer" + style="color: #005a8e" + href="https://legal.yahoo.com/xw/en/yahoo/privacy/topic/b2bprivacypolicy/index.html" + ><span style="color: #005a8e" + >Yahoo Privacy Policy</span + ></a + ><span style="color: #797e82" + > | </span + ><a + target="_blank" + rel="noopener noreferrer" + style="color: #005a8e" + href="https://console.tld/terms-of-service-trial.html" + ><span style="color: #005a8e" + >Terms of Service</span + ></a + ><span style="color: #797e82" + > | </span + ><a + target="_blank" + rel="noopener noreferrer" + style="color: #005a8e" + href="https://console.tld/support" + ><span style="color: #005a8e">Support</span></a + > + </p> + <p style="margin: 10px 0"> + <a + target="_blank" + rel="noopener noreferrer" + style="color: inherit; text-decoration: none" + href="https://console.tld/tenant/trial-tenant/account/notifications" + >Click + <span style="color: #005a8e"><u>here</u></span> + to manage your notifications setting.</a + ><br /> + </p> + </div> + </td> + </tr> + </tbody> + </table> + </div> + <!--[if mso | IE]></td></tr></table><![endif]--> + </td> + </tr> + </tbody> + </table> + </div> + <!--[if mso | IE]></td></tr></table><![endif]--> + </div> + </body> +</html> diff --git a/default_build_settings.cmake b/default_build_settings.cmake index 1e2981a2780..2da6af61211 100644 --- a/default_build_settings.cmake +++ b/default_build_settings.cmake @@ -130,16 +130,12 @@ function(vespa_use_default_build_settings) message("-- CMAKE_SYSTEM_PROCESSOR = ${CMAKE_SYSTEM_PROCESSOR}") if(CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64") if(APPLE AND (("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang") OR ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "AppleClang"))) - elseif("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang") - # Require haswell cpu or newer when compiling with clang on linux. - set(DEFAULT_VESPA_CPU_ARCH_FLAGS "-march=haswell -mtune=skylake") + elseif(CMAKE_CXX_COMPILER_ID STREQUAL "GNU" AND CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL 13.0) + # Temporary workaround for https://gcc.gnu.org/bugzilla/show_bug.cgi?id=108599 + set(DEFAULT_VESPA_CPU_ARCH_FLAGS "-march=ivybridge") else() - if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU" AND CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL 13.0) - # Temporary workaround for https://gcc.gnu.org/bugzilla/show_bug.cgi?id=108599 - set(DEFAULT_VESPA_CPU_ARCH_FLAGS "-march=ivybridge") - else() - set(DEFAULT_VESPA_CPU_ARCH_FLAGS "-msse3 -mcx16 -mtune=intel") - endif() + # Default to haswell cpu or newer + set(DEFAULT_VESPA_CPU_ARCH_FLAGS "-march=haswell -mtune=skylake") endif() elseif(CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64") set(DEFAULT_VESPA_CPU_ARCH_FLAGS "-march=armv8.2-a+fp16+rcpc+dotprod+crypto -mtune=neoverse-n1") diff --git a/dependency-versions/pom.xml b/dependency-versions/pom.xml index 574b8c2f716..c688c5018c6 100644 --- a/dependency-versions/pom.xml +++ b/dependency-versions/pom.xml @@ -34,11 +34,11 @@ <!-- DO NOT UPGRADE THESE TO A NEW MAJOR VERSION WITHOUT CHECKING FOR BINARY COMPATIBILITY --> <aopalliance.vespa.version>1.0</aopalliance.vespa.version> <commons-logging.vespa.version>1.2</commons-logging.vespa.version> <!-- This version is exported by jdisc via jcl-over-slf4j. --> - <error-prone-annotations.vespa.version>2.22.0</error-prone-annotations.vespa.version> + <error-prone-annotations.vespa.version>2.23.0</error-prone-annotations.vespa.version> <guava.vespa.version>32.1.3-jre</guava.vespa.version> <guice.vespa.version>6.0.0</guice.vespa.version> - <jackson2.vespa.version>2.15.2</jackson2.vespa.version> - <jackson-databind.vespa.version>2.15.2</jackson-databind.vespa.version> + <jackson2.vespa.version>2.15.3</jackson2.vespa.version> + <jackson-databind.vespa.version>2.15.3</jackson-databind.vespa.version> <jakarta.inject.vespa.version>2.0.1</jakarta.inject.vespa.version> <javax.inject.vespa.version>1</javax.inject.vespa.version> <javax.servlet-api.vespa.version>3.1.0</javax.servlet-api.vespa.version> @@ -65,7 +65,7 @@ <assertj.vespa.version>3.24.2</assertj.vespa.version> <!-- Athenz dependencies. Make sure these dependencies match those in Vespa's internal repositories --> - <athenz.vespa.version>1.11.43</athenz.vespa.version> + <athenz.vespa.version>1.11.44</athenz.vespa.version> <aws-sdk.vespa.version>1.12.565</aws-sdk.vespa.version> <!-- Athenz END --> @@ -78,15 +78,19 @@ <bouncycastle.vespa.version>1.76</bouncycastle.vespa.version> <byte-buddy.vespa.version>1.14.9</byte-buddy.vespa.version> <checker-qual.vespa.version>3.38.0</checker-qual.vespa.version> + <commons-beanutils.vespa.version>1.9.4</commons-beanutils.vespa.version> <commons-codec.vespa.version>1.16.0</commons-codec.vespa.version> + <commons-collections.vespa.version>3.2.2</commons-collections.vespa.version> <commons-csv.vespa.version>1.10.0</commons-csv.vespa.version> + <commons-digester.vespa.version>3.2</commons-digester.vespa.version> <commons-exec.vespa.version>1.3</commons-exec.vespa.version> - <commons-io.vespa.version>2.14.0</commons-io.vespa.version> + <commons-io.vespa.version>2.15.0</commons-io.vespa.version> <commons-lang3.vespa.version>3.13.0</commons-lang3.vespa.version> <commons.math3.vespa.version>3.6.1</commons.math3.vespa.version> <commons-compress.vespa.version>1.24.0</commons-compress.vespa.version> + <commons-cli.vespa.version>1.6.0</commons-cli.vespa.version> <curator.vespa.version>5.5.0</curator.vespa.version> - <dropwizard.metrics.vespa.version>4.2.20</dropwizard.metrics.vespa.version> + <dropwizard.metrics.vespa.version>4.2.21</dropwizard.metrics.vespa.version> <eclipse-collections.vespa.version>11.1.0</eclipse-collections.vespa.version> <felix.vespa.version>7.0.5</felix.vespa.version> <felix.log.vespa.version>1.3.0</felix.log.vespa.version> @@ -97,7 +101,7 @@ <icu4j.vespa.version>73.2</icu4j.vespa.version> <java-jjwt.vespa.version>0.11.5</java-jjwt.vespa.version> <java-jwt.vespa.version>4.4.0</java-jwt.vespa.version> - <jaxb.runtime.vespa.version>4.0.3</jaxb.runtime.vespa.version> + <jaxb.runtime.vespa.version>4.0.4</jaxb.runtime.vespa.version> <jetty.vespa.version>11.0.17</jetty.vespa.version> <jetty-servlet-api.vespa.version>5.0.2</jetty-servlet-api.vespa.version> <jimfs.vespa.version>1.3.0</jimfs.vespa.version> @@ -115,17 +119,19 @@ <mojo-executor.vespa.version>2.4.0</mojo-executor.vespa.version> <netty.vespa.version>4.1.100.Final</netty.vespa.version> <netty-tcnative.vespa.version>2.0.62.Final</netty-tcnative.vespa.version> - <onnxruntime.vespa.version>1.15.1</onnxruntime.vespa.version> + <onnxruntime.vespa.version>1.16.1</onnxruntime.vespa.version> <opennlp.vespa.version>2.3.0</opennlp.vespa.version> <opentest4j.vespa.version>1.3.0</opentest4j.vespa.version> - <org.json.vespa.version>20230618</org.json.vespa.version> + <org.json.vespa.version>20231013</org.json.vespa.version> <org.lz4.vespa.version>1.8.0</org.lz4.vespa.version> <prometheus.client.vespa.version>0.16.0</prometheus.client.vespa.version> <protobuf.vespa.version>3.24.4</protobuf.vespa.version> <questdb.vespa.version>7.3.3</questdb.vespa.version> <spifly.vespa.version>1.3.6</spifly.vespa.version> <snappy.vespa.version>1.1.10.5</snappy.vespa.version> - <surefire.vespa.version>3.1.2</surefire.vespa.version> + <surefire.vespa.version>3.2.1</surefire.vespa.version> + <velocity.vespa.version>2.3</velocity.vespa.version> + <velocity.tools.vespa.version>3.1</velocity.tools.vespa.version> <wiremock.vespa.version>3.2.0</wiremock.vespa.version> <xerces.vespa.version>2.12.2</xerces.vespa.version> <zero-allocation-hashing.vespa.version>0.16</zero-allocation-hashing.vespa.version> @@ -135,6 +141,7 @@ <!-- CAUTION: upgrading junit for tenants poms may break testing frameworks --> <junit.vespa.tenant.version>5.8.1</junit.vespa.tenant.version> <junit.platform.vespa.tenant.version>1.8.1</junit.platform.vespa.tenant.version> + <surefire.vespa.tenant.version>${surefire.vespa.version}</surefire.vespa.tenant.version> <!-- Maven plugins --> <clover-maven-plugin.vespa.version>4.5.0</clover-maven-plugin.vespa.version> @@ -143,22 +150,22 @@ <maven-bundle-plugin.vespa.version>5.1.9</maven-bundle-plugin.vespa.version> <maven-compiler-plugin.vespa.version>3.11.0</maven-compiler-plugin.vespa.version> <maven-core.vespa.version>3.9.5</maven-core.vespa.version> - <maven-dependency-plugin.vespa.version>3.6.0</maven-dependency-plugin.vespa.version> + <maven-dependency-plugin.vespa.version>3.6.1</maven-dependency-plugin.vespa.version> <maven-deploy-plugin.vespa.version>3.1.1</maven-deploy-plugin.vespa.version> <maven-enforcer-plugin.vespa.version>3.4.1</maven-enforcer-plugin.vespa.version> - <maven-failsafe-plugin.vespa.version>3.1.2</maven-failsafe-plugin.vespa.version> + <maven-failsafe-plugin.vespa.version>3.2.1</maven-failsafe-plugin.vespa.version> <maven-gpg-plugin.vespa.version>3.1.0</maven-gpg-plugin.vespa.version> <maven-install-plugin.vespa.version>3.1.1</maven-install-plugin.vespa.version> <maven-jar-plugin.vespa.version>3.3.0</maven-jar-plugin.vespa.version> <maven-javadoc-plugin.vespa.version>3.6.0</maven-javadoc-plugin.vespa.version> <maven-plugin-api.vespa.version>${maven-core.vespa.version}</maven-plugin-api.vespa.version> - <maven-plugin-tools.vespa.version>3.9.0</maven-plugin-tools.vespa.version> + <maven-plugin-tools.vespa.version>3.10.1</maven-plugin-tools.vespa.version> <maven-resources-plugin.vespa.version>3.3.1</maven-resources-plugin.vespa.version> <maven-resolver.vespa.version>1.9.16</maven-resolver.vespa.version> <maven-shade-plugin.vespa.version>3.5.1</maven-shade-plugin.vespa.version> <maven-site-plugin.vespa.version>3.12.1</maven-site-plugin.vespa.version> <maven-source-plugin.vespa.version>3.3.0</maven-source-plugin.vespa.version> - <properties-maven-plugin.vespa.version>1.2.0</properties-maven-plugin.vespa.version> + <properties-maven-plugin.vespa.version>1.2.1</properties-maven-plugin.vespa.version> <versions-maven-plugin.vespa.version>2.16.1</versions-maven-plugin.vespa.version> </properties> diff --git a/dist/vespa.spec b/dist/vespa.spec index 4a4e01a44ff..cd1381fd48d 100644 --- a/dist/vespa.spec +++ b/dist/vespa.spec @@ -42,7 +42,7 @@ License: Commercial URL: http://vespa.ai Source0: vespa-%{version}.tar.gz -BuildRequires: vespa-build-dependencies = 1.0.1 +BuildRequires: vespa-build-dependencies = 1.2.0 Requires: %{name}-base = %{version}-%{release} Requires: %{name}-base-libs = %{version}-%{release} @@ -196,7 +196,7 @@ Requires: vespa-protobuf = 3.21.12 Requires: protobuf Requires: llvm-libs %endif -Requires: vespa-onnxruntime = 1.15.1 +Requires: vespa-onnxruntime = 1.16.1 %description libs @@ -280,29 +280,6 @@ Requires: %{name}-base-libs = %{version}-%{release} Vespa - The open big data serving engine - devel package -%package ann-benchmark - -Summary: Vespa - The open big data serving engine - ann-benchmark - -Requires: %{name}-base-libs = %{version}-%{release} -Requires: %{name}-libs = %{version}-%{release} -%if 0%{?el8} -Requires: python39 -%endif -%if 0%{?el9} -Requires: python3 -%endif -%if 0%{?fedora} -Requires: python3 -%endif - -%description ann-benchmark - -Vespa - The open big data serving engine - ann-benchmark - -Python binding for the Vespa implementation of an HNSW index for -nearest neighbor search used for low-level benchmarking. - %prep %if 0%{?installdir:1} %if 0%{?source_base:1} @@ -383,10 +360,6 @@ export JAVA_HOME=%{?_java_home} export JAVA_HOME=/usr/lib/jvm/java-17-openjdk %endif export PATH="$JAVA_HOME/bin:$PATH" -%if 0%{?el8} -python3.9 -m pip install --user pytest -%endif -export PYTHONPATH="$PYTHONPATH:/usr/local/lib/$(basename $(readlink -f $(which python3)))/site-packages" #%{?_use_mvn_wrapper:./mvnw}%{!?_use_mvn_wrapper:mvn} --batch-mode -nsu -T 1C -Dmaven.javadoc.skip=true test make test ARGS="--output-on-failure %{_smp_mflags}" %endif @@ -537,7 +510,6 @@ fi %{_prefix}/lib/jars/zookeeper-command-line-client-jar-with-dependencies.jar %{_prefix}/lib/perl5 %{_prefix}/libexec -%exclude %{_prefix}/libexec/vespa_ann_benchmark %exclude %{_prefix}/libexec/vespa/common-env.sh %exclude %{_prefix}/libexec/vespa/vespa-wrapper %exclude %{_prefix}/libexec/vespa/find-pid @@ -763,12 +735,4 @@ fi %{_prefix}/include %{_prefix}/share/cmake -%files ann-benchmark -%if %{_defattr_is_vespa_vespa} -%defattr(-,%{_vespa_user},%{_vespa_group},-) -%endif -%dir %{_prefix} -%dir %{_prefix}/libexec -%{_prefix}/libexec/vespa_ann_benchmark - %changelog diff --git a/document/src/vespa/document/bucket/bucketid.cpp b/document/src/vespa/document/bucket/bucketid.cpp index c077d2dd4f6..fcc9a0df4f6 100644 --- a/document/src/vespa/document/bucket/bucketid.cpp +++ b/document/src/vespa/document/bucket/bucketid.cpp @@ -8,7 +8,6 @@ #include <vespa/vespalib/stllike/hash_set.hpp> #include <vespa/vespalib/util/stringfmt.h> #include <limits> -#include <xxh3.h> using vespalib::nbostream; using vespalib::asciistream; @@ -79,8 +78,7 @@ void BucketId::initialize() noexcept { uint64_t BucketId::hash::operator () (const BucketId& bucketId) const noexcept { - const uint64_t raw_id = bucketId.getId(); - return XXH3_64bits(&raw_id, sizeof(uint64_t)); + return vespalib::xxhash::xxh3_64(bucketId.getId()); } vespalib::string diff --git a/documentapi/src/tests/policies/policies_test.cpp b/documentapi/src/tests/policies/policies_test.cpp index 9dd73a71920..7091b63b6b3 100644 --- a/documentapi/src/tests/policies/policies_test.cpp +++ b/documentapi/src/tests/policies/policies_test.cpp @@ -806,7 +806,7 @@ Test::requireThatContentPolicyIsRandomWithoutState() ContentPolicy &policy = setupContentPolicy( frame, param, "storage/cluster.mycluster/distributor/*/default", 5); - ASSERT_TRUE(policy.getSystemState() == nullptr); + ASSERT_FALSE(policy.getSystemState()); std::set<string> lst; for (uint32_t i = 0; i < 666; i++) { @@ -858,12 +858,12 @@ Test::requireThatContentPolicyIsTargetedWithState() "cluster=mycluster;slobroks=tcp/localhost:%d;clusterconfigid=%s;syncinit", slobrok.port(), getDefaultDistributionConfig(2, 5).c_str()); ContentPolicy &policy = setupContentPolicy(frame, param, "storage/cluster.mycluster/distributor/*/default", 5); - ASSERT_TRUE(policy.getSystemState() == nullptr); + ASSERT_FALSE(policy.getSystemState()); { std::vector<mbus::RoutingNode*> leaf; ASSERT_TRUE(frame.select(leaf, 1)); leaf[0]->handleReply(std::make_unique<WrongDistributionReply>("distributor:5 storage:5")); - ASSERT_TRUE(policy.getSystemState() != nullptr); + ASSERT_TRUE(policy.getSystemState()); EXPECT_EQUAL(policy.getSystemState()->toString(), "distributor:5 storage:5"); } std::set<string> lst; @@ -897,12 +897,12 @@ Test::requireThatContentPolicyCombinesSystemAndSlobrokState() ContentPolicy &policy = setupContentPolicy( frame, param, "storage/cluster.mycluster/distributor/*/default", 1); - ASSERT_TRUE(policy.getSystemState() == nullptr); + ASSERT_FALSE(policy.getSystemState()); { std::vector<mbus::RoutingNode*> leaf; ASSERT_TRUE(frame.select(leaf, 1)); leaf[0]->handleReply(std::make_unique<WrongDistributionReply>("distributor:99 storage:99")); - ASSERT_TRUE(policy.getSystemState() != nullptr); + ASSERT_TRUE(policy.getSystemState()); EXPECT_EQUAL(policy.getSystemState()->toString(), "distributor:99 storage:99"); } for (int i = 0; i < 666; i++) { diff --git a/documentapi/src/vespa/documentapi/messagebus/policies/contentpolicy.cpp b/documentapi/src/vespa/documentapi/messagebus/policies/contentpolicy.cpp index e391699b750..ea27d42e790 100644 --- a/documentapi/src/vespa/documentapi/messagebus/policies/contentpolicy.cpp +++ b/documentapi/src/vespa/documentapi/messagebus/policies/contentpolicy.cpp @@ -43,7 +43,7 @@ namespace { class CallBack : public config::IFetcherCallback<storage::lib::Distribution::DistributionConfig> { public: - CallBack(ContentPolicy & policy) : _policy(policy) { } + explicit CallBack(ContentPolicy & policy) : _policy(policy) { } void configure(std::unique_ptr<storage::lib::Distribution::DistributionConfig> config) override { _policy.configure(std::move(config)); } @@ -78,13 +78,13 @@ string ContentPolicy::init() ContentPolicy::~ContentPolicy() = default; string -ContentPolicy::createConfigId(const string & clusterName) const +ContentPolicy::createConfigId(const string & clusterName) { return clusterName; } string -ContentPolicy::createPattern(const string & clusterName, int distributor) const +ContentPolicy::createPattern(const string & clusterName, int distributor) { vespalib::asciistream ost; @@ -103,7 +103,8 @@ void ContentPolicy::configure(std::unique_ptr<vespa::config::content::StorDistributionConfig> config) { try { - _nextDistribution = std::make_unique<storage::lib::Distribution>(*config); + std::lock_guard guard(_rw_lock); + _distribution = std::make_unique<storage::lib::Distribution>(*config); } catch (const std::exception& e) { LOG(warning, "Got exception when configuring distribution, config id was %s", _clusterConfigId.c_str()); throw e; @@ -116,8 +117,9 @@ ContentPolicy::doSelect(mbus::RoutingContext &context) const mbus::Message &msg = context.getMessage(); int distributor = -1; + auto [cur_state, cur_distribution] = internal_state_snapshot(); - if (_state.get()) { + if (cur_state) { document::BucketId id; switch(msg.getType()) { case DocumentProtocol::MESSAGE_PUTDOCUMENT: @@ -168,15 +170,10 @@ ContentPolicy::doSelect(mbus::RoutingContext &context) // Pick a distributor using ideal state algorithm try { - // Update distribution here, to make it not take lock in average case - if (_nextDistribution) { - _distribution = std::move(_nextDistribution); - _nextDistribution.reset(); - } - assert(_distribution.get()); - distributor = _distribution->getIdealDistributorNode(*_state, id); + assert(cur_distribution); + distributor = cur_distribution->getIdealDistributorNode(*cur_state, id); } catch (storage::lib::TooFewBucketBitsInUseException& e) { - auto reply = std::make_unique<WrongDistributionReply>(_state->toString()); + auto reply = std::make_unique<WrongDistributionReply>(cur_state->toString()); reply->addError(mbus::Error( DocumentProtocol::ERROR_WRONG_DISTRIBUTION, "Too few distribution bits used for given cluster state")); @@ -185,7 +182,7 @@ ContentPolicy::doSelect(mbus::RoutingContext &context) } catch (storage::lib::NoDistributorsAvailableException& e) { // No distributors available in current cluster state. Remove // cluster state we cannot use and send to random target - _state.reset(); + reset_state(); distributor = -1; } } @@ -216,7 +213,7 @@ ContentPolicy::getRecipient(mbus::RoutingContext& context, int distributor) return mbus::Hop::parse(entries[random() % entries.size()].second + "/default"); } - return mbus::Hop(); + return {}; } void @@ -226,9 +223,9 @@ ContentPolicy::merge(mbus::RoutingContext &context) mbus::Reply::UP reply = it.removeReply(); if (reply->getType() == DocumentProtocol::REPLY_WRONGDISTRIBUTION) { - updateStateFromReply(static_cast<WrongDistributionReply&>(*reply)); + updateStateFromReply(dynamic_cast<WrongDistributionReply&>(*reply)); } else if (reply->hasErrors()) { - _state.reset(); + reset_state(); } context.setReply(std::move(reply)); @@ -237,8 +234,8 @@ ContentPolicy::merge(mbus::RoutingContext &context) void ContentPolicy::updateStateFromReply(WrongDistributionReply& wdr) { - std::unique_ptr<storage::lib::ClusterState> newState( - new storage::lib::ClusterState(wdr.getSystemState())); + auto newState = std::make_unique<storage::lib::ClusterState>(wdr.getSystemState()); + std::lock_guard guard(_rw_lock); if (!_state || newState->getVersion() >= _state->getVersion()) { if (_state) { wdr.getTrace().trace(1, make_string("System state changed from version %u to %u", @@ -256,4 +253,28 @@ ContentPolicy::updateStateFromReply(WrongDistributionReply& wdr) } } +ContentPolicy::StateSnapshot +ContentPolicy::internal_state_snapshot() +{ + std::shared_lock guard(_rw_lock); + return {_state, _distribution}; +} + +std::shared_ptr<const storage::lib::ClusterState> +ContentPolicy::getSystemState() const noexcept +{ + std::shared_lock guard(_rw_lock); + return _state; +} + +void +ContentPolicy::reset_state() +{ + // It's possible for the caller to race between checking and resetting the state, + // but this should never lead to a worse outcome than sending to a random distributor + // as if no state had been cached prior. + std::lock_guard guard(_rw_lock); + _state.reset(); +} + } // documentapi diff --git a/documentapi/src/vespa/documentapi/messagebus/policies/contentpolicy.h b/documentapi/src/vespa/documentapi/messagebus/policies/contentpolicy.h index e49ad378b90..7a3675c3001 100644 --- a/documentapi/src/vespa/documentapi/messagebus/policies/contentpolicy.h +++ b/documentapi/src/vespa/documentapi/messagebus/policies/contentpolicy.h @@ -6,55 +6,62 @@ #include <vespa/vdslib/distribution/distribution.h> #include <vespa/document/bucket/bucketidfactory.h> #include <vespa/messagebus/routing/hop.h> +#include <shared_mutex> namespace config { class ICallback; class ConfigFetcher; } -namespace storage { -namespace lib { +namespace storage::lib { class Distribution; class ClusterState; } -} namespace documentapi { class ContentPolicy : public ExternSlobrokPolicy { private: - document::BucketIdFactory _bucketIdFactory; - std::unique_ptr<storage::lib::ClusterState> _state; - string _clusterName; - string _clusterConfigId; - std::unique_ptr<config::ICallback> _callBack; - std::unique_ptr<config::ConfigFetcher> _configFetcher; - std::unique_ptr<storage::lib::Distribution> _distribution; - std::unique_ptr<storage::lib::Distribution> _nextDistribution; + document::BucketIdFactory _bucketIdFactory; + mutable std::shared_mutex _rw_lock; + std::shared_ptr<const storage::lib::ClusterState> _state; + string _clusterName; + string _clusterConfigId; + std::unique_ptr<config::ICallback> _callBack; + std::unique_ptr<config::ConfigFetcher> _configFetcher; + std::shared_ptr<const storage::lib::Distribution> _distribution; + + using StateSnapshot = std::pair<std::shared_ptr<const storage::lib::ClusterState>, + std::shared_ptr<const storage::lib::Distribution>>; + + // Acquires _lock + [[nodiscard]] StateSnapshot internal_state_snapshot(); mbus::Hop getRecipient(mbus::RoutingContext& context, int distributor); + // Acquires _lock + void updateStateFromReply(WrongDistributionReply& reply); + // Acquires _lock + void reset_state(); public: - ContentPolicy(const string& param); - ~ContentPolicy(); + explicit ContentPolicy(const string& param); + ~ContentPolicy() override; void doSelect(mbus::RoutingContext &context) override; void merge(mbus::RoutingContext &context) override; - void updateStateFromReply(WrongDistributionReply& reply); - /** * @return a pointer to the system state registered with this policy. If - * we haven't received a system state yet, returns NULL. + * we haven't received a system state yet, returns nullptr. */ - const storage::lib::ClusterState* getSystemState() const { return _state.get(); } + std::shared_ptr<const storage::lib::ClusterState> getSystemState() const noexcept; virtual void configure(std::unique_ptr<storage::lib::Distribution::DistributionConfig> config); string init() override; private: - string createConfigId(const string & clusterName) const; - string createPattern(const string & clusterName, int distributor) const; + static string createConfigId(const string & clusterName); + static string createPattern(const string & clusterName, int distributor); }; } diff --git a/eval/src/apps/analyze_onnx_model/CMakeLists.txt b/eval/src/apps/analyze_onnx_model/CMakeLists.txt index a8f984550f9..6ab7a17e109 100644 --- a/eval/src/apps/analyze_onnx_model/CMakeLists.txt +++ b/eval/src/apps/analyze_onnx_model/CMakeLists.txt @@ -6,4 +6,6 @@ vespa_add_executable(eval_analyze_onnx_model_app INSTALL bin DEPENDS vespaeval + EXTERNAL_DEPENDS + dl ) diff --git a/eval/src/apps/analyze_onnx_model/analyze_onnx_model.cpp b/eval/src/apps/analyze_onnx_model/analyze_onnx_model.cpp index 2358a6c263e..051c5027999 100644 --- a/eval/src/apps/analyze_onnx_model/analyze_onnx_model.cpp +++ b/eval/src/apps/analyze_onnx_model/analyze_onnx_model.cpp @@ -10,6 +10,10 @@ #include <vespa/vespalib/util/guard.h> #include <vespa/vespalib/util/stringfmt.h> #include <charconv> +#ifdef __linux__ +#include <malloc.h> +#include <dlfcn.h> +#endif using vespalib::make_string_short::fmt; @@ -54,10 +58,13 @@ void extract(const vespalib::string &str, const vespalib::string &prefix, vespal } } struct MemoryUsage { - size_t size; - size_t rss; + size_t vm_size; + size_t rss_size; + size_t malloc_peak; + size_t malloc_current; }; +#ifdef __linux__ static const vespalib::string UNKNOWN = "unknown"; size_t convert(const vespalib::string & s) { @@ -85,12 +92,38 @@ MemoryUsage extract_memory_usage() { extract(line, "VmRSS:", vm_rss); } } - return {convert(vm_size), convert(vm_rss)}; + MemoryUsage usage = {}; + usage.vm_size = convert(vm_size); + usage.rss_size = convert(vm_rss); + +#if __GLIBC_PREREQ(2, 33) + struct mallinfo2 info = mallinfo2(); + usage.malloc_peak = size_t(info.usmblks); + usage.malloc_current = size_t(info.arena + info.hblkhd); +#else + struct mallinfo info = mallinfo(); + + if (dlsym(RTLD_NEXT, "is_vespamalloc") != nullptr) { + // Vespamalloc reports arena in 1M blocks as an 'int' is too small. + usage.malloc_peak = size_t(info.usmblks) * 1_Mi; + usage.malloc_current = size_t(info.arena + info.hblkhd) * 1_Mi; + } else { + usage.malloc_peak = size_t(info.usmblks); + usage.malloc_current = size_t(info.arena + info.hblkhd); + } +#endif + return usage; +} +#else +MemoryUsage extract_memory_usage() { + return { 0, 0, 0, 0 }; } +#endif void report_memory_usage(const vespalib::string &desc) { - MemoryUsage vm = extract_memory_usage(); - fprintf(stderr, "vm_size: %zu kB, vm_rss: %zu kB (%s)\n", vm.size/1024, vm.rss/1024, desc.c_str()); + MemoryUsage m = extract_memory_usage(); + fprintf(stderr, "vm_size: %zu kB, vm_rss: %zu kB, malloc_peak: %zu kb, malloc_curr: %zu (%s)\n", + m.vm_size/1_Ki, m.rss_size/1_Ki, m.malloc_peak/1_Ki, m.malloc_current/1_Ki, desc.c_str()); } struct Options { @@ -286,8 +319,10 @@ int probe_types() { types.setString(output.name, output_type.to_spec()); } MemoryUsage vm_after = extract_memory_usage(); - root.setLong("vm_size", vm_after.size - vm_before.size); - root.setLong("vm_rss", vm_after.rss - vm_before.rss); + root.setLong("vm_size", vm_after.vm_size - vm_before.vm_size); + root.setLong("vm_rss", vm_after.rss_size - vm_before.rss_size); + root.setLong("malloc_peak", vm_after.malloc_peak - vm_before.malloc_peak); + root.setLong("malloc_current", vm_after.malloc_current - vm_before.malloc_current); write_compact(result, std_out); return 0; } diff --git a/eval/src/tests/eval/fast_value/fast_value_test.cpp b/eval/src/tests/eval/fast_value/fast_value_test.cpp index c734b11e2d3..2332ab3bf8a 100644 --- a/eval/src/tests/eval/fast_value/fast_value_test.cpp +++ b/eval/src/tests/eval/fast_value/fast_value_test.cpp @@ -4,9 +4,11 @@ #include <vespa/eval/eval/fast_value.h> #include <vespa/eval/eval/value_codec.h> #include <vespa/eval/eval/test/gen_spec.h> +#include <vespa/vespalib/util/stringfmt.h> #include <vespa/vespalib/gtest/gtest.h> using namespace vespalib; +using namespace vespalib::make_string_short; using namespace vespalib::eval; using namespace vespalib::eval::test; @@ -114,9 +116,9 @@ TEST(FastValueTest, insert_subspace) { } TEST(FastValueTest, insert_empty_subspace) { - auto addr = [](){ return ConstArrayRef<string_id>(); }; + auto addr = []() { return ConstArrayRef<string_id>(); }; auto type = ValueType::from_spec("double"); - auto value = std::make_unique<FastValue<double,true>>(type, 0, 1, 1); + auto value = std::make_unique<FastValue<double, true>>(type, 0, 1, 1); EXPECT_EQ(value->index().size(), 0); { auto [cells, added] = value->insert_subspace(addr()); @@ -124,7 +126,8 @@ TEST(FastValueTest, insert_empty_subspace) { EXPECT_EQ(value->index().size(), 1); ASSERT_EQ(cells.size(), 1); cells[0] = 10.0; - }{ + } + { auto [cells, added] = value->insert_subspace(addr()); EXPECT_FALSE(added); EXPECT_EQ(value->index().size(), 1); @@ -137,6 +140,27 @@ TEST(FastValueTest, insert_empty_subspace) { EXPECT_EQ(actual, expected); } +void +verifyFastValueSize(TensorSpec spec, uint32_t elems, size_t expected) { + for (uint32_t i=0; i < elems; i++) { + spec.add({{"country", fmt("no%d", i)}}, 17.0); + } + auto value = value_from_spec(spec, FastValueBuilderFactory::get()); + EXPECT_EQ(expected, value->get_memory_usage().allocatedBytes()); +} + +TEST(FastValueTest, document_fast_value_memory_usage) { + EXPECT_EQ(232, sizeof(FastValue<float,true>)); + FastValue<float,true> test(ValueType::from_spec("tensor<float>(country{})"), 1, 1, 1); + EXPECT_EQ(412, test.get_memory_usage().allocatedBytes()); + + verifyFastValueSize(TensorSpec("tensor<float>(country{})"), 1, 412); + verifyFastValueSize(TensorSpec("tensor<float>(country{})"), 10, 792); + verifyFastValueSize(TensorSpec("tensor<float>(country{})"), 20, 1280); + verifyFastValueSize(TensorSpec("tensor<float>(country{})"), 50, 2296); + verifyFastValueSize(TensorSpec("tensor<float>(country{})"), 100, 4288); +} + using SA = std::vector<vespalib::stringref>; TEST(FastValueBuilderTest, scalar_add_subspace_robustness) { diff --git a/eval/src/vespa/eval/eval/fast_value.hpp b/eval/src/vespa/eval/eval/fast_value.hpp index 0ad5124a6c9..cc9eb663b76 100644 --- a/eval/src/vespa/eval/eval/fast_value.hpp +++ b/eval/src/vespa/eval/eval/fast_value.hpp @@ -36,7 +36,7 @@ struct FastCells { size_t capacity; size_t size; mutable alloc::Alloc memory; - FastCells(size_t initial_capacity); + explicit FastCells(size_t initial_capacity); FastCells(const FastCells &) = delete; FastCells & operator = (const FastCells &) = delete; ~FastCells(); @@ -184,6 +184,7 @@ struct FastValue final : Value, ValueBuilder<T> { } MemoryUsage get_memory_usage() const override { MemoryUsage usage = self_memory_usage<FastValue<T,transient>>(); + usage.merge(vector_extra_memory_usage(my_type.dimensions())); usage.merge(vector_extra_memory_usage(get_view(my_handles))); usage.merge(my_index.map.estimate_extra_memory_usage()); usage.merge(my_cells.estimate_extra_memory_usage()); diff --git a/eval/src/vespa/eval/eval/value_builder_factory.h b/eval/src/vespa/eval/eval/value_builder_factory.h index a0266956f80..8f4a10b7281 100644 --- a/eval/src/vespa/eval/eval/value_builder_factory.h +++ b/eval/src/vespa/eval/eval/value_builder_factory.h @@ -11,7 +11,7 @@ namespace vespalib::eval { * downcasting to actual builder with specialized cell type. **/ struct ValueBuilderBase { - virtual ~ValueBuilderBase() {} + virtual ~ValueBuilderBase() = default; }; /** @@ -59,7 +59,7 @@ private: { assert(check_cell_type<T>(type.cell_type())); auto base = create_value_builder_base(type, transient, num_mapped_dims_in, subspace_size_in, expected_subspaces); - ValueBuilder<T> *builder = static_cast<ValueBuilder<T>*>(base.get()); + auto *builder = static_cast<ValueBuilder<T>*>(base.get()); base.release(); return std::unique_ptr<ValueBuilder<T>>(builder); } @@ -82,7 +82,7 @@ public: return create_value_builder<T>(type, false, type.count_mapped_dimensions(), type.dense_subspace_size(), 1); } std::unique_ptr<Value> copy(const Value &value) const; - virtual ~ValueBuilderFactory() {} + virtual ~ValueBuilderFactory() = default; protected: virtual std::unique_ptr<ValueBuilderBase> create_value_builder_base(const ValueType &type, bool transient, size_t num_mapped_dims_in, size_t subspace_size_in, size_t expected_subspaces) const = 0; diff --git a/eval/src/vespa/eval/eval/value_type.cpp b/eval/src/vespa/eval/eval/value_type.cpp index cda72acbc34..1a83de9b0f9 100644 --- a/eval/src/vespa/eval/eval/value_type.cpp +++ b/eval/src/vespa/eval/eval/value_type.cpp @@ -71,7 +71,7 @@ struct MyJoin { vespalib::string concat_dim; MyJoin(const DimensionList &lhs, const DimensionList &rhs) : mismatch(false), dimensions(), concat_dim() { my_join(lhs, rhs); } - MyJoin(const DimensionList &lhs, const DimensionList &rhs, vespalib::string concat_dim_in) + MyJoin(const DimensionList &lhs, const DimensionList &rhs, const vespalib::string & concat_dim_in) : mismatch(false), dimensions(), concat_dim(concat_dim_in) { my_join(lhs, rhs); } ~MyJoin(); private: @@ -152,6 +152,8 @@ ValueType::error_if(bool has_error, ValueType else_type) } } +ValueType::ValueType(const ValueType &) = default; +ValueType & ValueType::operator =(const ValueType &) = default; ValueType::~ValueType() = default; bool @@ -302,6 +304,7 @@ std::vector<vespalib::string> ValueType::dimension_names() const { std::vector<vespalib::string> result; + result.reserve(_dimensions.size()); for (const auto &dimension: _dimensions) { result.push_back(dimension.name); } @@ -367,7 +370,7 @@ ValueType::make_type(CellType cell_type, std::vector<Dimension> dimensions_in) if (!verify_dimensions(dimensions_in)) { return error_type(); } - return ValueType(cell_type, std::move(dimensions_in)); + return {cell_type, std::move(dimensions_in)}; } ValueType diff --git a/eval/src/vespa/eval/eval/value_type.h b/eval/src/vespa/eval/eval/value_type.h index 99cc61c0af1..b35e23ee4e6 100644 --- a/eval/src/vespa/eval/eval/value_type.h +++ b/eval/src/vespa/eval/eval/value_type.h @@ -49,9 +49,9 @@ private: public: ValueType(ValueType &&) noexcept = default; - ValueType(const ValueType &) = default; + ValueType(const ValueType &); ValueType &operator=(ValueType &&) noexcept = default; - ValueType &operator=(const ValueType &) = default; + ValueType &operator=(const ValueType &); ~ValueType(); CellType cell_type() const { return _cell_type; } CellMeta cell_meta() const { return {_cell_type, is_double()}; } @@ -88,7 +88,7 @@ public: const std::vector<vespalib::string> &to) const; ValueType cell_cast(CellType to_cell_type) const; - static ValueType error_type() { return ValueType(); } + static ValueType error_type() { return {}; } static ValueType make_type(CellType cell_type, std::vector<Dimension> dimensions_in); static ValueType double_type() { return make_type(CellType::DOUBLE, {}); } static ValueType from_spec(const vespalib::string &spec); diff --git a/eval/src/vespa/eval/onnx/onnx_wrapper.h b/eval/src/vespa/eval/onnx/onnx_wrapper.h index 651461a45e6..205256079da 100644 --- a/eval/src/vespa/eval/onnx/onnx_wrapper.h +++ b/eval/src/vespa/eval/onnx/onnx_wrapper.h @@ -2,11 +2,7 @@ #pragma once -#ifdef __APPLE__ -#include <onnxruntime/core/session/onnxruntime_cxx_api.h> -#else #include <onnxruntime/onnxruntime_cxx_api.h> -#endif #include <vespa/vespalib/stllike/string.h> #include <vespa/eval/eval/value_type.h> #include <vespa/eval/eval/value.h> diff --git a/flags/src/main/java/com/yahoo/vespa/flags/FetchVector.java b/flags/src/main/java/com/yahoo/vespa/flags/FetchVector.java index 4c4bada7552..b5a944430f3 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/FetchVector.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/FetchVector.java @@ -22,8 +22,11 @@ public class FetchVector { * Note: If this enum is changed, you must also change {@link DimensionHelper}. */ public enum Dimension { - /** Application id from ApplicationId::toSerializedForm(TenantName, ApplicationName) on the form tenant:applicationName. */ - APPLICATION_ID, + /** + * Application from ApplicationId::toSerializedFormWithoutInstance() of the form tenant:applicationName. + * <p><em>WARNING: NOT ApplicationId</em> - see {@link #INSTANCE_ID}.</p> + */ + APPLICATION, /** * Cloud from com.yahoo.config.provision.CloudName::value, e.g. yahoo, aws, gcp. diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index 56cd06d3b35..0f1815e44b6 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -13,7 +13,7 @@ import java.util.Optional; import java.util.TreeMap; import java.util.function.Predicate; -import static com.yahoo.vespa.flags.FetchVector.Dimension.APPLICATION_ID; +import static com.yahoo.vespa.flags.FetchVector.Dimension.APPLICATION; import static com.yahoo.vespa.flags.FetchVector.Dimension.INSTANCE_ID; import static com.yahoo.vespa.flags.FetchVector.Dimension.CLOUD_ACCOUNT; import static com.yahoo.vespa.flags.FetchVector.Dimension.CLUSTER_ID; @@ -305,16 +305,10 @@ public class Flags { INSTANCE_ID); public static final UnboundBooleanFlag ENABLE_CROWDSTRIKE = defineFeatureFlag( - "enable-crowdstrike", true, List.of("andreer"), "2023-04-13", "2023-10-14", + "enable-crowdstrike", true, List.of("andreer"), "2023-04-13", "2023-11-14", "Whether to enable CrowdStrike.", "Takes effect on next host admin tick", HOSTNAME); - public static final UnboundBooleanFlag RANDOMIZED_ENDPOINT_NAMES = defineFeatureFlag( - "randomized-endpoint-names", false, List.of("andreer"), "2023-04-26", "2023-10-14", - "Whether to use randomized endpoint names", - "Takes effect on application deployment", - INSTANCE_ID, APPLICATION_ID, TENANT_ID); - public static final UnboundBooleanFlag ENABLE_THE_ONE_THAT_SHOULD_NOT_BE_NAMED = defineFeatureFlag( "enable-the-one-that-should-not-be-named", false, List.of("hmusum"), "2023-05-08", "2023-11-01", "Whether to enable the one program that should not be named", @@ -340,6 +334,20 @@ public class Flags { "Takes effect at redeployment", INSTANCE_ID); + public static final UnboundBooleanFlag EXCLUSIVE_PROVISIONING = defineFeatureFlag( + "exclusive-provisioning", false, + List.of("hakonhall"), "2023-10-12", "2023-12-20", + "Whether to provision a host exclusively to an application ID only based on exclusive=\"true\" from services.xml. " + + "Enabling this will produce hosts with exclusiveTo[ApplicationId] without provisionedToApplicationId.", + "Takes immediate effect when provisioning new hosts"); + + public static final UnboundBooleanFlag MAKE_EXCLUSIVE = defineFeatureFlag( + "make-exclusive", false, + List.of("hakonhall"), "2023-10-20", "2023-12-20", + "Allow an exclusive allocation to a non-exclusive host, but if so, make the host exclusive.", + "Takes immediate effect on any following preparation of clusters", + INSTANCE_ID, TENANT_ID, VESPA_VERSION); + public static final UnboundBooleanFlag WRITE_CONFIG_SERVER_SESSION_DATA_AS_ONE_BLOB = defineFeatureFlag( "write-config-server-session-data-as-blob", false, List.of("hmusum"), "2023-07-19", "2023-11-01", @@ -354,21 +362,21 @@ public class Flags { public static final UnboundBooleanFlag MORE_WIREGUARD = defineFeatureFlag( "more-wireguard", false, - List.of("andreer"), "2023-08-21", "2023-10-14", + List.of("andreer"), "2023-08-21", "2023-11-14", "Use wireguard in INternal enCLAVES", "Takes effect on next host-admin run", HOSTNAME, CLOUD_ACCOUNT); public static final UnboundBooleanFlag IPV6_AWS_TARGET_GROUPS = defineFeatureFlag( "ipv6-aws-target-groups", false, - List.of("andreer"), "2023-08-28", "2023-10-14", + List.of("andreer"), "2023-08-28", "2023-11-14", "Always use IPv6 target groups for load balancers in aws", "Takes effect on next load-balancer provisioning", HOSTNAME, CLOUD_ACCOUNT); public static final UnboundBooleanFlag PROVISION_IPV6_ONLY_AWS = defineFeatureFlag( "provision-ipv6-only", false, - List.of("andreer"), "2023-08-28", "2023-10-14", + List.of("andreer"), "2023-08-28", "2023-11-14", "Provision without private IPv4 addresses in INternal enCLAVES in AWS", "Takes effect on next host provisioning / run of host-admin", HOSTNAME, CLOUD_ACCOUNT); @@ -380,20 +388,6 @@ public class Flags { "Takes effect immediately", INSTANCE_ID, CLUSTER_ID, CLUSTER_TYPE); - public static final UnboundBooleanFlag ASSIGN_RANDOMIZED_ID = defineFeatureFlag( - "assign-randomized-id", true, - List.of("mortent"), "2023-08-31", "2024-02-01", - "Whether to assign randomized id to the application", - "Takes effect immediately", - INSTANCE_ID); - - public static final UnboundIntFlag ASSIGNED_RANDOMIZED_ID_RATE = defineIntFlag( - "assign-randomized-id-rate", 5, - List.of("mortent"), "2023-09-11", "2024-02-01", - "Rate for requesting assigned ids for existing certificates. Rate is per maintainer cycle.", - "Takes effect immediately", - INSTANCE_ID); - public static final UnboundIntFlag CONTENT_LAYER_METADATA_FEATURE_LEVEL = defineIntFlag( "content-layer-metadata-feature-level", 0, List.of("vekterli"), "2022-09-12", "2024-02-01", @@ -411,30 +405,30 @@ public class Flags { public static final UnboundStringFlag UNKNOWN_CONFIG_DEFINITION = defineStringFlag( "unknown-config-definition", "warn", - List.of("hmusum"), "2023-09-25", "2023-11-01", - "How to handle user config referencing unknown config definitions. Valid values are log, warn, fail", + List.of("hmusum"), "2023-09-25", "2023-11-15", + "How to handle user config referencing unknown config definitions. Valid values are 'warn' and 'fail'", "Takes effect at redeployment", INSTANCE_ID); - public static final UnboundBooleanFlag LEGACY_ENDPOINTS = defineFeatureFlag( - "legacy-endpoints", true, List.of("mpolden", "tokle"), "2023-09-29", "2024-03-01", - "Whether legacy (non-anonymized) endpoints should be created in DNS", - "Takes effect on redeployment through controller", - INSTANCE_ID, APPLICATION_ID, TENANT_ID); - public static final UnboundIntFlag SEARCH_HANDLER_THREADPOOL = defineIntFlag( "search-handler-threadpool", 2, List.of("bjorncs", "baldersheim"), "2023-10-01", "2024-01-01", "Adjust search handler threadpool size", "Takes effect at redeployment", - APPLICATION_ID); + APPLICATION); public static final UnboundStringFlag ENDPOINT_CONFIG = defineStringFlag( "endpoint-config", "legacy", List.of("mpolden", "tokle"), "2023-10-06", "2024-02-01", "Set the endpoint config to use for an application. Must be 'legacy', 'combined' or 'generated'. See EndpointConfig for further details", "Takes effect on next deployment through controller", - APPLICATION_ID); + TENANT_ID, APPLICATION, INSTANCE_ID); + + public static final UnboundBooleanFlag CLOUD_TRIAL_NOTIFICATIONS = defineFeatureFlag( + "cloud-trial-notifications", false, + List.of("bjorncs", "oyving"), "2023-10-13", "2024-03-01", + "Whether to send cloud trial email notifications", + "Takes effect immediately"); /** WARNING: public for testing: All flags should be defined in {@link Flags}. */ public static UnboundBooleanFlag defineFeatureFlag(String flagId, boolean defaultValue, List<String> owners, diff --git a/flags/src/main/java/com/yahoo/vespa/flags/json/DimensionHelper.java b/flags/src/main/java/com/yahoo/vespa/flags/json/DimensionHelper.java index dea88d1687b..7298f090be2 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/json/DimensionHelper.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/json/DimensionHelper.java @@ -4,8 +4,6 @@ package com.yahoo.vespa.flags.json; import com.yahoo.vespa.flags.FetchVector; import java.util.HashMap; -import java.util.LinkedHashMap; -import java.util.List; import java.util.Map; import java.util.stream.Collectors; @@ -17,7 +15,9 @@ public class DimensionHelper { private static final Map<FetchVector.Dimension, String> serializedDimensions = new HashMap<>(); static { - serializedDimensions.put(FetchVector.Dimension.APPLICATION_ID, "application"); + // WARNING: If you ever change the serialized form of a dimension, ensure the new serialized + // flag data are pushed out everywhere before removing support for old format, see VESPA-27760. + serializedDimensions.put(FetchVector.Dimension.APPLICATION, "application"); serializedDimensions.put(FetchVector.Dimension.CLOUD, "cloud"); serializedDimensions.put(FetchVector.Dimension.CLOUD_ACCOUNT, "cloud-account"); serializedDimensions.put(FetchVector.Dimension.CLUSTER_ID, "cluster-id"); diff --git a/hosted-api/src/main/java/ai/vespa/hosted/api/MultiPartStreamer.java b/hosted-api/src/main/java/ai/vespa/hosted/api/MultiPartStreamer.java index c56dd219879..21c189e2549 100644 --- a/hosted-api/src/main/java/ai/vespa/hosted/api/MultiPartStreamer.java +++ b/hosted-api/src/main/java/ai/vespa/hosted/api/MultiPartStreamer.java @@ -23,7 +23,7 @@ import java.util.stream.Collectors; import java.util.stream.Stream; /** - * Used to create builders for multi part http body entities, which stream their data. + * Used to create builders for multipart HTTP body entities, which stream their data. * * @author jonmv */ diff --git a/hosted-tenant-base/pom.xml b/hosted-tenant-base/pom.xml index 729150bcef1..9851659bef0 100644 --- a/hosted-tenant-base/pom.xml +++ b/hosted-tenant-base/pom.xml @@ -192,7 +192,7 @@ <plugin> <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-surefire-plugin</artifactId> - <version>${surefire.vespa.version}</version> + <version>${surefire.vespa.tenant.version}</version> <configuration> <groups>${test.categories}</groups> <redirectTestOutputToFile>false</redirectTestOutputToFile> @@ -215,7 +215,7 @@ <plugin> <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-surefire-report-plugin</artifactId> - <version>${surefire.vespa.version}</version> + <version>${surefire.vespa.tenant.version}</version> <configuration> <reportsDirectory>${env.TEST_DIR}</reportsDirectory> </configuration> diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/linguistics/LinguisticsAnnotator.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/linguistics/LinguisticsAnnotator.java index d65d25aa537..191d067effe 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/linguistics/LinguisticsAnnotator.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/linguistics/LinguisticsAnnotator.java @@ -12,6 +12,7 @@ import com.yahoo.language.Linguistics; import com.yahoo.language.process.StemMode; import com.yahoo.language.process.Token; import com.yahoo.language.process.Tokenizer; +import com.yahoo.text.Text; import java.util.HashMap; import java.util.Map; @@ -71,7 +72,7 @@ public class LinguisticsAnnotator { Tokenizer tokenizer = factory.getTokenizer(); String input = (text.getString().length() <= config.getMaxTokenizeLength()) ? text.getString() - : text.getString().substring(0, config.getMaxTokenizeLength()); + : Text.substringByCodepoints(text.getString(), 0, config.getMaxTokenizeLength()); Iterable<Token> tokens = tokenizer.tokenize(input, config.getLanguage(), config.getStemMode(), config.getRemoveAccents()); TermOccurrences termOccurrences = new TermOccurrences(config.getMaxTermOccurrences()); @@ -88,9 +89,9 @@ public class LinguisticsAnnotator { * Creates a TERM annotation which has the lowercase value as annotation (only) if it is different from the * original. * - * @param termToLowerCase The term to lower case. - * @param origTerm The original term. - * @return the created TERM annotation. + * @param termToLowerCase the term to lower case + * @param origTerm the original term + * @return the created TERM annotation */ public static Annotation lowerCaseTermAnnotation(String termToLowerCase, String origTerm) { String annotationValue = toLowerCase(termToLowerCase); diff --git a/linguistics/src/main/java/com/yahoo/language/LinguisticsCase.java b/linguistics/src/main/java/com/yahoo/language/LinguisticsCase.java index 143acc174f0..5ad6a382abd 100644 --- a/linguistics/src/main/java/com/yahoo/language/LinguisticsCase.java +++ b/linguistics/src/main/java/com/yahoo/language/LinguisticsCase.java @@ -7,7 +7,7 @@ import java.util.Locale; /** * This class provides a case normalization operation to be used e.g. when - * document search should be case insensitive. + * document search should be case-insensitive. * * @author Simon Thoresen Hult */ diff --git a/messagebus/src/main/java/com/yahoo/messagebus/Routable.java b/messagebus/src/main/java/com/yahoo/messagebus/Routable.java index 06d92c48dd3..a94541ac5f6 100755 --- a/messagebus/src/main/java/com/yahoo/messagebus/Routable.java +++ b/messagebus/src/main/java/com/yahoo/messagebus/Routable.java @@ -10,7 +10,7 @@ import com.yahoo.text.Utf8String; * A routable can be regarded as a protocol-defined value with additional message bus related state. The state is what * differentiates two Routables that carry the same value. This includes the application context attached to the * routable and the {@link CallStack} used to track the path of the routable within messagebus. When a routable is - * copied (if the protocol supports it) only the value part is copied. The state must be explicitly transfered by + * copied (if the protocol supports it) only the value part is copied. The state must be explicitly transferred by * invoking the {@link #swapState(Routable)} method. That method is used to transfer the state from a message to the * corresponding reply, or to a different message if the application decides to replace it. * diff --git a/messagebus/src/main/java/com/yahoo/messagebus/Sequencer.java b/messagebus/src/main/java/com/yahoo/messagebus/Sequencer.java index 2244559b54a..4e10a72c858 100644 --- a/messagebus/src/main/java/com/yahoo/messagebus/Sequencer.java +++ b/messagebus/src/main/java/com/yahoo/messagebus/Sequencer.java @@ -145,10 +145,10 @@ public class Sequencer implements MessageHandler, ReplyHandler { } private class SequencedSendTask implements Messenger.Task { - private final Message msg; + private Message msg; SequencedSendTask(Message msg) { this.msg = msg; } - @Override public void run() { sequencedSend(msg); } - @Override public void destroy() { msg.discard(); } + @Override public void run() { sequencedSend(msg); msg = null; } + @Override public void destroy() { if (msg != null) msg.discard(); } } private void sendNextInSequence(long seqId) { diff --git a/messagebus/src/test/java/com/yahoo/messagebus/SequencerTestCase.java b/messagebus/src/test/java/com/yahoo/messagebus/SequencerTestCase.java index f06ff4f5f73..b077de80467 100644 --- a/messagebus/src/test/java/com/yahoo/messagebus/SequencerTestCase.java +++ b/messagebus/src/test/java/com/yahoo/messagebus/SequencerTestCase.java @@ -6,8 +6,17 @@ import org.junit.jupiter.api.Test; import java.util.LinkedList; import java.util.Queue; - -import static org.junit.jupiter.api.Assertions.*; +import java.util.Set; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.Executor; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; /** * @author Simon Thoresen Hult @@ -102,6 +111,51 @@ public class SequencerTestCase { assertEquals(0, dst.size()); } + @Test + void testRecursiveSending() throws InterruptedException { + // This test queues up a lot of replies, and then has them all ready to return at once. + int n = 10000; + CountDownLatch latch = new CountDownLatch(n); + CountDownLatch started = new CountDownLatch(1); + AtomicReference<Reply> waiting = new AtomicReference<>(); + Executor executor = Executors.newSingleThreadExecutor(); + MessageHandler sender = message -> { + Runnable task = () -> { + Reply reply = new EmptyReply(); + reply.swapState(message); + reply.setMessage(message); + if (waiting.compareAndSet(null, reply)) started.countDown(); + else reply.popHandler().handleReply(reply); + }; + if (Math.random() < 0.5) executor.execute(task); // Usually, RPC thread runs this. + else task.run(); // But on, e.g., timeouts, it runs in the caller thread instead. + }; + + Queue<Message> answered = new ConcurrentLinkedQueue<>(); + ReplyHandler handler = reply -> { + answered.add(reply.getMessage()); + latch.countDown(); + }; + + Messenger messenger = new Messenger(); + messenger.start(); + Sequencer sequencer = new Sequencer(sender, messenger); // Not using the messenger results in a stack overflow error. + + Queue<Message> sent = new ConcurrentLinkedQueue<>(); + for (int i = 0; i < 10000; i++) { + Message message = new MyMessage(true, 1); + message.pushHandler(handler); + sequencer.handleMessage(message); + sent.add(message); + } + + assertTrue(started.await(10, TimeUnit.SECONDS)); + waiting.get().popHandler().handleReply(waiting.get()); + assertTrue(latch.await(10, TimeUnit.SECONDS), "All messages should obtain a reply within 10s"); + assertEquals(Set.copyOf(sent), Set.copyOf(answered)); // Order is not guaranteed at all! + messenger.destroy(); + } + private static class TestQueue extends LinkedList<Routable> implements ReplyHandler { void checkReply(boolean hasSeqId, long seqId) { diff --git a/metrics/src/main/java/ai/vespa/metrics/ControllerMetrics.java b/metrics/src/main/java/ai/vespa/metrics/ControllerMetrics.java index f03c54aa822..34cf2d98ef8 100644 --- a/metrics/src/main/java/ai/vespa/metrics/ControllerMetrics.java +++ b/metrics/src/main/java/ai/vespa/metrics/ControllerMetrics.java @@ -61,7 +61,12 @@ public enum ControllerMetrics implements VespaMetrics { METERING_MEMORY_GB("metering.memoryGB", Unit.GIGABYTE, "Controller: Metering memory GB"), METERING_VCPU("metering.vcpu", Unit.VCPU, "Controller: Metering VCPU"), METERING_LAST_REPORTED("metering_last_reported", Unit.SECONDS_SINCE_EPOCH, "Controller: Metering last reported"), - METERING_TOTAL_REPORTED("metering_total_reported", Unit.ITEM, "Controller: Metering total reported (sum of resources)"); + METERING_TOTAL_REPORTED("metering_total_reported", Unit.ITEM, "Controller: Metering total reported (sum of resources)"), + + MAIL_SENT("mail.sent", Unit.OPERATION, "Mail sent"), + MAIL_FAILED("mail.failed", Unit.OPERATION, "Mail delivery failed"), + MAIL_THROTTLED("mail.throttled", Unit.OPERATION, "Mail delivery throttled"); + private final String name; private final Unit unit; diff --git a/metrics/src/main/java/ai/vespa/metrics/docs/MetricSetDocumentation.java b/metrics/src/main/java/ai/vespa/metrics/docs/MetricSetDocumentation.java index b046d55c089..a15f2916091 100644 --- a/metrics/src/main/java/ai/vespa/metrics/docs/MetricSetDocumentation.java +++ b/metrics/src/main/java/ai/vespa/metrics/docs/MetricSetDocumentation.java @@ -45,8 +45,14 @@ public class MetricSetDocumentation { referenceBuilder.append(String.format(""" --- # Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + # Note: This file is generated by + # https://github.com/vespa-engine/vespa/blob/master/metrics/src/main/java/ai/vespa/metrics/docs/MetricSetDocumentation.java title: "%s Metric Set" - ---""", name)); + --- + <p> + This document provides reference documentation for the %s metric set, including suffixes present per metric. + If the suffix column contains "N/A" then the base name of the corresponding metric is used with no suffix. + </p>""", name, name)); metricsByType.keySet() .stream() .sorted() diff --git a/metrics/src/main/java/ai/vespa/metrics/set/InfrastructureMetricSet.java b/metrics/src/main/java/ai/vespa/metrics/set/InfrastructureMetricSet.java index 9443a08e28b..36750adb749 100644 --- a/metrics/src/main/java/ai/vespa/metrics/set/InfrastructureMetricSet.java +++ b/metrics/src/main/java/ai/vespa/metrics/set/InfrastructureMetricSet.java @@ -181,6 +181,10 @@ public class InfrastructureMetricSet { addMetric(metrics, ControllerMetrics.METERING_AGE_SECONDS.min()); addMetric(metrics, ControllerMetrics.METERING_LAST_REPORTED.max()); + addMetric(metrics, ControllerMetrics.MAIL_SENT.count()); + addMetric(metrics, ControllerMetrics.MAIL_FAILED.count()); + addMetric(metrics, ControllerMetrics.MAIL_THROTTLED.count()); + return metrics; } diff --git a/model-integration/src/main/java/ai/vespa/modelintegration/evaluator/OnnxEvaluator.java b/model-integration/src/main/java/ai/vespa/modelintegration/evaluator/OnnxEvaluator.java index 628fe933bf5..cd698eb1647 100644 --- a/model-integration/src/main/java/ai/vespa/modelintegration/evaluator/OnnxEvaluator.java +++ b/model-integration/src/main/java/ai/vespa/modelintegration/evaluator/OnnxEvaluator.java @@ -151,8 +151,8 @@ public class OnnxEvaluator implements AutoCloseable { throw new IllegalArgumentException("No such file: " + model.path().get()); } if (tryCuda && isCudaError(e) && !options.gpuDeviceRequired()) { - LOG.log(Level.WARNING, "Failed to create session with CUDA using GPU device " + - options.gpuDeviceNumber() + ". Falling back to CPU", e); + LOG.log(Level.INFO, "Failed to create session with CUDA using GPU device " + + options.gpuDeviceNumber() + ". Falling back to CPU. Reason: " + e.getMessage()); return createSession(model, runtime, options, false); } if (isCudaError(e)) { diff --git a/model-integration/src/main/java/ai/vespa/modelintegration/evaluator/OnnxEvaluatorOptions.java b/model-integration/src/main/java/ai/vespa/modelintegration/evaluator/OnnxEvaluatorOptions.java index 6dd2c5b05af..cefafc3654b 100644 --- a/model-integration/src/main/java/ai/vespa/modelintegration/evaluator/OnnxEvaluatorOptions.java +++ b/model-integration/src/main/java/ai/vespa/modelintegration/evaluator/OnnxEvaluatorOptions.java @@ -41,6 +41,7 @@ public class OnnxEvaluatorOptions { options.setExecutionMode(executionMode); options.setInterOpNumThreads(executionMode == PARALLEL ? interOpThreads : 1); options.setIntraOpNumThreads(intraOpThreads); + options.setCPUArenaAllocator(false); if (loadCuda) { options.addCUDA(gpuDeviceNumber); } diff --git a/model-integration/src/main/java/ai/vespa/modelintegration/evaluator/TensorConverter.java b/model-integration/src/main/java/ai/vespa/modelintegration/evaluator/TensorConverter.java index 952adc36621..2612702e99b 100644 --- a/model-integration/src/main/java/ai/vespa/modelintegration/evaluator/TensorConverter.java +++ b/model-integration/src/main/java/ai/vespa/modelintegration/evaluator/TensorConverter.java @@ -11,6 +11,7 @@ import ai.onnxruntime.OrtException; import ai.onnxruntime.OrtSession; import ai.onnxruntime.TensorInfo; import ai.onnxruntime.ValueInfo; +import ai.onnxruntime.platform.Fp16Conversions; import ai.vespa.rankingexpression.importer.onnx.OnnxImporter; import com.yahoo.tensor.DimensionSizes; import com.yahoo.tensor.IndexedTensor; @@ -56,7 +57,6 @@ class TensorConverter { throw new IllegalArgumentException("OnnxEvaluator currently only supports tensors with indexed dimensions"); } IndexedTensor tensor = (IndexedTensor) vespaTensor; - ByteBuffer buffer = ByteBuffer.allocateDirect((int)tensor.size() * onnxTensorInfo.type.size).order(ByteOrder.nativeOrder()); if (onnxTensorInfo.type == OnnxJavaType.FLOAT) { for (int i = 0; i < tensor.size(); i++) @@ -88,6 +88,17 @@ class TensorConverter { buffer.putLong((long) tensor.get(i)); return OnnxTensor.createTensor(environment, buffer.rewind().asLongBuffer(), tensor.shape()); } + if (onnxTensorInfo.type == OnnxJavaType.FLOAT16) { + for (int i = 0; i < tensor.size(); i++) { + buffer.putShort(Fp16Conversions.floatToFp16((float)tensor.get(i))); + } + return OnnxTensor.createTensor(environment, buffer.rewind(), tensor.shape(), OnnxJavaType.FLOAT16); + } + if (onnxTensorInfo.type == OnnxJavaType.BFLOAT16) { + for (int i = 0; i < tensor.size(); i++) + buffer.putShort(Fp16Conversions.floatToBf16((float)tensor.get(i))); + return OnnxTensor.createTensor(environment, buffer.rewind(), tensor.shape(), OnnxJavaType.BFLOAT16); + } throw new IllegalArgumentException("OnnxEvaluator does not currently support value type " + onnxTensorInfo.type); } @@ -132,6 +143,16 @@ class TensorConverter { for (long i = 0; i < sizes.totalSize(); i++) builder.cellByDirectIndex(i, buffer.get()); } + else if (tensorInfo.type == OnnxJavaType.FLOAT16) { + ShortBuffer buffer = onnxTensor.getShortBuffer(); + for (long i = 0; i < sizes.totalSize(); i++) + builder.cellByDirectIndex(i, Fp16Conversions.fp16ToFloat(buffer.get())); + } + else if (tensorInfo.type == OnnxJavaType.BFLOAT16) { + ShortBuffer buffer = onnxTensor.getShortBuffer(); + for (long i = 0; i < sizes.totalSize(); i++) + builder.cellByDirectIndex(i, Fp16Conversions.bf16ToFloat((buffer.get()))); + } else { throw new IllegalArgumentException("OnnxEvaluator does not currently support value type " + onnxTensor.getInfo().type); } @@ -183,6 +204,7 @@ class TensorConverter { switch (onnxType) { case ONNX_TENSOR_ELEMENT_DATA_TYPE_INT8: return TensorType.Value.INT8; case ONNX_TENSOR_ELEMENT_DATA_TYPE_BFLOAT16: return TensorType.Value.BFLOAT16; + case ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT16: return TensorType.Value.FLOAT; case ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT: return TensorType.Value.FLOAT; case ONNX_TENSOR_ELEMENT_DATA_TYPE_DOUBLE: return TensorType.Value.DOUBLE; } diff --git a/model-integration/src/test/java/ai/vespa/modelintegration/evaluator/OnnxEvaluatorTest.java b/model-integration/src/test/java/ai/vespa/modelintegration/evaluator/OnnxEvaluatorTest.java index db2e9db1277..75da4d163bb 100644 --- a/model-integration/src/test/java/ai/vespa/modelintegration/evaluator/OnnxEvaluatorTest.java +++ b/model-integration/src/test/java/ai/vespa/modelintegration/evaluator/OnnxEvaluatorTest.java @@ -9,9 +9,16 @@ import org.junit.Test; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Paths; +import java.util.ArrayList; import java.util.HashMap; +import java.util.List; import java.util.Map; +import java.util.logging.Level; +import java.util.logging.Logger; +import java.util.logging.Handler; +import java.util.logging.LogRecord; + import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import static org.junit.Assume.assumeTrue; @@ -83,12 +90,14 @@ public class OnnxEvaluatorTest { var runtime = new OnnxRuntime(); assertEvaluate(runtime, "add_double.onnx", "tensor(d0[1]):[3]", "tensor(d0[1]):[1]", "tensor(d0[1]):[2]"); assertEvaluate(runtime, "add_float.onnx", "tensor<float>(d0[1]):[3]", "tensor<float>(d0[1]):[1]", "tensor<float>(d0[1]):[2]"); + assertEvaluate(runtime, "add_float16.onnx", "tensor<float>(d0[1]):[3]", "tensor<float>(d0[1]):[1]", "tensor<float>(d0[1]):[2]"); + //Add is not a supported operation for bfloat16 types in onnx operators. + assertEvaluate(runtime, "sign_bfloat16.onnx", "tensor<bfloat16>(d0[1]):[1]", "tensor<bfloat16>(d0[1]):[1]"); + assertEvaluate(runtime, "add_int64.onnx", "tensor<double>(d0[1]):[3]", "tensor<double>(d0[1]):[1]", "tensor<double>(d0[1]):[2]"); assertEvaluate(runtime, "cast_int8_float.onnx", "tensor<float>(d0[1]):[-128]", "tensor<int8>(d0[1]):[128]"); assertEvaluate(runtime, "cast_float_int8.onnx", "tensor<int8>(d0[1]):[-1]", "tensor<float>(d0[1]):[255]"); - - // ONNX Runtime 1.8.0 does not support much of bfloat16 yet - // assertEvaluate("cast_bfloat16_float.onnx", "tensor<float>(d0[1]):[1]", "tensor<bfloat16>(d0[1]):[1]"); + assertEvaluate(runtime,"cast_bfloat16_float.onnx", "tensor<float>(d0[1]):[1]", "tensor<bfloat16>(d0[1]):[1]"); } @Test @@ -170,6 +179,29 @@ public class OnnxEvaluatorTest { evaluator.close(); } + @Test + public void testLoggingMessages() throws IOException { + assumeTrue(OnnxRuntime.isRuntimeAvailable()); + Logger logger = Logger.getLogger(OnnxEvaluator.class.getName()); + CustomLogHandler logHandler = new CustomLogHandler(); + logger.addHandler(logHandler); + var runtime = new OnnxRuntime(); + var model = Files.readAllBytes(Paths.get("src/test/models/onnx/simple/simple.onnx")); + OnnxEvaluatorOptions options = new OnnxEvaluatorOptions(); + options.setGpuDevice(0); + var evaluator = runtime.evaluatorOf(model,options); + evaluator.close(); + List<LogRecord> records = logHandler.getLogRecords(); + assertEquals(1,records.size()); + assertEquals(Level.INFO,records.get(0).getLevel()); + String message = records.get(0).getMessage(); + assertEquals("Failed to create session with CUDA using GPU device 0. " + + "Falling back to CPU. Reason: Error code - ORT_EP_FAIL - message:" + + " Failed to find CUDA shared provider", message); + logger.removeHandler(logHandler); + + } + private void assertEvaluate(OnnxRuntime runtime, String model, String output, String... input) { OnnxEvaluator evaluator = runtime.evaluatorOf("src/test/models/onnx/" + model); Map<String, Tensor> inputs = new HashMap<>(); @@ -182,4 +214,25 @@ public class OnnxEvaluatorTest { assertEquals(expected.type().valueType(), result.type().valueType()); } + static class CustomLogHandler extends Handler { + private List<LogRecord> records = new ArrayList<>(); + + @Override + public void publish(LogRecord record) { + records.add(record); + } + + @Override + public void flush() { + } + + @Override + public void close() throws SecurityException { + } + + public List<LogRecord> getLogRecords() { + return records; + } + } + } diff --git a/model-integration/src/test/models/onnx/add_float16.onnx b/model-integration/src/test/models/onnx/add_float16.onnx new file mode 100644 index 00000000000..df0f7fdcdba --- /dev/null +++ b/model-integration/src/test/models/onnx/add_float16.onnx @@ -0,0 +1,19 @@ +add_float16.py:f + +input1 +input2output"AddaddZ +input1 + + + +Z +input2 + + + +b +output + + + +B
\ No newline at end of file diff --git a/model-integration/src/test/models/onnx/add_float16.py b/model-integration/src/test/models/onnx/add_float16.py new file mode 100755 index 00000000000..a637cf8b0dd --- /dev/null +++ b/model-integration/src/test/models/onnx/add_float16.py @@ -0,0 +1,27 @@ +# Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +import onnx +from onnx import helper, TensorProto + +INPUT_1 = helper.make_tensor_value_info('input1', TensorProto.FLOAT16, [1]) +INPUT_2 = helper.make_tensor_value_info('input2', TensorProto.FLOAT16, [1]) +OUTPUT = helper.make_tensor_value_info('output', TensorProto.FLOAT16, [1]) + +nodes = [ + helper.make_node( + 'Add', + ['input1', 'input2'], + ['output'], + ), +] +graph_def = helper.make_graph( + nodes, + 'add', + [ + INPUT_1, + INPUT_2 + ], + [OUTPUT], +) +model_def = helper.make_model(graph_def, producer_name='add_float16.py', opset_imports=[onnx.OperatorSetIdProto(version=12)]) +onnx.save(model_def, 'add_float16.onnx') diff --git a/model-integration/src/test/models/onnx/cast_bfloat16_float.onnx b/model-integration/src/test/models/onnx/cast_bfloat16_float.onnx index cb19592abf4..9fcbd7f1b3c 100644 --- a/model-integration/src/test/models/onnx/cast_bfloat16_float.onnx +++ b/model-integration/src/test/models/onnx/cast_bfloat16_float.onnx @@ -1,4 +1,4 @@ -cast_bfloat16_float.py:U +cast_bfloat16_float.py:U ! input1output"Cast* to castZ @@ -9,4 +9,4 @@ output -B
\ No newline at end of file +B
\ No newline at end of file diff --git a/model-integration/src/test/models/onnx/cast_bfloat16_float.py b/model-integration/src/test/models/onnx/cast_bfloat16_float.py index 51d04747958..952e4c469c1 100755 --- a/model-integration/src/test/models/onnx/cast_bfloat16_float.py +++ b/model-integration/src/test/models/onnx/cast_bfloat16_float.py @@ -20,5 +20,5 @@ graph_def = helper.make_graph( [INPUT_1], [OUTPUT], ) -model_def = helper.make_model(graph_def, producer_name='cast_bfloat16_float.py', opset_imports=[onnx.OperatorSetIdProto(version=12)]) +model_def = helper.make_model(graph_def, producer_name='cast_bfloat16_float.py', opset_imports=[onnx.OperatorSetIdProto(version=19)]) onnx.save(model_def, 'cast_bfloat16_float.onnx') diff --git a/model-integration/src/test/models/onnx/sign_bfloat16.onnx b/model-integration/src/test/models/onnx/sign_bfloat16.onnx new file mode 100644 index 00000000000..176451108ba --- /dev/null +++ b/model-integration/src/test/models/onnx/sign_bfloat16.onnx @@ -0,0 +1,11 @@ +sign_bfloat16.py:J + +input1output"SignsignZ +input1 + + +b +output + + +B
\ No newline at end of file diff --git a/model-integration/src/test/models/onnx/sign_bfloat16.py b/model-integration/src/test/models/onnx/sign_bfloat16.py new file mode 100755 index 00000000000..b74e48e91c8 --- /dev/null +++ b/model-integration/src/test/models/onnx/sign_bfloat16.py @@ -0,0 +1,25 @@ +# Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +import onnx +from onnx import helper, TensorProto + +INPUT_1 = helper.make_tensor_value_info('input1', TensorProto.BFLOAT16, [1]) +OUTPUT = helper.make_tensor_value_info('output', TensorProto.BFLOAT16, [1]) + +nodes = [ + helper.make_node( + 'Sign', + ['input1'], + ['output'], + ), +] +graph_def = helper.make_graph( + nodes, + 'sign', + [ + INPUT_1 + ], + [OUTPUT], +) +model_def = helper.make_model(graph_def, producer_name='sign_bfloat16.py', opset_imports=[onnx.OperatorSetIdProto(version=19)]) +onnx.save(model_def, 'sign_bfloat16.onnx') diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/SyncFileInfo.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/SyncFileInfo.java index c722779c3c6..c65f2abb6fd 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/SyncFileInfo.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/SyncFileInfo.java @@ -84,6 +84,13 @@ public class SyncFileInfo { } else if (filename.startsWith("start-services.out-")) { compression = Compression.ZSTD; dir = "logs/start-services/"; + } else if (filename.startsWith("nginx-error")) { + compression = Compression.ZSTD; + if ("nginx-error.log".equals(filename)) { + if (!rotatedOnly) remoteFilename = "nginx-error.log"; + minDurationBetweenSync = rotatedOnly ? Duration.ofHours(1) : Duration.ZERO; + } + dir = "logs/nginx/"; } else { compression = filename.endsWith(".zst") ? Compression.NONE : Compression.ZSTD; if (rotatedOnly && compression != Compression.NONE) diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java index c3ec9e91343..43dc3d72c46 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java @@ -518,6 +518,8 @@ public class NodeAgentImpl implements NodeAgent { container = Optional.of(updateContainerIfNeeded(context, container.get())); } + serviceDumper.processServiceDumpRequest(context); + startServicesIfNeeded(context); resumeNodeIfNeeded(context); if (healthChecker.isPresent()) { @@ -531,7 +533,6 @@ public class NodeAgentImpl implements NodeAgent { throw ConvergenceException.ofTransient("Refusing to resume until warm up period ends (" + (timeLeft.isNegative() ? "next tick" : "in " + timeLeft) + ")"); } - serviceDumper.processServiceDumpRequest(context); // Because it's more important to stop a bad release from rolling out in prod, // we put the resume call last. So if we fail after updating the node repo attributes diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/SyncFileInfoTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/SyncFileInfoTest.java index f3a25778459..8e56741274e 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/SyncFileInfoTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/SyncFileInfoTest.java @@ -40,6 +40,9 @@ public class SyncFileInfoTest { private static final Path zkLogPath1 = fileSystem.getPath("/opt/vespa/logs/zookeeper.configserver.1.log"); private static final Path startServicesPath1 = fileSystem.getPath("/opt/vespa/logs/start-services.out"); private static final Path startServicesPath2 = fileSystem.getPath("/opt/vespa/logs/start-services.out-20230808100143"); + private static final Path rotatedNginxErrorLog = fileSystem.getPath("/opt/vespa/logs/nginx/nginx-error.log.20231019-1234555"); + private static final Path currentNginxErrorLog = fileSystem.getPath("/opt/vespa/logs/nginx/nginx-error.log"); + private static final Path nginxAccessLog = fileSystem.getPath("/opt/vespa/logs/nginx/nginx-access.log.20231019-1234"); @Test void access_logs() { @@ -96,6 +99,22 @@ public class SyncFileInfoTest { } @Test + void nginx_error_logs() { + new UnixPath(currentNginxErrorLog).createParents().createNewFile().setLastModifiedTime(Instant.parse("2022-05-09T14:22:11Z")); + assertForLogFile(currentNginxErrorLog, "s3://vespa-data-bucket/vespa/music/main/h432a/logs/nginx/nginx-error.log.zst", ZSTD, Duration.ofHours(1),true); + assertForLogFile(currentNginxErrorLog, "s3://vespa-data-bucket/vespa/music/main/h432a/logs/nginx/nginx-error.log.zst", ZSTD, Duration.ZERO,false); + + new UnixPath(rotatedNginxErrorLog).createParents().createNewFile().setLastModifiedTime(Instant.parse("2022-05-09T14:22:11Z")); + assertForLogFile(rotatedNginxErrorLog, "s3://vespa-data-bucket/vespa/music/main/h432a/logs/nginx/nginx-error.log.20231019-1234555.zst", ZSTD, true); + assertForLogFile(rotatedNginxErrorLog, "s3://vespa-data-bucket/vespa/music/main/h432a/logs/nginx/nginx-error.log.20231019-1234555.zst", ZSTD, false); + + // Does not sync access logs + new UnixPath(nginxAccessLog).createParents().createNewFile().setLastModifiedTime(Instant.parse("2022-05-09T14:22:11Z")); + Optional<SyncFileInfo> sfi = SyncFileInfo.forLogFile(nodeArchiveUri, nginxAccessLog, false, ApplicationId.defaultId()); + assertEquals(Optional.empty(), sfi); + } + + @Test void start_services() { assertForLogFile(startServicesPath1, null, null, true); assertForLogFile(startServicesPath2, "s3://vespa-data-bucket/vespa/music/main/h432a/logs/start-services/start-services.out-20230808100143.zst", ZSTD, true); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java index 44e9e0a0d42..567a5c03f43 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java @@ -113,7 +113,7 @@ public final class Node implements Nodelike { this.modelName = Objects.requireNonNull(modelName, "A null modelName is not permitted"); this.reservedTo = Objects.requireNonNull(reservedTo, "reservedTo cannot be null"); this.exclusiveToApplicationId = Objects.requireNonNull(exclusiveToApplicationId, "exclusiveToApplicationId cannot be null"); - this.provisionedForApplicationId = Objects.requireNonNull(exclusiveToApplicationId, "provisionedForApplicationId cannot be null"); + this.provisionedForApplicationId = Objects.requireNonNull(provisionedForApplicationId, "provisionedForApplicationId cannot be null"); this.hostTTL = Objects.requireNonNull(hostTTL, "hostTTL cannot be null"); this.hostEmptyAt = Objects.requireNonNull(hostEmptyAt, "hostEmptyAt cannot be null"); this.exclusiveToClusterType = Objects.requireNonNull(exclusiveToClusterType, "exclusiveToClusterType cannot be null"); @@ -494,7 +494,7 @@ public final class Node implements Nodelike { public Node withExclusiveToApplicationId(ApplicationId exclusiveTo) { return new Node(id, extraId, ipConfig, hostname, parentHostname, flavor, status, state, allocation, history, - type, reports, modelName, reservedTo, Optional.ofNullable(exclusiveTo), provisionedForApplicationId.filter(__ -> exclusiveTo != null), hostTTL, hostEmptyAt, + type, reports, modelName, reservedTo, Optional.ofNullable(exclusiveTo), provisionedForApplicationId, hostTTL, hostEmptyAt, exclusiveToClusterType, switchHostname, trustStoreItems, cloudAccount, wireguardPubKey); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java index 83db3712c17..449e1c07bf8 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java @@ -212,6 +212,11 @@ public class NodeRepository extends AbstractComponent { ( !zone().cloud().allowHostSharing() && !sharedHosts.value().isEnabled(clusterSpec.type().name())); } + /** Whether the nodes of this cluster must be running on hosts that are specifically provisioned for the application. */ + public boolean exclusiveProvisioning(ClusterSpec clusterSpec) { + return !zone.cloud().allowHostSharing() && clusterSpec.isExclusive(); + } + /** * Returns ACLs for the children of the given host. * diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Applications.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Applications.java index 5b0180bad43..e68b2f6103b 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Applications.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Applications.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.hosted.provision.applications; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ApplicationLockException; import com.yahoo.config.provision.ApplicationTransaction; +import com.yahoo.config.provision.ApplicationMutex; import com.yahoo.transaction.Mutex; import com.yahoo.transaction.NestedTransaction; import com.yahoo.vespa.hosted.provision.persistence.CuratorDb; @@ -64,18 +65,18 @@ public class Applications { } /** Create a lock which provides exclusive rights to making changes to the given application */ - public Mutex lock(ApplicationId application) { - return db.lock(application); + public ApplicationMutex lock(ApplicationId application) { + return new ApplicationMutex(application, db.lock(application)); } /** Create a lock with a timeout which provides exclusive rights to making changes to the given application */ - public Mutex lock(ApplicationId application, Duration timeout) { - return db.lock(application, timeout); + public ApplicationMutex lock(ApplicationId application, Duration timeout) { + return new ApplicationMutex(application, db.lock(application, timeout)); } /** Create a lock which provides exclusive rights to perform a maintenance deployment */ - public Mutex lockMaintenance(ApplicationId application) { - return db.lockMaintenance(application); + public ApplicationMutex lockMaintenance(ApplicationId application) { + return new ApplicationMutex(application, db.lockMaintenance(application)); } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirer.java index 0f1d3e93cc3..c3fea72fab9 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirer.java @@ -99,7 +99,8 @@ public class FailedExpirer extends NodeRepositoryMaintainer { .map(Node::hostname) .toList(); if (unparkedChildren.isEmpty()) { - return Optional.of(nodeRepository.nodes().park(node.hostname(), true, Agent.FailedExpirer, + // Only forcing de-provisioning of off premises nodes + return Optional.of(nodeRepository.nodes().park(node.hostname(), nodeRepository.zone().cloud().dynamicProvisioning(), Agent.FailedExpirer, "Parked by FailedExpirer due to " + reason.get())); } else { log.info(String.format("Expired failed node %s was not parked because of unparked children: %s", diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/HostCapacityMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/HostCapacityMaintainer.java index 25cfcf2cda9..3c42972ee0b 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/HostCapacityMaintainer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/HostCapacityMaintainer.java @@ -13,7 +13,10 @@ import com.yahoo.config.provision.NodeType; import com.yahoo.jdisc.Metric; import com.yahoo.lang.MutableInteger; import com.yahoo.transaction.Mutex; +import com.yahoo.vespa.flags.BooleanFlag; +import com.yahoo.vespa.flags.FetchVector; import com.yahoo.vespa.flags.FlagSource; +import com.yahoo.vespa.flags.Flags; import com.yahoo.vespa.flags.ListFlag; import com.yahoo.vespa.flags.PermanentFlags; import com.yahoo.vespa.flags.custom.ClusterCapacity; @@ -59,6 +62,7 @@ public class HostCapacityMaintainer extends NodeRepositoryMaintainer { private final HostProvisioner hostProvisioner; private final ListFlag<ClusterCapacity> preprovisionCapacityFlag; + private final BooleanFlag makeExclusiveFlag; private final ProvisioningThrottler throttler; HostCapacityMaintainer(NodeRepository nodeRepository, @@ -69,6 +73,7 @@ public class HostCapacityMaintainer extends NodeRepositoryMaintainer { super(nodeRepository, interval, metric); this.hostProvisioner = hostProvisioner; this.preprovisionCapacityFlag = PermanentFlags.PREPROVISION_CAPACITY.bindTo(flagSource); + this.makeExclusiveFlag = Flags.MAKE_EXCLUSIVE.bindTo(flagSource); this.throttler = new ProvisioningThrottler(nodeRepository, metric); } @@ -187,6 +192,11 @@ public class HostCapacityMaintainer extends NodeRepositoryMaintainer { */ private List<Node> provisionUntilNoDeficit(NodeList nodeList) { List<ClusterCapacity> preprovisionCapacity = preprovisionCapacityFlag.value(); + ApplicationId application = ApplicationId.defaultId(); + boolean makeExclusive = makeExclusiveFlag.with(FetchVector.Dimension.TENANT_ID, application.tenant().value()) + .with(FetchVector.Dimension.INSTANCE_ID, application.serializedForm()) + .with(FetchVector.Dimension.VESPA_VERSION, Vtag.currentVersion.toFullString()) + .value(); // Worst-case each ClusterCapacity in preprovisionCapacity will require an allocation. int maxProvisions = preprovisionCapacity.size(); @@ -194,7 +204,7 @@ public class HostCapacityMaintainer extends NodeRepositoryMaintainer { var nodesPlusProvisioned = new ArrayList<>(nodeList.asList()); for (int numProvisions = 0;; ++numProvisions) { var nodesPlusProvisionedPlusAllocated = new ArrayList<>(nodesPlusProvisioned); - Optional<ClusterCapacity> deficit = allocatePreprovisionCapacity(preprovisionCapacity, nodesPlusProvisionedPlusAllocated); + Optional<ClusterCapacity> deficit = allocatePreprovisionCapacity(application, preprovisionCapacity, nodesPlusProvisionedPlusAllocated, makeExclusive); if (deficit.isEmpty()) { return nodesPlusProvisionedPlusAllocated; } @@ -204,32 +214,39 @@ public class HostCapacityMaintainer extends NodeRepositoryMaintainer { } ClusterCapacity clusterCapacityDeficit = deficit.get(); - var clusterType = Optional.ofNullable(clusterCapacityDeficit.clusterType()); nodesPlusProvisioned.addAll(provisionHosts(clusterCapacityDeficit.count(), toNodeResources(clusterCapacityDeficit), - clusterType.map(ClusterSpec.Type::from), + Optional.ofNullable(clusterCapacityDeficit.clusterType()), nodeList)); } } - private List<Node> provisionHosts(int count, NodeResources nodeResources, Optional<ClusterSpec.Type> clusterType, NodeList allNodes) { + private List<Node> provisionHosts(int count, NodeResources nodeResources, Optional<String> clusterType, NodeList allNodes) { try { if (throttler.throttle(allNodes, Agent.HostCapacityMaintainer)) { throw new NodeAllocationException("Host provisioning is being throttled", true); } Version osVersion = nodeRepository().osVersions().targetFor(NodeType.host).orElse(Version.emptyVersion); List<Integer> provisionIndices = nodeRepository().database().readProvisionIndices(count); + HostSharing sharingMode = nodeRepository().exclusiveAllocation(asSpec(clusterType, 0)) ? HostSharing.exclusive : HostSharing.shared; HostProvisionRequest request = new HostProvisionRequest(provisionIndices, NodeType.host, nodeResources, ApplicationId.defaultId(), osVersion, - HostSharing.shared, clusterType, Optional.empty(), + sharingMode, clusterType.map(ClusterSpec.Type::valueOf), Optional.empty(), nodeRepository().zone().cloud().account(), false); List<Node> hosts = new ArrayList<>(); - hostProvisioner.provisionHosts(request, - resources -> true, - provisionedHosts -> { - hosts.addAll(provisionedHosts.stream().map(host -> host.generateHost(Duration.ZERO)).toList()); - nodeRepository().nodes().addNodes(hosts, Agent.HostCapacityMaintainer); - }); + Runnable waiter; + try (var lock = nodeRepository().nodes().lockUnallocated()) { + waiter = hostProvisioner.provisionHosts(request, + resources -> true, + provisionedHosts -> { + hosts.addAll(provisionedHosts.stream() + .map(host -> host.generateHost(Duration.ZERO)) + .map(host -> host.withExclusiveToApplicationId(null)) + .toList()); + nodeRepository().nodes().addNodes(hosts, Agent.HostCapacityMaintainer); + }); + } + waiter.run(); return hosts; } catch (NodeAllocationException | IllegalArgumentException | IllegalStateException e) { throw new NodeAllocationException("Failed to provision " + count + " " + nodeResources + ": " + e.getMessage(), @@ -246,12 +263,14 @@ public class HostCapacityMaintainer extends NodeRepositoryMaintainer { * they are added to {@code mutableNodes} * @return the part of a cluster capacity it was unable to allocate, if any */ - private Optional<ClusterCapacity> allocatePreprovisionCapacity(List<ClusterCapacity> preprovisionCapacity, - ArrayList<Node> mutableNodes) { + private Optional<ClusterCapacity> allocatePreprovisionCapacity(ApplicationId application, + List<ClusterCapacity> preprovisionCapacity, + ArrayList<Node> mutableNodes, + boolean makeExclusive) { for (int clusterIndex = 0; clusterIndex < preprovisionCapacity.size(); ++clusterIndex) { ClusterCapacity clusterCapacity = preprovisionCapacity.get(clusterIndex); LockedNodeList allNodes = new LockedNodeList(mutableNodes, () -> {}); - List<Node> candidates = findCandidates(clusterCapacity, clusterIndex, allNodes); + List<Node> candidates = findCandidates(application, clusterCapacity, clusterIndex, allNodes, makeExclusive); int deficit = Math.max(0, clusterCapacity.count() - candidates.size()); if (deficit > 0) { return Optional.of(clusterCapacity.withCount(deficit)); @@ -264,40 +283,50 @@ public class HostCapacityMaintainer extends NodeRepositoryMaintainer { return Optional.empty(); } - private List<Node> findCandidates(ClusterCapacity clusterCapacity, int clusterIndex, LockedNodeList allNodes) { + private List<Node> findCandidates(ApplicationId application, ClusterCapacity clusterCapacity, int clusterIndex, LockedNodeList allNodes, boolean makeExclusive) { NodeResources nodeResources = toNodeResources(clusterCapacity); // We'll allocate each ClusterCapacity as a unique cluster in a dummy application - ApplicationId applicationId = ApplicationId.defaultId(); - ClusterSpec.Id clusterId = ClusterSpec.Id.from(String.valueOf(clusterIndex)); - ClusterSpec.Type type = clusterCapacity.clusterType() != null - ? ClusterSpec.Type.from(clusterCapacity.clusterType()) - : ClusterSpec.Type.content; - ClusterSpec clusterSpec = ClusterSpec.request(type, clusterId) - // build() requires a version, even though it is not (should not be) used - .vespaVersion(Vtag.currentVersion) - .build(); + ClusterSpec cluster = asSpec(Optional.ofNullable(clusterCapacity.clusterType()), clusterIndex); NodeSpec nodeSpec = NodeSpec.from(clusterCapacity.count(), 1, nodeResources, false, true, nodeRepository().zone().cloud().account(), Duration.ZERO); var allocationContext = IP.Allocation.Context.from(nodeRepository().zone().cloud().name(), nodeSpec.cloudAccount().isExclave(nodeRepository().zone()), nodeRepository().nameResolver()); - NodePrioritizer prioritizer = new NodePrioritizer(allNodes, applicationId, clusterSpec, nodeSpec, - true, allocationContext, nodeRepository().nodes(), nodeRepository().resourcesCalculator(), - nodeRepository().spareCount()); - List<NodeCandidate> nodeCandidates = prioritizer.collect(); + NodePrioritizer prioritizer = new NodePrioritizer(allNodes, application, cluster, nodeSpec, + true, false, allocationContext, nodeRepository().nodes(), + nodeRepository().resourcesCalculator(), nodeRepository().spareCount(), + nodeRepository().exclusiveAllocation(cluster), makeExclusive); + List<NodeCandidate> nodeCandidates = prioritizer.collect() + .stream() + .filter(node -> node.violatesExclusivity(cluster, + application, + nodeRepository().exclusiveAllocation(cluster), + false, + nodeRepository().zone().cloud().allowHostSharing(), + allNodes, + makeExclusive) + != NodeCandidate.ExclusivityViolation.YES) + .toList(); MutableInteger index = new MutableInteger(0); return nodeCandidates .stream() .limit(clusterCapacity.count()) .map(candidate -> candidate.toNode() - .allocate(applicationId, - ClusterMembership.from(clusterSpec, index.next()), + .allocate(application, + ClusterMembership.from(cluster, index.next()), nodeResources, nodeRepository().clock().instant())) .toList(); } + private static ClusterSpec asSpec(Optional<String> clusterType, int index) { + return ClusterSpec.request(clusterType.map(ClusterSpec.Type::from).orElse(ClusterSpec.Type.content), + ClusterSpec.Id.from(String.valueOf(index))) + .vespaVersion(Vtag.currentVersion) // Needed, but should not be used here. + .build(); + } + private static NodeResources toNodeResources(ClusterCapacity clusterCapacity) { return new NodeResources(clusterCapacity.vcpu(), clusterCapacity.memoryGb(), diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/HostResumeProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/HostResumeProvisioner.java index 1d459f7dd14..2e3c6d1755a 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/HostResumeProvisioner.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/HostResumeProvisioner.java @@ -61,10 +61,13 @@ public class HostResumeProvisioner extends NodeRepositoryMaintainer { log.log(Level.INFO, "Could not provision " + host.hostname() + ", will retry in " + interval() + ": " + Exceptions.toMessageString(e)); } catch (FatalProvisioningException e) { + // FatalProvisioningException is thrown if node is not found in the cloud, allow for + // some time for the state to propagate + if (host.history().age(clock().instant()).getSeconds() < 30) continue; failures++; log.log(Level.SEVERE, "Failed to provision " + host.hostname() + ", failing out the host recursively", e); - nodeRepository().nodes().failOrMarkRecursively( - host.hostname(), Agent.HostResumeProvisioner, "Failed by HostResumeProvisioner due to provisioning failure"); + nodeRepository().nodes().parkRecursively( + host.hostname(), Agent.HostResumeProvisioner, true, "Failed by HostResumeProvisioner due to provisioning failure"); } catch (RuntimeException e) { if (e.getCause() instanceof NamingException) log.log(Level.INFO, "Could not provision " + host.hostname() + ", will retry in " + interval() + ": " + Exceptions.toMessageString(e)); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/InfraApplicationRedeployer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/InfraApplicationRedeployer.java new file mode 100644 index 00000000000..434e6161a4c --- /dev/null +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/InfraApplicationRedeployer.java @@ -0,0 +1,117 @@ +package com.yahoo.vespa.hosted.provision.maintenance; + +import com.yahoo.component.annotation.Inject; +import com.yahoo.concurrent.DaemonThreadFactory; +import com.yahoo.concurrent.UncheckedTimeoutException; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.Deployment; +import com.yahoo.config.provision.InfraDeployer; +import com.yahoo.config.provision.NodeType; +import com.yahoo.transaction.Mutex; +import com.yahoo.vespa.applicationmodel.InfrastructureApplication; +import com.yahoo.vespa.hosted.provision.Node.State; +import com.yahoo.vespa.hosted.provision.NodeList; +import com.yahoo.vespa.hosted.provision.NodeRepository; + +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.ConcurrentSkipListSet; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.function.Function; +import java.util.function.Supplier; +import java.util.logging.Logger; + +import static java.util.logging.Level.FINE; +import static java.util.logging.Level.INFO; +import static java.util.logging.Level.WARNING; + +/** + * Performs on-demand redeployment of the {@link InfrastructureApplication}s, to minimise time between + * host provisioning for a deployment completing, and deployment of its application containers succeeding. + * + * @author jonmv + */ +public class InfraApplicationRedeployer implements AutoCloseable { + + private static final Logger log = Logger.getLogger(InfraApplicationRedeployer.class.getName()); + + private final ExecutorService executor = Executors.newSingleThreadExecutor(new DaemonThreadFactory("infra-application-redeployer-")); + private final Set<InfrastructureApplication> readiedTypes = new ConcurrentSkipListSet<>(); + private final InfraDeployer deployer; + private final Function<ApplicationId, Mutex> locks; + private final Supplier<NodeList> nodes; + + @Inject + public InfraApplicationRedeployer(InfraDeployer deployer, NodeRepository nodes) { + this(deployer, nodes.applications()::lockMaintenance, nodes.nodes()::list); + } + + InfraApplicationRedeployer(InfraDeployer deployer, Function<ApplicationId, Mutex> locks, Supplier<NodeList> nodes) { + this.deployer = deployer; + this.locks = locks; + this.nodes = nodes; + } + + public void readied(NodeType type) { + applicationOf(type).ifPresent(this::readied); + } + + private void readied(InfrastructureApplication application) { + if (application == null) return; + if (readiedTypes.add(application)) executor.execute(() -> checkAndRedeploy(application)); + } + + private void checkAndRedeploy(InfrastructureApplication application) { + if ( ! readiedTypes.remove(application)) return; + try (Mutex lock = locks.apply(application.id())) { + if (application.nodeType().isHost() && nodes.get().state(State.ready).nodeType(application.nodeType()).isEmpty()) return; + log.log(FINE, () -> "Redeploying " + application.id() + " after completing provisioning for " + application.name()); + try { + deployer.getDeployment(application.id()).ifPresent(Deployment::activate); + childOf(application).ifPresent(this::readied); + } + catch (RuntimeException e) { + log.log(INFO, "Failed redeploying " + application.id() + ", will be retried by maintainer", e); + } + } + catch (UncheckedTimeoutException collision) { + readied(application); + } + } + + private static Optional<InfrastructureApplication> applicationOf(NodeType type) { + return switch (type) { + case host -> Optional.of(InfrastructureApplication.TENANT_HOST); + case confighost -> Optional.of(InfrastructureApplication.CONFIG_SERVER_HOST); + case config -> Optional.of(InfrastructureApplication.CONFIG_SERVER); + case controllerhost -> Optional.of(InfrastructureApplication.CONTROLLER_HOST); + case controller -> Optional.of(InfrastructureApplication.CONTROLLER); + case proxyhost -> Optional.of(InfrastructureApplication.PROXY_HOST); + default -> Optional.empty(); + }; + } + + private static Optional<InfrastructureApplication> childOf(InfrastructureApplication application) { + return switch (application) { + case CONFIG_SERVER_HOST -> Optional.of(InfrastructureApplication.CONFIG_SERVER); + case CONTROLLER_HOST -> Optional.of(InfrastructureApplication.CONTROLLER); + default -> Optional.empty(); + }; + } + + @Override + public void close() { + executor.shutdown(); + try { + if (executor.awaitTermination(10, TimeUnit.SECONDS)) return; + log.log(WARNING, "Redeployer did not shut down within 10 seconds"); + } + catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + executor.shutdownNow(); + } + +} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java index 23e7fe16797..6c4be09c489 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java @@ -111,7 +111,7 @@ public class NodeFailer extends NodeRepositoryMaintainer { for (Node node : activeNodes) { Instant graceTimeStart = clock().instant().minus(nodeRepository().nodes().suspended(node) ? suspendedDownTimeLimit : downTimeLimit); - if (node.isDown() && node.history().hasEventBefore(History.Event.Type.down, graceTimeStart) && !applicationSuspended(node) && !undergoingCmr(node)) { + if (node.isDown() && node.history().hasEventBefore(History.Event.Type.down, graceTimeStart) && !applicationSuspended(node) && !affectedByMaintenance(node)) { // Allow a grace period after node re-activation if (!node.history().hasEventAfter(History.Event.Type.activated, graceTimeStart)) failingNodes.add(new FailingNode(node, "Node has been down longer than " + downTimeLimit)); @@ -146,7 +146,7 @@ public class NodeFailer extends NodeRepositoryMaintainer { /** Returns whether node has any kind of hardware issue */ static boolean hasHardwareIssue(Node node, NodeList allNodes) { Node host = node.parentHostname().flatMap(allNodes::node).orElse(node); - return reasonsToFailHost(host).size() > 0; + return !reasonsToFailHost(host).isEmpty(); } private boolean applicationSuspended(Node node) { @@ -159,17 +159,18 @@ public class NodeFailer extends NodeRepositoryMaintainer { } } - private boolean undergoingCmr(Node node) { + /** Is a maintenance event affecting this node? */ + private boolean affectedByMaintenance(Node node) { return node.reports().getReport("vcmr") - .map(report -> - SlimeUtils.entriesStream(report.getInspector().field("upcoming")) - .anyMatch(cmr -> { - var startTime = cmr.field("plannedStartTime").asLong(); - var endTime = cmr.field("plannedEndTime").asLong(); - var now = clock().instant().getEpochSecond(); - return now > startTime && now < endTime; - }) - ).orElse(false); + .map(report -> + SlimeUtils.entriesStream(report.getInspector().field("upcoming")) + .anyMatch(cmr -> { + var startTime = cmr.field("plannedStartTime").asLong(); + var endTime = cmr.field("plannedEndTime").asLong(); + var now = clock().instant().getEpochSecond(); + return now > startTime && now < endTime; + }) + ).orElse(false); } /** Is the node and all active children suspended? */ diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ProvisionedExpirer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ProvisionedExpirer.java index 2d513390cf5..24901cb10a9 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ProvisionedExpirer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ProvisionedExpirer.java @@ -38,10 +38,9 @@ public class ProvisionedExpirer extends Expirer { for (Node expiredNode : expired) { if (expiredNode.type() != NodeType.host) continue; - nodeRepository().nodes().parkRecursively(expiredNode.hostname(), Agent.ProvisionedExpirer, "Node is stuck in provisioned"); - if (MAXIMUM_ALLOWED_EXPIRED_HOSTS < ++previouslyExpired) { - nodeRepository.nodes().deprovision(expiredNode.hostname(), Agent.ProvisionedExpirer, nodeRepository.clock().instant()); - } + boolean forceDeprovision = MAXIMUM_ALLOWED_EXPIRED_HOSTS < ++previouslyExpired; + nodeRepository().nodes().parkRecursively(expiredNode.hostname(), Agent.ProvisionedExpirer, + forceDeprovision, "Node is stuck in provisioned"); } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/IP.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/IP.java index b4968c66a67..264e981558a 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/IP.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/IP.java @@ -222,6 +222,14 @@ public record IP() { this.hostnames = List.copyOf(Objects.requireNonNull(hostnames, "hostnames must be non-null")); } + /** The number of hosts in this pool: each host has a name and/or one or two IP addresses. */ + public long size() { + return hostnames().isEmpty() ? + Math.max(ipAddresses.addresses.stream().filter(IP::isV4).count(), + ipAddresses.addresses.stream().filter(IP::isV6).count()) : + hostnames().size(); + } + public List<String> ips() { return ipAddresses.addresses; } /** diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Nodes.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Nodes.java index d70490c8e9a..5c0614dd8d6 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Nodes.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Nodes.java @@ -6,8 +6,8 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ApplicationTransaction; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Flavor; -import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; +import com.yahoo.config.provision.ApplicationMutex; import com.yahoo.config.provision.Zone; import com.yahoo.time.TimeBudget; import com.yahoo.transaction.Mutex; @@ -106,24 +106,10 @@ public class Nodes { public NodeList list(Node.State... inState) { NodeList allNodes = NodeList.copyOf(db.readNodes()); NodeList nodes = inState.length == 0 ? allNodes : allNodes.state(Set.of(inState)); - nodes = NodeList.copyOf(nodes.stream().map(node -> specifyFully(node, allNodes)).toList()); + nodes = NodeList.copyOf(nodes.stream().toList()); return nodes; } - // Repair underspecified node resources. TODO: Remove this after June 2023 - private Node specifyFully(Node node, NodeList allNodes) { - if (node.resources().isUnspecified()) return node; - - if (node.resources().bandwidthGbpsIsUnspecified()) - node = node.with(new Flavor(node.resources().withBandwidthGbps(0.3)), Agent.system, clock.instant()); - if ( node.resources().architecture() == NodeResources.Architecture.any) { - Optional<Node> parent = allNodes.parentOf(node); - if (parent.isPresent()) - node = node.with(new Flavor(node.resources().with(parent.get().resources().architecture())), Agent.system, clock.instant()); - } - return node; - } - /** Returns a locked list of all nodes in this repository */ public LockedNodeList list(Mutex lock) { return new LockedNodeList(list().asList(), lock); @@ -238,6 +224,23 @@ public class Nodes { performOn(nodes, (node, mutex) -> write(node.with(node.allocation().get().removable(true, reusable)), mutex)); } + /** Sets the exclusiveToApplicationId field. The nodes must be tenant hosts without the field already. */ + public void setExclusiveToApplicationId(List<Node> hosts, ApplicationMutex lock) { + List<Node> hostsToWrite = hosts.stream() + .filter(host -> !host.exclusiveToApplicationId().equals(Optional.of(lock.application()))) + .peek(host -> { + if (host.type() != NodeType.host) + throw new IllegalArgumentException("Unable to set " + host + " exclusive to " + lock.application() + + ": the node is not a tenant host"); + if (host.exclusiveToApplicationId().isPresent()) + throw new IllegalArgumentException("Unable to set " + host + " exclusive to " + lock.application() + + ": it is already set exclusive to " + host.exclusiveToApplicationId().get()); + }) + .map(host -> host.withExclusiveToApplicationId(lock.application())) + .toList(); + write(hostsToWrite, lock); + } + /** * Deactivates these nodes in a transaction and returns the nodes in the new state which will hold if the * transaction commits. @@ -400,8 +403,8 @@ public class Nodes { * * @return List of all the parked nodes in their new state */ - public List<Node> parkRecursively(String hostname, Agent agent, String reason) { - return moveRecursively(hostname, Node.State.parked, agent, Optional.of(reason)); + public List<Node> parkRecursively(String hostname, Agent agent, boolean forceDeprovision, String reason) { + return moveRecursively(hostname, Node.State.parked, agent, forceDeprovision, Optional.of(reason)); } /** @@ -430,12 +433,12 @@ public class Nodes { } } - private List<Node> moveRecursively(String hostname, Node.State toState, Agent agent, Optional<String> reason) { + private List<Node> moveRecursively(String hostname, Node.State toState, Agent agent, boolean forceDeprovision, Optional<String> reason) { try (RecursiveNodeMutexes locked = lockAndGetRecursively(hostname, Optional.empty())) { List<Node> moved = new ArrayList<>(); NestedTransaction transaction = new NestedTransaction(); for (NodeMutex node : locked.nodes().nodes()) - moved.add(move(node.node().hostname(), toState, agent, false, reason, transaction)); + moved.add(move(node.node().hostname(), toState, agent, forceDeprovision, reason, transaction)); transaction.commit(); return moved; } @@ -481,11 +484,12 @@ public class Nodes { } } - /* - * This method is used by the REST API to handle readying nodes for new allocations. For Linux - * containers this will remove the node from node repository, otherwise the node will be moved to state ready. + /** + * This method is used by the REST API to handle readying nodes for new allocations. + * Tenant containers will be removed, while other nodes will be moved to the ready state. + * Returns true if a node was updated, or false if the node was removed, or already was ready. */ - public Node markNodeAvailableForNewAllocation(String hostname, Agent agent, String reason) { + public boolean markNodeAvailableForNewAllocation(String hostname, Agent agent, String reason) { try (NodeMutex nodeMutex = lockAndGetRequired(hostname)) { Node node = nodeMutex.node(); if (node.type() == NodeType.tenant) { @@ -495,17 +499,18 @@ public class Nodes { NestedTransaction transaction = new NestedTransaction(); db.removeNodes(List.of(node), transaction); transaction.commit(); - return node; + return false; } - if (node.state() == Node.State.ready) return node; + if (node.state() == Node.State.ready) return false; Node parentHost = node.parentHostname().flatMap(this::node).orElse(node); List<String> failureReasons = NodeFailer.reasonsToFailHost(parentHost); if (!failureReasons.isEmpty()) illegal(node + " cannot be readied because it has hard failures: " + failureReasons); - return setReady(nodeMutex, agent, reason); + setReady(nodeMutex, agent, reason); + return true; } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java index 40d8394142b..5efe5d8b2a8 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java @@ -283,7 +283,7 @@ public class NodeSerializer { SlimeUtils.optionalString(object.field(modelNameKey)), SlimeUtils.optionalString(object.field(reservedToKey)).map(TenantName::from), SlimeUtils.optionalString(object.field(exclusiveToApplicationIdKey)).map(ApplicationId::fromSerializedForm), - SlimeUtils.optionalString(object.field(exclusiveToApplicationIdKey)).map(ApplicationId::fromSerializedForm), // TODO: change to provisionedForApplicationIdKey + SlimeUtils.optionalString(object.field(provisionedForApplicationIdKey)).map(ApplicationId::fromSerializedForm), SlimeUtils.optionalDuration(object.field(hostTTLKey)), SlimeUtils.optionalInstant(object.field(hostEmptyAtKey)), SlimeUtils.optionalString(object.field(exclusiveToClusterTypeKey)).map(ClusterSpec.Type::from), diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/CapacityPolicies.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/CapacityPolicies.java index f9ac7367778..1e9adea4e95 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/CapacityPolicies.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/CapacityPolicies.java @@ -9,6 +9,7 @@ import com.yahoo.config.provision.ClusterResources; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.NodeResources; +import com.yahoo.config.provision.NodeResources.DiskSpeed; import com.yahoo.config.provision.Zone; import com.yahoo.vespa.flags.PermanentFlags; import com.yahoo.vespa.flags.StringFlag; @@ -90,7 +91,10 @@ public class CapacityPolicies { } public NodeResources specifyFully(NodeResources resources, ClusterSpec clusterSpec, ApplicationId applicationId) { - return resources.withUnspecifiedNumbersFrom(defaultResources(clusterSpec, applicationId)); + NodeResources amended = resources.withUnspecifiedFieldsFrom(defaultResources(clusterSpec, applicationId).with(DiskSpeed.any)); + // TODO jonmv: remove this after all apps are 8.248.8 or above; architecture for admin nodes was not picked up before this. + if (clusterSpec.vespaVersion().isBefore(Version.fromString("8.248.8"))) amended = amended.with(resources.architecture()); + return amended; } private NodeResources defaultResources(ClusterSpec clusterSpec, ApplicationId applicationId) { @@ -149,7 +153,9 @@ public class CapacityPolicies { return Architecture.valueOf(adminClusterNodeArchitecture.with(INSTANCE_ID, instance.serializedForm()).value()); } - /** Returns the resources for the newest version not newer than that requested in the cluster spec. */ + /** + * Returns the resources for the newest version not newer than that requested in the cluster spec. + */ static NodeResources versioned(ClusterSpec spec, Map<Version, NodeResources> resources) { return requireNonNull(new TreeMap<>(resources).floorEntry(spec.vespaVersion()), "no default resources applicable for " + spec + " among: " + resources) diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostProvisioner.java index 09d6f96d88e..38cbfa7fe5f 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostProvisioner.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostProvisioner.java @@ -22,14 +22,21 @@ public interface HostProvisioner { enum HostSharing { - /** The host must be provisioned exclusively for the applicationId */ + /** The host must be provisioned exclusively for the application ID. */ + provision, + + /** The host must be exclusive to a single application ID */ exclusive, /** The host must be provisioned to be shared with other applications. */ shared, /** The client has no requirements on whether the host must be provisioned exclusively or shared. */ - any + any; + + public boolean isExclusiveAllocation() { + return this == provision || this == exclusive; + } } @@ -43,8 +50,10 @@ public interface HostProvisioner { * written to ZK immediately in case the config server goes down while waiting * for the provisioning to finish. * @throws NodeAllocationException if the cloud provider cannot satisfy the request + * @return a runnable that waits for the provisioning request to finish. It can be run without holding any locks, + * but may fail with an exception that should be propagated to the user initiating prepare() */ - void provisionHosts(HostProvisionRequest request, Predicate<NodeResources> realHostResourcesWithinLimits, Consumer<List<ProvisionedHost>> whenProvisioned) throws NodeAllocationException; + Runnable provisionHosts(HostProvisionRequest request, Predicate<NodeResources> realHostResourcesWithinLimits, Consumer<List<ProvisionedHost>> whenProvisioned) throws NodeAllocationException; /** * Continue provisioning of given list of Nodes. diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/InfraDeployerImpl.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/InfraDeployerImpl.java index 9cc1f2e05ef..39c14be4d2b 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/InfraDeployerImpl.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/InfraDeployerImpl.java @@ -13,7 +13,6 @@ import com.yahoo.config.provision.HostSpec; import com.yahoo.config.provision.InfraDeployer; import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.Provisioner; -import com.yahoo.transaction.Mutex; import com.yahoo.transaction.NestedTransaction; import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.maintenance.InfrastructureVersions; @@ -60,7 +59,7 @@ public class InfraDeployerImpl implements InfraDeployer { .forEach(api -> { var application = api.getApplicationId(); var deployment = new InfraDeployment(api); - try { + try (var lock = nodeRepository.applications().lockMaintenance(application)) { deployment.activate(); } catch (RuntimeException e) { logger.log(Level.INFO, "Failed to activate " + application, e); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java index 87e3bea1af9..e1be5b48e2d 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java @@ -84,9 +84,10 @@ class NodeAllocation { private final NodeRepository nodeRepository; private final Optional<String> requiredHostFlavor; + private final boolean makeExclusive; NodeAllocation(NodeList allNodes, ApplicationId application, ClusterSpec cluster, NodeSpec requested, - Supplier<Integer> nextIndex, NodeRepository nodeRepository) { + Supplier<Integer> nextIndex, NodeRepository nodeRepository, boolean makeExclusive) { this.allNodes = allNodes; this.application = application; this.cluster = cluster; @@ -99,6 +100,7 @@ class NodeAllocation { .with(FetchVector.Dimension.CLUSTER_ID, cluster.id().value()) .value()) .filter(s -> !s.isBlank()); + this.makeExclusive = makeExclusive; } /** @@ -117,21 +119,21 @@ class NodeAllocation { ClusterMembership membership = allocation.membership(); if ( ! allocation.owner().equals(application)) continue; // wrong application if ( ! membership.cluster().satisfies(cluster)) continue; // wrong cluster id/type - if ( candidate.state() == Node.State.active && allocation.removable()) continue; // don't accept; causes removal - if ( candidate.state() == Node.State.active && candidate.wantToFail()) continue; // don't accept; causes failing - if ( indexes.contains(membership.index())) continue; // duplicate index (just to be sure) + if (candidate.state() == Node.State.active && allocation.removable()) continue; // don't accept; causes removal + if (candidate.state() == Node.State.active && candidate.wantToFail()) continue; // don't accept; causes failing + if (indexes.contains(membership.index())) continue; // duplicate index (just to be sure) if (nodeRepository.zone().cloud().allowEnclave() && candidate.parent.isPresent() && ! candidate.parent.get().cloudAccount().equals(requested.cloudAccount())) continue; // wrong account boolean resizeable = requested.considerRetiring() && candidate.isResizable; - if ((! saturated() && hasCompatibleResources(candidate) && requested.acceptable(candidate)) || acceptIncompatible(candidate)) { + if (( ! saturated() && hasCompatibleResources(candidate) && requested.acceptable(candidate)) || acceptIncompatible(candidate)) { candidate = candidate.withNode(); if (candidate.isValid()) acceptNode(candidate, shouldRetire(candidate, candidates), resizeable); } } - else if (! saturated() && hasCompatibleResources(candidate)) { - if (! nodeRepository.nodeResourceLimits().isWithinRealLimits(candidate, application, cluster)) { + else if ( ! saturated() && hasCompatibleResources(candidate)) { + if ( ! nodeRepository.nodeResourceLimits().isWithinRealLimits(candidate, application, cluster)) { ++rejectedDueToInsufficientRealResources; continue; } @@ -139,9 +141,13 @@ class NodeAllocation { ++rejectedDueToClashingParentHost; continue; } - if ( violatesExclusivity(candidate)) { - ++rejectedDueToExclusivity; - continue; + switch (violatesExclusivity(candidate)) { + case PARENT_HOST_NOT_EXCLUSIVE -> candidate = candidate.withExclusiveParent(true); + case NONE -> {} + case YES -> { + ++rejectedDueToExclusivity; + continue; + } } if (candidate.wantToRetire()) { continue; @@ -169,7 +175,7 @@ class NodeAllocation { if (candidate.parent.map(node -> node.status().wantToUpgradeFlavor()).orElse(false)) return Retirement.violatesHostFlavorGeneration; if (candidate.wantToRetire()) return Retirement.hardRequest; if (candidate.preferToRetire() && candidate.replaceableBy(candidates)) return Retirement.softRequest; - if (violatesExclusivity(candidate)) return Retirement.violatesExclusivity; + if (violatesExclusivity(candidate) != NodeCandidate.ExclusivityViolation.NONE) return Retirement.violatesExclusivity; if (requiredHostFlavor.isPresent() && ! candidate.parent.map(node -> node.flavor().name()).equals(requiredHostFlavor)) return Retirement.violatesHostFlavor; if (candidate.violatesSpares) return Retirement.violatesSpares; return Retirement.none; @@ -186,39 +192,15 @@ class NodeAllocation { } private boolean offeredNodeHasParentHostnameAlreadyAccepted(NodeCandidate candidate) { - for (NodeCandidate acceptedNode : nodes.values()) { - if (acceptedNode.parentHostname().isPresent() && candidate.parentHostname().isPresent() && - acceptedNode.parentHostname().get().equals(candidate.parentHostname().get())) { - return true; - } - } - return false; - } - - private boolean violatesExclusivity(NodeCandidate candidate) { if (candidate.parentHostname().isEmpty()) return false; - if (requested.type() != NodeType.tenant) return false; - - // In zones which does not allow host sharing, exclusivity is violated if... - if ( ! nodeRepository.zone().cloud().allowHostSharing()) { - // TODO: Write this in a way that is simple to read - // If either the parent is dedicated to a cluster type different from this cluster - return ! candidate.parent.flatMap(Node::exclusiveToClusterType).map(cluster.type()::equals).orElse(true) || - // or the parent is dedicated to a different application - ! candidate.parent.flatMap(Node::exclusiveToApplicationId).map(application::equals).orElse(true) || - // or this cluster requires exclusivity, but the host is not exclusive - (nodeRepository.exclusiveAllocation(cluster) && candidate.parent.flatMap(Node::exclusiveToApplicationId).isEmpty()); - } + return nodes.values().stream().anyMatch(acceptedNode -> acceptedNode.parentHostname().equals(candidate.parentHostname())); + } - // In zones with shared hosts we require that if either of the nodes on the host requires exclusivity, - // then all the nodes on the host must have the same owner - for (Node nodeOnHost : allNodes.childrenOf(candidate.parentHostname().get())) { - if (nodeOnHost.allocation().isEmpty()) continue; - if (nodeRepository.exclusiveAllocation(cluster) || nodeOnHost.allocation().get().membership().cluster().isExclusive()) { - if ( ! nodeOnHost.allocation().get().owner().equals(application)) return true; - } - } - return false; + private NodeCandidate.ExclusivityViolation violatesExclusivity(NodeCandidate candidate) { + return candidate.violatesExclusivity(cluster, application, + nodeRepository.exclusiveAllocation(cluster), + nodeRepository.exclusiveProvisioning(cluster), + nodeRepository.zone().cloud().allowHostSharing(), allNodes, makeExclusive); } /** @@ -399,6 +381,14 @@ class NodeAllocation { return requested.type(); } + List<Node> parentsRequiredToBeExclusive() { + return nodes.values() + .stream() + .filter(candidate -> candidate.exclusiveParent) + .map(candidate -> candidate.parent.orElseThrow()) + .toList(); + } + List<Node> finalNodes() { GroupAssigner groupAssigner = new GroupAssigner(requested, allNodes, nodeRepository.clock()); Collection<NodeCandidate> finalNodes = groupAssigner.assignTo(nodes.values()); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeCandidate.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeCandidate.java index 1efc35b416d..1547a266e15 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeCandidate.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeCandidate.java @@ -4,11 +4,13 @@ package com.yahoo.vespa.hosted.provision.provisioning; import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ClusterMembership; +import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Flavor; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; import com.yahoo.vespa.hosted.provision.LockedNodeList; import com.yahoo.vespa.hosted.provision.Node; +import com.yahoo.vespa.hosted.provision.NodeList; import com.yahoo.vespa.hosted.provision.Nodelike; import com.yahoo.vespa.hosted.provision.node.Allocation; import com.yahoo.vespa.hosted.provision.node.IP; @@ -20,6 +22,8 @@ import java.util.Objects; import java.util.Optional; import java.util.logging.Logger; +import static com.yahoo.collections.Optionals.emptyOrEqual; + /** * A node candidate containing the details required to prioritize it for allocation. This is immutable. * @@ -59,7 +63,11 @@ public abstract class NodeCandidate implements Nodelike, Comparable<NodeCandidat /** This node can be resized to the new NodeResources */ final boolean isResizable; - private NodeCandidate(NodeResources freeParentCapacity, Optional<Node> parent, boolean violatesSpares, boolean exclusiveSwitch, boolean isSurplus, boolean isNew, boolean isResizeable) { + /** The parent host must become exclusive to the implied application */ + final boolean exclusiveParent; + + private NodeCandidate(NodeResources freeParentCapacity, Optional<Node> parent, boolean violatesSpares, boolean exclusiveSwitch, + boolean exclusiveParent, boolean isSurplus, boolean isNew, boolean isResizeable) { if (isResizeable && isNew) throw new IllegalArgumentException("A new node cannot be resizable"); @@ -67,6 +75,7 @@ public abstract class NodeCandidate implements Nodelike, Comparable<NodeCandidat this.parent = parent; this.violatesSpares = violatesSpares; this.exclusiveSwitch = exclusiveSwitch; + this.exclusiveParent = exclusiveParent; this.isSurplus = isSurplus; this.isNew = isNew; this.isResizable = isResizeable; @@ -95,6 +104,8 @@ public abstract class NodeCandidate implements Nodelike, Comparable<NodeCandidat /** Returns a copy of this with exclusive switch set to given value */ public abstract NodeCandidate withExclusiveSwitch(boolean exclusiveSwitch); + public abstract NodeCandidate withExclusiveParent(boolean exclusiveParent); + /** * Returns the node instance of this candidate, allocating it if necessary. * @@ -224,7 +235,7 @@ public abstract class NodeCandidate implements Nodelike, Comparable<NodeCandidat /** Returns a copy of this with node set to given value */ NodeCandidate withNode(Node node, boolean retiredNow) { - return new ConcreteNodeCandidate(node, retiredNow, freeParentCapacity, parent, violatesSpares, exclusiveSwitch, isSurplus, isNew, isResizable); + return new ConcreteNodeCandidate(node, retiredNow, freeParentCapacity, parent, violatesSpares, exclusiveSwitch, exclusiveParent, isSurplus, isNew, isResizable); } /** Returns the switch priority, based on switch exclusivity, of this compared to other */ @@ -267,7 +278,7 @@ public abstract class NodeCandidate implements Nodelike, Comparable<NodeCandidat boolean isSurplus, boolean isNew, boolean isResizeable) { - return new ConcreteNodeCandidate(node, false, freeParentCapacity, Optional.of(parent), violatesSpares, true, isSurplus, isNew, isResizeable); + return new ConcreteNodeCandidate(node, false, freeParentCapacity, Optional.of(parent), violatesSpares, true, false, isSurplus, isNew, isResizeable); } public static NodeCandidate createNewChild(NodeResources resources, @@ -276,15 +287,15 @@ public abstract class NodeCandidate implements Nodelike, Comparable<NodeCandidat boolean violatesSpares, LockedNodeList allNodes, IP.Allocation.Context ipAllocationContext) { - return new VirtualNodeCandidate(resources, freeParentCapacity, parent, violatesSpares, true, allNodes, ipAllocationContext); + return new VirtualNodeCandidate(resources, freeParentCapacity, parent, violatesSpares, true, false, allNodes, ipAllocationContext); } public static NodeCandidate createNewExclusiveChild(Node node, Node parent) { - return new ConcreteNodeCandidate(node, false, node.resources(), Optional.of(parent), false, true, false, true, false); + return new ConcreteNodeCandidate(node, false, node.resources(), Optional.of(parent), false, true, false, false, true, false); } public static NodeCandidate createStandalone(Node node, boolean isSurplus, boolean isNew) { - return new ConcreteNodeCandidate(node, false, node.resources(), Optional.empty(), false, true, isSurplus, isNew, false); + return new ConcreteNodeCandidate(node, false, node.resources(), Optional.empty(), false, true, false, isSurplus, isNew, false); } /** A candidate backed by a node */ @@ -296,9 +307,9 @@ public abstract class NodeCandidate implements Nodelike, Comparable<NodeCandidat ConcreteNodeCandidate(Node node, boolean retiredNow, NodeResources freeParentCapacity, Optional<Node> parent, - boolean violatesSpares, boolean exclusiveSwitch, + boolean violatesSpares, boolean exclusiveSwitch, boolean exclusiveParent, boolean isSurplus, boolean isNew, boolean isResizeable) { - super(freeParentCapacity, parent, violatesSpares, exclusiveSwitch, isSurplus, isNew, isResizeable); + super(freeParentCapacity, parent, violatesSpares, exclusiveSwitch, exclusiveParent, isSurplus, isNew, isResizeable); this.retiredNow = retiredNow; this.node = Objects.requireNonNull(node, "Node cannot be null"); } @@ -336,7 +347,7 @@ public abstract class NodeCandidate implements Nodelike, Comparable<NodeCandidat @Override public NodeCandidate allocate(ApplicationId owner, ClusterMembership membership, NodeResources requestedResources, Instant at) { return new ConcreteNodeCandidate(node.allocate(owner, membership, requestedResources, at), retiredNow, - freeParentCapacity, parent, violatesSpares, exclusiveSwitch, isSurplus, isNew, isResizable); + freeParentCapacity, parent, violatesSpares, exclusiveSwitch, exclusiveParent, isSurplus, isNew, isResizable); } /** Called when the node described by this candidate must be created */ @@ -346,7 +357,13 @@ public abstract class NodeCandidate implements Nodelike, Comparable<NodeCandidat @Override public NodeCandidate withExclusiveSwitch(boolean exclusiveSwitch) { return new ConcreteNodeCandidate(node, retiredNow, freeParentCapacity, parent, violatesSpares, exclusiveSwitch, - isSurplus, isNew, isResizable); + exclusiveParent, isSurplus, isNew, isResizable); + } + + @Override + public NodeCandidate withExclusiveParent(boolean exclusiveParent) { + return new ConcreteNodeCandidate(node, retiredNow, freeParentCapacity, parent, violatesSpares, exclusiveSwitch, + exclusiveParent, isSurplus, isNew, isResizable); } @Override @@ -387,9 +404,10 @@ public abstract class NodeCandidate implements Nodelike, Comparable<NodeCandidat Node parent, boolean violatesSpares, boolean exclusiveSwitch, + boolean exclusiveParent, LockedNodeList allNodes, IP.Allocation.Context ipAllocationContext) { - super(freeParentCapacity, Optional.of(parent), violatesSpares, exclusiveSwitch, false, true, false); + super(freeParentCapacity, Optional.of(parent), violatesSpares, exclusiveSwitch, exclusiveParent, false, true, false); this.resources = resources; this.allNodes = allNodes; this.ipAllocationContext = ipAllocationContext; @@ -449,13 +467,18 @@ public abstract class NodeCandidate implements Nodelike, Comparable<NodeCandidat NodeType.tenant) .cloudAccount(parent.get().cloudAccount()) .build(); - return new ConcreteNodeCandidate(node, false, freeParentCapacity, parent, violatesSpares, exclusiveSwitch, isSurplus, isNew, isResizable); + return new ConcreteNodeCandidate(node, false, freeParentCapacity, parent, violatesSpares, exclusiveSwitch, exclusiveParent, isSurplus, isNew, isResizable); } @Override public NodeCandidate withExclusiveSwitch(boolean exclusiveSwitch) { - return new VirtualNodeCandidate(resources, freeParentCapacity, parent.get(), violatesSpares, exclusiveSwitch, allNodes, ipAllocationContext); + return new VirtualNodeCandidate(resources, freeParentCapacity, parent.get(), violatesSpares, exclusiveSwitch, exclusiveParent, allNodes, ipAllocationContext); + } + + @Override + public NodeCandidate withExclusiveParent(boolean exclusiveParent) { + return new VirtualNodeCandidate(resources, freeParentCapacity, parent.get(), violatesSpares, exclusiveSwitch, exclusiveParent, allNodes, ipAllocationContext); } @Override @@ -492,7 +515,7 @@ public abstract class NodeCandidate implements Nodelike, Comparable<NodeCandidat private InvalidNodeCandidate(NodeResources resources, NodeResources freeParentCapacity, Node parent, String invalidReason) { - super(freeParentCapacity, Optional.of(parent), false, false, false, true, false); + super(freeParentCapacity, Optional.of(parent), false, false, false, false, true, false); this.resources = resources; this.invalidReason = invalidReason; } @@ -540,6 +563,11 @@ public abstract class NodeCandidate implements Nodelike, Comparable<NodeCandidat } @Override + public NodeCandidate withExclusiveParent(boolean exclusiveParent) { + return this; + } + + @Override public Node toNode() { throw new IllegalStateException("Candidate node on " + parent.get() + " is invalid: " + invalidReason); } @@ -559,4 +587,61 @@ public abstract class NodeCandidate implements Nodelike, Comparable<NodeCandidat } + public enum ExclusivityViolation { + NONE, YES, + + /** No violation IF AND ONLY IF the parent host's exclusiveToApplicationId is set to this application. */ + PARENT_HOST_NOT_EXCLUSIVE + } + + public ExclusivityViolation violatesExclusivity(ClusterSpec cluster, ApplicationId application, + boolean exclusiveAllocation, boolean exclusiveProvisioning, + boolean hostSharing, NodeList allNodes, boolean makeExclusive) { + if (parentHostname().isEmpty()) return ExclusivityViolation.NONE; + if (type() != NodeType.tenant) return ExclusivityViolation.NONE; + + if (hostSharing) { + // In zones with shared hosts we require that if any node on the host requires exclusivity, + // then all the nodes on the host must have the same owner. + for (Node nodeOnHost : allNodes.childrenOf(parentHostname().get())) { + if (nodeOnHost.allocation().isEmpty()) continue; + if (exclusiveAllocation || nodeOnHost.allocation().get().membership().cluster().isExclusive()) { + if ( ! nodeOnHost.allocation().get().owner().equals(application)) return ExclusivityViolation.YES; + } + } + } else { + // the parent is exclusive to another cluster type + if ( ! emptyOrEqual(parent.flatMap(Node::exclusiveToClusterType), cluster.type())) + return ExclusivityViolation.YES; + + // the parent is provisioned for another application + if ( ! emptyOrEqual(parent.flatMap(Node::provisionedForApplicationId), application)) + return ExclusivityViolation.YES; + + // this cluster requires a parent that was provisioned for this application + if (exclusiveProvisioning && parent.flatMap(Node::provisionedForApplicationId).isEmpty()) + return ExclusivityViolation.YES; + + // the parent is exclusive to another application + if ( ! emptyOrEqual(parent.flatMap(Node::exclusiveToApplicationId), application)) + return ExclusivityViolation.YES; + + // this cluster requires exclusivity, but the parent is not exclusive + if (exclusiveAllocation && parent.flatMap(Node::exclusiveToApplicationId).isEmpty()) + return canMakeHostExclusive(makeExclusive, type(), hostSharing) ? + ExclusivityViolation.PARENT_HOST_NOT_EXCLUSIVE : + ExclusivityViolation.YES; + } + + return ExclusivityViolation.NONE; + } + + /** + * Whether it is allowed to take a host not exclusive to anyone, and make it exclusive to an application. + * Returns false if {@code makeExclusive} is false, which can be used to guard this feature. + */ + public static boolean canMakeHostExclusive(boolean makeExclusive, NodeType type, boolean allowHostSharing) { + return makeExclusive && type == NodeType.tenant && !allowHostSharing; + } + } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodePrioritizer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodePrioritizer.java index d0b13bb7f6c..b92d6fb6d18 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodePrioritizer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodePrioritizer.java @@ -9,7 +9,6 @@ import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeList; import com.yahoo.vespa.hosted.provision.node.IP; import com.yahoo.vespa.hosted.provision.node.Nodes; -import com.yahoo.vespa.hosted.provision.persistence.NameResolver; import java.util.ArrayList; import java.util.Collections; @@ -40,14 +39,17 @@ public class NodePrioritizer { private final IP.Allocation.Context ipAllocationContext; private final Nodes nodes; private final boolean dynamicProvisioning; + private final boolean allowHostSharing; + private final boolean exclusiveAllocation; + private final boolean makeExclusive; private final boolean canAllocateToSpareHosts; private final boolean topologyChange; private final int currentClusterSize; private final Set<Node> spareHosts; public NodePrioritizer(LockedNodeList allNodes, ApplicationId application, ClusterSpec clusterSpec, NodeSpec nodeSpec, - boolean dynamicProvisioning, IP.Allocation.Context ipAllocationContext, Nodes nodes, - HostResourcesCalculator hostResourcesCalculator, int spareCount) { + boolean dynamicProvisioning, boolean allowHostSharing, IP.Allocation.Context ipAllocationContext, Nodes nodes, + HostResourcesCalculator hostResourcesCalculator, int spareCount, boolean exclusiveAllocation, boolean makeExclusive) { this.allNodes = allNodes; this.calculator = hostResourcesCalculator; this.capacity = new HostCapacity(this.allNodes, hostResourcesCalculator); @@ -55,6 +57,9 @@ public class NodePrioritizer { this.clusterSpec = clusterSpec; this.application = application; this.dynamicProvisioning = dynamicProvisioning; + this.allowHostSharing = allowHostSharing; + this.exclusiveAllocation = exclusiveAllocation; + this.makeExclusive = makeExclusive; this.spareHosts = dynamicProvisioning ? capacity.findSpareHostsInDynamicallyProvisionedZones(this.allNodes.asList()) : capacity.findSpareHosts(this.allNodes.asList(), spareCount); @@ -122,7 +127,13 @@ public class NodePrioritizer { if (nodes.suspended(host)) continue; // Hosts that are suspended may be down for some time, e.g. for OS upgrade if (host.reservedTo().isPresent() && !host.reservedTo().get().equals(application.tenant())) continue; if (host.reservedTo().isPresent() && application.instance().isTester()) continue; - if (host.exclusiveToApplicationId().isPresent() && ! fitsPerfectly(host)) continue; + if (makeExclusive) { + if ( ! allowHostSharing && exclusiveAllocation && ! fitsPerfectly(host)) continue; + } else { + if (host.exclusiveToApplicationId().isPresent() && ! fitsPerfectly(host)) continue; + } + if ( ! host.provisionedForApplicationId().map(application::equals).orElse(true)) continue; + if ( ! host.exclusiveToApplicationId().map(application::equals).orElse(true)) continue; if ( ! host.exclusiveToClusterType().map(clusterSpec.type()::equals).orElse(true)) continue; if (spareHosts.contains(host) && !canAllocateToSpareHosts) continue; if ( ! capacity.hasCapacity(host, requested.resources().get())) continue; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java index 2eb21caa6a0..e2f1b7358cf 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java @@ -12,7 +12,7 @@ import com.yahoo.config.provision.HostFilter; import com.yahoo.config.provision.HostSpec; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; -import com.yahoo.config.provision.ProvisionLock; +import com.yahoo.config.provision.ApplicationMutex; import com.yahoo.config.provision.ProvisionLogger; import com.yahoo.config.provision.Provisioner; import com.yahoo.config.provision.Zone; @@ -157,8 +157,8 @@ public class NodeRepositoryProvisioner implements Provisioner { } @Override - public ProvisionLock lock(ApplicationId application) { - return new ProvisionLock(application, nodeRepository.applications().lock(application)); + public ApplicationMutex lock(ApplicationId application) { + return new ApplicationMutex(application, nodeRepository.applications().lock(application)); } /** diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Preparer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Preparer.java index 0ffd42aedba..83afe92d025 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Preparer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Preparer.java @@ -8,9 +8,14 @@ import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.NodeAllocationException; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; +import com.yahoo.config.provision.ApplicationMutex; import com.yahoo.jdisc.Metric; import com.yahoo.text.internal.SnippetGenerator; import com.yahoo.transaction.Mutex; +import com.yahoo.vespa.applicationmodel.InfrastructureApplication; +import com.yahoo.vespa.flags.BooleanFlag; +import com.yahoo.vespa.flags.FetchVector; +import com.yahoo.vespa.flags.Flags; import com.yahoo.vespa.hosted.provision.LockedNodeList; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeList; @@ -18,6 +23,7 @@ import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.node.Agent; import com.yahoo.vespa.hosted.provision.node.IP; import com.yahoo.vespa.hosted.provision.provisioning.HostProvisioner.HostSharing; +import com.yahoo.yolean.Exceptions; import java.util.LinkedHashSet; import java.util.List; @@ -44,12 +50,14 @@ public class Preparer { private final Optional<HostProvisioner> hostProvisioner; private final Optional<LoadBalancerProvisioner> loadBalancerProvisioner; private final ProvisioningThrottler throttler; + private final BooleanFlag makeExclusiveFlag; public Preparer(NodeRepository nodeRepository, Optional<HostProvisioner> hostProvisioner, Optional<LoadBalancerProvisioner> loadBalancerProvisioner, Metric metric) { this.nodeRepository = nodeRepository; this.hostProvisioner = hostProvisioner; this.loadBalancerProvisioner = loadBalancerProvisioner; this.throttler = new ProvisioningThrottler(nodeRepository, metric); + this.makeExclusiveFlag = Flags.MAKE_EXCLUSIVE.bindTo(nodeRepository.flagSource()); } /** @@ -69,11 +77,15 @@ public class Preparer { loadBalancerProvisioner.ifPresent(provisioner -> provisioner.prepare(application, cluster, requested)); + boolean makeExclusive = makeExclusiveFlag.with(FetchVector.Dimension.TENANT_ID, application.tenant().value()) + .with(FetchVector.Dimension.INSTANCE_ID, application.serializedForm()) + .with(FetchVector.Dimension.VESPA_VERSION, cluster.vespaVersion().toFullString()) + .value(); // Try preparing in memory without global unallocated lock. Most of the time there should be no changes, // and we can return nodes previously allocated. LockedNodeList allNodes = nodeRepository.nodes().list(PROBE_LOCK); NodeIndices indices = new NodeIndices(cluster.id(), allNodes); - NodeAllocation probeAllocation = prepareAllocation(application, cluster, requested, indices::probeNext, allNodes); + NodeAllocation probeAllocation = prepareAllocation(application, cluster, requested, indices::probeNext, allNodes, makeExclusive); if (probeAllocation.fulfilledAndNoChanges()) { List<Node> acceptedNodes = probeAllocation.finalNodes(); indices.commitProbe(); @@ -81,16 +93,25 @@ public class Preparer { } else { // There were some changes, so re-do the allocation with locks indices.resetProbe(); - return prepareWithLocks(application, cluster, requested, indices); + return prepareWithLocks(application, cluster, requested, indices, makeExclusive); } } + private ApplicationMutex parentLockOrNull(boolean makeExclusive, NodeType type) { + return NodeCandidate.canMakeHostExclusive(makeExclusive, type, nodeRepository.zone().cloud().allowHostSharing()) ? + nodeRepository.applications().lock(InfrastructureApplication.withNodeType(type.parentNodeType()).id()) : + null; + } + /// Note that this will write to the node repo. - private List<Node> prepareWithLocks(ApplicationId application, ClusterSpec cluster, NodeSpec requested, NodeIndices indices) { + private List<Node> prepareWithLocks(ApplicationId application, ClusterSpec cluster, NodeSpec requested, NodeIndices indices, boolean makeExclusive) { + Runnable waiter = null; + List<Node> acceptedNodes; try (Mutex lock = nodeRepository.applications().lock(application); + ApplicationMutex parentLockOrNull = parentLockOrNull(makeExclusive, requested.type()); Mutex allocationLock = nodeRepository.nodes().lockUnallocated()) { LockedNodeList allNodes = nodeRepository.nodes().list(allocationLock); - NodeAllocation allocation = prepareAllocation(application, cluster, requested, indices::next, allNodes); + NodeAllocation allocation = prepareAllocation(application, cluster, requested, indices::next, allNodes, makeExclusive); NodeType hostType = allocation.nodeType().hostType(); if (canProvisionDynamically(hostType) && allocation.hostDeficit().isPresent()) { HostSharing sharing = hostSharing(cluster, hostType); @@ -127,12 +148,13 @@ public class Preparer { requested.cloudAccount(), deficit.dueToFlavorUpgrade()); Predicate<NodeResources> realHostResourcesWithinLimits = resources -> nodeRepository.nodeResourceLimits().isWithinRealLimits(resources, application, cluster); - hostProvisioner.get().provisionHosts(request, realHostResourcesWithinLimits, whenProvisioned); + waiter = hostProvisioner.get().provisionHosts(request, realHostResourcesWithinLimits, whenProvisioned); } catch (NodeAllocationException e) { // Mark the nodes that were written to ZK in the consumer for deprovisioning. While these hosts do // not exist, we cannot remove them from ZK here because other nodes may already have been // allocated on them, so let HostDeprovisioner deal with it - hosts.forEach(host -> nodeRepository.nodes().deprovision(host.hostname(), Agent.system, nodeRepository.clock().instant())); + hosts.forEach(host -> nodeRepository.nodes().parkRecursively(host.hostname(), Agent.system, true, + "Failed to provision: " + Exceptions.toMessageString(e))); throw e; } } else if (allocation.hostDeficit().isPresent() && requested.canFail() && @@ -140,7 +162,7 @@ public class Preparer { // Non-dynamically provisioned zone with a deficit because we just now retired some nodes. // Try again, but without retiring indices.resetProbe(); - List<Node> accepted = prepareWithLocks(application, cluster, cns.withoutRetiring(), indices); + List<Node> accepted = prepareWithLocks(application, cluster, cns.withoutRetiring(), indices, makeExclusive); log.warning("Prepared " + application + " " + cluster.id() + " without retirement due to lack of capacity"); return accepted; } @@ -150,7 +172,12 @@ public class Preparer { allocation.allocationFailureDetails(), true); // Carry out and return allocation - List<Node> acceptedNodes = allocation.finalNodes(); + if (parentLockOrNull != null) { + List<Node> exclusiveParents = allocation.parentsRequiredToBeExclusive(); + nodeRepository.nodes().setExclusiveToApplicationId(exclusiveParents, parentLockOrNull); + // TODO: also update tags + } + acceptedNodes = allocation.finalNodes(); nodeRepository.nodes().reserve(allocation.reservableNodes()); nodeRepository.nodes().addReservedNodes(new LockedNodeList(allocation.newNodes(), allocationLock)); @@ -160,14 +187,16 @@ public class Preparer { .filter(node -> node.parentHostname().isEmpty() || activeHosts.parentOf(node).isPresent()) .toList(); } - return acceptedNodes; } + + if (waiter != null) waiter.run(); + return acceptedNodes; } private NodeAllocation prepareAllocation(ApplicationId application, ClusterSpec cluster, NodeSpec requested, - Supplier<Integer> nextIndex, LockedNodeList allNodes) { + Supplier<Integer> nextIndex, LockedNodeList allNodes, boolean makeExclusive) { validateAccount(requested.cloudAccount(), application, allNodes); - NodeAllocation allocation = new NodeAllocation(allNodes, application, cluster, requested, nextIndex, nodeRepository); + NodeAllocation allocation = new NodeAllocation(allNodes, application, cluster, requested, nextIndex, nodeRepository, makeExclusive); var allocationContext = IP.Allocation.Context.from(nodeRepository.zone().cloud().name(), requested.cloudAccount().isExclave(nodeRepository.zone()), nodeRepository.nameResolver()); @@ -176,10 +205,13 @@ public class Preparer { cluster, requested, nodeRepository.zone().cloud().dynamicProvisioning(), + nodeRepository.zone().cloud().allowHostSharing(), allocationContext, nodeRepository.nodes(), nodeRepository.resourcesCalculator(), - nodeRepository.spareCount()); + nodeRepository.spareCount(), + nodeRepository.exclusiveAllocation(cluster), + makeExclusive); allocation.offer(prioritizer.collect()); return allocation; } @@ -208,7 +240,9 @@ public class Preparer { private HostSharing hostSharing(ClusterSpec cluster, NodeType hostType) { if ( hostType.isSharable()) - return nodeRepository.exclusiveAllocation(cluster) ? HostSharing.exclusive : HostSharing.any; + return nodeRepository.exclusiveProvisioning(cluster) ? HostSharing.provision : + nodeRepository.exclusiveAllocation(cluster) ? HostSharing.exclusive : + HostSharing.any; else return HostSharing.any; } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisionedHost.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisionedHost.java index 7da80440667..8a84cfef09a 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisionedHost.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisionedHost.java @@ -31,6 +31,7 @@ public class ProvisionedHost { private final Flavor hostFlavor; private final NodeType hostType; private final Optional<ApplicationId> provisionedForApplicationId; + private final Optional<ApplicationId> exclusiveToApplicationId; private final Optional<ClusterSpec.Type> exclusiveToClusterType; private final List<HostName> nodeHostnames; private final NodeResources nodeResources; @@ -38,7 +39,9 @@ public class ProvisionedHost { private final CloudAccount cloudAccount; public ProvisionedHost(String id, String hostHostname, Flavor hostFlavor, NodeType hostType, - Optional<ApplicationId> provisionedForApplicationId, Optional<ClusterSpec.Type> exclusiveToClusterType, + Optional<ApplicationId> provisionedForApplicationId, + Optional<ApplicationId> exclusiveToApplicationId, + Optional<ClusterSpec.Type> exclusiveToClusterType, List<HostName> nodeHostnames, NodeResources nodeResources, Version osVersion, CloudAccount cloudAccount) { if (!hostType.isHost()) throw new IllegalArgumentException(hostType + " is not a host"); @@ -47,6 +50,7 @@ public class ProvisionedHost { this.hostFlavor = Objects.requireNonNull(hostFlavor, "Host flavor must be set"); this.hostType = Objects.requireNonNull(hostType, "Host type must be set"); this.provisionedForApplicationId = Objects.requireNonNull(provisionedForApplicationId, "provisionedForApplicationId must be set"); + this.exclusiveToApplicationId = Objects.requireNonNull(exclusiveToApplicationId, "exclusiveToApplicationId must be set"); this.exclusiveToClusterType = Objects.requireNonNull(exclusiveToClusterType, "exclusiveToClusterType must be set"); this.nodeHostnames = validateNodeAddresses(nodeHostnames); this.nodeResources = Objects.requireNonNull(nodeResources, "Node resources must be set"); @@ -68,6 +72,7 @@ public class ProvisionedHost { .status(Status.initial().withOsVersion(OsVersion.EMPTY.withCurrent(Optional.of(osVersion)))) .cloudAccount(cloudAccount); provisionedForApplicationId.ifPresent(builder::provisionedForApplicationId); + exclusiveToApplicationId.ifPresent(builder::exclusiveToApplicationId); exclusiveToClusterType.ifPresent(builder::exclusiveToClusterType); if ( ! hostTTL.isZero()) builder.hostTTL(hostTTL); return builder.build(); @@ -85,6 +90,7 @@ public class ProvisionedHost { public Flavor hostFlavor() { return hostFlavor; } public NodeType hostType() { return hostType; } public Optional<ApplicationId> provisionedForApplicationId() { return provisionedForApplicationId; } + public Optional<ApplicationId> exclusiveToApplicationId() { return exclusiveToApplicationId; } public Optional<ClusterSpec.Type> exclusiveToClusterType() { return exclusiveToClusterType; } public List<HostName> nodeHostnames() { return nodeHostnames; } public NodeResources nodeResources() { return nodeResources; } @@ -103,6 +109,7 @@ public class ProvisionedHost { hostFlavor.equals(that.hostFlavor) && hostType == that.hostType && provisionedForApplicationId.equals(that.provisionedForApplicationId) && + exclusiveToApplicationId.equals(that.exclusiveToApplicationId) && exclusiveToClusterType.equals(that.exclusiveToClusterType) && nodeHostnames.equals(that.nodeHostnames) && nodeResources.equals(that.nodeResources) && @@ -112,7 +119,7 @@ public class ProvisionedHost { @Override public int hashCode() { - return Objects.hash(id, hostHostname, hostFlavor, hostType, provisionedForApplicationId, exclusiveToClusterType, nodeHostnames, nodeResources, osVersion, cloudAccount); + return Objects.hash(id, hostHostname, hostFlavor, hostType, provisionedForApplicationId, exclusiveToApplicationId, exclusiveToClusterType, nodeHostnames, nodeResources, osVersion, cloudAccount); } @Override @@ -123,8 +130,9 @@ public class ProvisionedHost { ", hostFlavor=" + hostFlavor + ", hostType=" + hostType + ", provisionedForApplicationId=" + provisionedForApplicationId + + ", exclusiveToApplicationId=" + exclusiveToApplicationId + ", exclusiveToClusterType=" + exclusiveToClusterType + - ", nodeAddresses=" + nodeHostnames + + ", nodeHostnames=" + nodeHostnames + ", nodeResources=" + nodeResources + ", osVersion=" + osVersion + ", cloudAccount=" + cloudAccount + diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiHandler.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiHandler.java index 1ed138625ae..9080030f026 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiHandler.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiHandler.java @@ -9,6 +9,7 @@ import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Flavor; import com.yahoo.config.provision.HostFilter; import com.yahoo.config.provision.HostName; +import com.yahoo.config.provision.InfraDeployer; import com.yahoo.config.provision.NodeFlavors; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; @@ -44,6 +45,7 @@ import com.yahoo.vespa.hosted.provision.node.filter.NodeHostFilter; import com.yahoo.vespa.hosted.provision.node.filter.NodeOsVersionFilter; import com.yahoo.vespa.hosted.provision.node.filter.NodeTypeFilter; import com.yahoo.vespa.hosted.provision.node.filter.ParentHostFilter; +import com.yahoo.vespa.hosted.provision.maintenance.InfraApplicationRedeployer; import com.yahoo.vespa.hosted.provision.restapi.NodesResponse.ResponseType; import com.yahoo.vespa.orchestrator.Orchestrator; import com.yahoo.yolean.Exceptions; @@ -75,13 +77,16 @@ public class NodesV2ApiHandler extends ThreadedHttpRequestHandler { private final Orchestrator orchestrator; private final NodeRepository nodeRepository; private final NodeFlavors nodeFlavors; + private final InfraApplicationRedeployer infraApplicationRedeployer; @Inject - public NodesV2ApiHandler(Context parentCtx, Orchestrator orchestrator, NodeRepository nodeRepository, NodeFlavors flavors) { + public NodesV2ApiHandler(Context parentCtx, Orchestrator orchestrator, NodeRepository nodeRepository, + NodeFlavors flavors, InfraDeployer infraDeployer) { super(parentCtx); this.orchestrator = orchestrator; this.nodeRepository = nodeRepository; this.nodeFlavors = flavors; + this.infraApplicationRedeployer = new InfraApplicationRedeployer(infraDeployer, nodeRepository); } @Override @@ -138,7 +143,8 @@ public class NodesV2ApiHandler extends ThreadedHttpRequestHandler { Path path = new Path(request.getUri()); // Check paths to disallow illegal state changes if (path.matches("/nodes/v2/state/ready/{hostname}")) { - nodeRepository.nodes().markNodeAvailableForNewAllocation(path.get("hostname"), agent(request), "Readied through the nodes/v2 API"); + if (nodeRepository.nodes().markNodeAvailableForNewAllocation(path.get("hostname"), agent(request), "Readied through the nodes/v2 API")) + infraApplicationRedeployer.readied(nodeRepository.nodes().node(path.get("hostname")).get().type()); return new MessageResponse("Moved " + path.get("hostname") + " to " + Node.State.ready); } else if (path.matches("/nodes/v2/state/failed/{hostname}")) { @@ -147,7 +153,7 @@ public class NodesV2ApiHandler extends ThreadedHttpRequestHandler { " and marked " + hostnamesAsString(failedOrMarkedNodes.failing().asList()) + " as wantToFail"); } else if (path.matches("/nodes/v2/state/parked/{hostname}")) { - List<Node> parkedNodes = nodeRepository.nodes().parkRecursively(path.get("hostname"), agent(request), "Parked through the nodes/v2 API"); + List<Node> parkedNodes = nodeRepository.nodes().parkRecursively(path.get("hostname"), agent(request), false, "Parked through the nodes/v2 API"); return new MessageResponse("Moved " + hostnamesAsString(parkedNodes) + " to " + Node.State.parked); } else if (path.matches("/nodes/v2/state/dirty/{hostname}")) { @@ -508,4 +514,10 @@ public class NodesV2ApiHandler extends ThreadedHttpRequestHandler { } } + @Override + public void destroy() { + super.destroy(); + infraApplicationRedeployer.close(); + } + } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockHostProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockHostProvisioner.java index def3e003ab3..b5bb91af71a 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockHostProvisioner.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockHostProvisioner.java @@ -73,13 +73,13 @@ public class MockHostProvisioner implements HostProvisioner { } @Override - public void provisionHosts(HostProvisionRequest request, Predicate<NodeResources> realHostResourcesWithinLimits, Consumer<List<ProvisionedHost>> whenProvisioned) throws NodeAllocationException { + public Runnable provisionHosts(HostProvisionRequest request, Predicate<NodeResources> realHostResourcesWithinLimits, Consumer<List<ProvisionedHost>> whenProvisioned) throws NodeAllocationException { if (behaviour(Behaviour.failProvisionRequest)) throw new NodeAllocationException("No capacity for provision request", true); Flavor hostFlavor = hostFlavors.get(request.clusterType().orElse(ClusterSpec.Type.content)); if (hostFlavor == null) hostFlavor = flavors.stream() - .filter(f -> request.sharing() == HostSharing.exclusive ? compatible(f, request.resources()) - : satisfies(f, request.resources())) + .filter(f -> request.sharing().isExclusiveAllocation() ? compatible(f, request.resources()) + : satisfies(f, request.resources())) .filter(f -> realHostResourcesWithinLimits.test(f.resources())) .findFirst() .orElseThrow(() -> new NodeAllocationException("No host flavor matches " + request.resources(), true)); @@ -91,7 +91,8 @@ public class MockHostProvisioner implements HostProvisioner { hostHostname, hostFlavor, request.type(), - request.sharing() == HostSharing.exclusive ? Optional.of(request.owner()) : Optional.empty(), + request.sharing() == HostSharing.provision ? Optional.of(request.owner()) : Optional.empty(), + request.sharing().isExclusiveAllocation() ? Optional.of(request.owner()) : Optional.empty(), Optional.empty(), createHostnames(request.type(), hostFlavor, index), request.resources(), @@ -100,6 +101,7 @@ public class MockHostProvisioner implements HostProvisioner { } provisionedHosts.addAll(hosts); whenProvisioned.accept(hosts); + return () -> {}; } @Override diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockProvisioner.java index 377b1b3d4b4..e130f53fab5 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockProvisioner.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockProvisioner.java @@ -8,7 +8,7 @@ import com.yahoo.config.provision.Capacity; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.HostFilter; import com.yahoo.config.provision.HostSpec; -import com.yahoo.config.provision.ProvisionLock; +import com.yahoo.config.provision.ApplicationMutex; import com.yahoo.config.provision.ProvisionLogger; import com.yahoo.config.provision.Provisioner; @@ -36,7 +36,7 @@ public class MockProvisioner implements Provisioner { public void restart(ApplicationId application, HostFilter filter) { } @Override - public ProvisionLock lock(ApplicationId application) { + public ApplicationMutex lock(ApplicationId application) { return null; } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/RealDataScenarioTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/RealDataScenarioTest.java index f152cbb7a52..ad55b400735 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/RealDataScenarioTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/RealDataScenarioTest.java @@ -16,7 +16,7 @@ import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.Flavor; import com.yahoo.config.provision.HostSpec; import com.yahoo.config.provision.NodeResources; -import com.yahoo.config.provision.ProvisionLock; +import com.yahoo.config.provision.ApplicationMutex; import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.Zone; @@ -119,7 +119,7 @@ public class RealDataScenarioTest { .flatMap(Collection::stream) .toList(); NestedTransaction transaction = new NestedTransaction(); - tester.provisioner().activate(hostSpecs, new ActivationContext(0), new ApplicationTransaction(new ProvisionLock(app, () -> {}), transaction)); + tester.provisioner().activate(hostSpecs, new ActivationContext(0), new ApplicationTransaction(new ApplicationMutex(app, () -> {}), transaction)); transaction.commit(); } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/applications/ApplicationsTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/applications/ApplicationsTest.java index d378cb9a31b..1a5d5f0a37f 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/applications/ApplicationsTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/applications/ApplicationsTest.java @@ -3,7 +3,7 @@ package com.yahoo.vespa.hosted.provision.applications; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ApplicationTransaction; -import com.yahoo.config.provision.ProvisionLock; +import com.yahoo.config.provision.ApplicationMutex; import com.yahoo.transaction.NestedTransaction; import com.yahoo.vespa.hosted.provision.NodeRepositoryTester; import org.junit.Test; @@ -53,8 +53,8 @@ public class ApplicationsTest { assertEquals(List.of(), applications.ids()); } - private ProvisionLock provisionLock(ApplicationId application) { - return new ProvisionLock(application, () -> {}); + private ApplicationMutex provisionLock(ApplicationId application) { + return new ApplicationMutex(application, () -> {}); } } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityCheckerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityCheckerTest.java index 96338378892..df0f457b215 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityCheckerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityCheckerTest.java @@ -29,7 +29,7 @@ public class CapacityCheckerTest { var failurePath = tester.capacityChecker.worstCaseHostLossLeadingToFailure(); assertTrue(failurePath.isPresent()); assertTrue(tester.nodeRepository.nodes().list().nodeType(NodeType.host).asList().containsAll(failurePath.get().hostsCausingFailure)); - assertEquals(4, failurePath.get().hostsCausingFailure.size()); + assertEquals(5, failurePath.get().hostsCausingFailure.size()); } @Test diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/HostCapacityMaintainerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/HostCapacityMaintainerTest.java index bb30e0d985e..c804ade668c 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/HostCapacityMaintainerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/HostCapacityMaintainerTest.java @@ -24,9 +24,12 @@ import com.yahoo.config.provision.Zone; import com.yahoo.docproc.jdisc.metric.NullMetric; import com.yahoo.net.HostName; import com.yahoo.test.ManualClock; +import com.yahoo.vespa.flags.Flags; import com.yahoo.vespa.flags.InMemoryFlagSource; import com.yahoo.vespa.flags.PermanentFlags; import com.yahoo.vespa.flags.custom.ClusterCapacity; +import com.yahoo.vespa.flags.custom.HostResources; +import com.yahoo.vespa.flags.custom.SharedHost; import com.yahoo.vespa.hosted.provision.LockedNodeList; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.Node.State; @@ -278,6 +281,84 @@ public class HostCapacityMaintainerTest { } @Test + public void respects_exclusive_allocation() { + tester = new DynamicProvisioningTester(Cloud.builder().name(CloudName.AWS).dynamicProvisioning(true).allowHostSharing(false).build(), new MockNameResolver()); + NodeResources resources1 = new NodeResources(24, 64, 100, 10); + setPreprovisionCapacityFlag(tester, + new ClusterCapacity(1, resources1.vcpu(), resources1.memoryGb(), resources1.diskGb(), + resources1.bandwidthGbps(), resources1.diskSpeed().name(), + resources1.storageType().name(), resources1.architecture().name(), + "container"), + new ClusterCapacity(1, resources1.vcpu(), resources1.memoryGb(), resources1.diskGb(), + resources1.bandwidthGbps(), resources1.diskSpeed().name(), + resources1.storageType().name(), resources1.architecture().name(), + null)); + tester.flagSource.withBooleanFlag(Flags.MAKE_EXCLUSIVE.id(), true); + tester.maintain(); + + // Hosts are provisioned + assertEquals(2, tester.provisionedHostsMatching(resources1)); + assertEquals(0, tester.hostProvisioner.deprovisionedHosts()); + assertEquals(Optional.empty(), tester.nodeRepository.nodes().node("host100").flatMap(Node::exclusiveToApplicationId)); + assertEquals(Optional.empty(), tester.nodeRepository.nodes().node("host101").flatMap(Node::exclusiveToApplicationId)); + + // Next maintenance run does nothing + tester.assertNodesUnchanged(); + + // One host is allocated exclusively to some other application + tester.nodeRepository.nodes().write(tester.nodeRepository.nodes().node("host100").get() + .withExclusiveToApplicationId(ApplicationId.from("t", "a", "i")), + () -> { }); + + tester.maintain(); + + // New hosts are provisioned, and the empty exclusive host is deallocated + assertEquals(2, tester.provisionedHostsMatching(resources1)); + assertEquals(1, tester.hostProvisioner.deprovisionedHosts()); + + // Next maintenance run does nothing + tester.assertNodesUnchanged(); + } + + @Test + public void works_as_before_without_make_exclusive() { + // TODO(hakon): Remove test once make-exclusive has rolled out + tester = new DynamicProvisioningTester(Cloud.builder().name(CloudName.AWS).dynamicProvisioning(true).allowHostSharing(false).build(), new MockNameResolver()); + NodeResources resources1 = new NodeResources(24, 64, 100, 10); + setPreprovisionCapacityFlag(tester, + new ClusterCapacity(1, resources1.vcpu(), resources1.memoryGb(), resources1.diskGb(), + resources1.bandwidthGbps(), resources1.diskSpeed().name(), + resources1.storageType().name(), resources1.architecture().name(), + null)); + tester.flagSource.withJacksonFlag(PermanentFlags.SHARED_HOST.id(), + new SharedHost(List.of(new HostResources(48d, 128d, 200d, 20d, "fast", "remote", null, 4, "x86_64"))), + SharedHost.class); + tester.maintain(); + + // Hosts are provisioned + assertEquals(1, tester.provisionedHostsMatching(resources1)); + assertEquals(0, tester.hostProvisioner.deprovisionedHosts()); + assertEquals(Optional.empty(), tester.nodeRepository.nodes().node("host100").flatMap(Node::exclusiveToApplicationId)); + + // Next maintenance run does nothing + tester.assertNodesUnchanged(); + + // One host is allocated exclusively to some other application + tester.nodeRepository.nodes().write(tester.nodeRepository.nodes().node("host100").get() + .withExclusiveToApplicationId(ApplicationId.from("t", "a", "i")), + () -> { }); + + tester.maintain(); + + // New hosts are provisioned, and the empty exclusive host is deallocated + assertEquals(1, tester.provisionedHostsMatching(resources1)); + assertEquals(1, tester.hostProvisioner.deprovisionedHosts()); + + // Next maintenance run does nothing + tester.assertNodesUnchanged(); + } + + @Test public void test_minimum_capacity() { tester = new DynamicProvisioningTester(); NodeResources resources1 = new NodeResources(24, 64, 100, 10); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/HostResumeProvisionerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/HostResumeProvisionerTest.java index 77986d03da2..74db071b060 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/HostResumeProvisionerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/HostResumeProvisionerTest.java @@ -98,7 +98,11 @@ public class HostResumeProvisionerTest { Stream.of(host, node).map(n -> n.ipConfig().primary()).allMatch(List::isEmpty)); hostResumeProvisioner.maintain(); - assertEquals(Set.of("host100", "host100-1"), tester.nodeRepository().nodes().list(Node.State.failed).hostnames()); + assertEquals(Set.of(), tester.nodeRepository().nodes().list(Node.State.parked).deprovisioning().hostnames()); + tester.clock().advance(Duration.ofSeconds(60)); + + hostResumeProvisioner.maintain(); + assertEquals(Set.of("host100", "host100-1"), tester.nodeRepository().nodes().list(Node.State.parked).deprovisioning().hostnames()); } private void deployApplication() { diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/InfraApplicationRedeployerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/InfraApplicationRedeployerTest.java new file mode 100644 index 00000000000..7a8129ad275 --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/InfraApplicationRedeployerTest.java @@ -0,0 +1,172 @@ +package com.yahoo.vespa.hosted.provision.maintenance; + +import com.yahoo.concurrent.UncheckedTimeoutException; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.Deployment; +import com.yahoo.config.provision.Flavor; +import com.yahoo.config.provision.InfraDeployer; +import com.yahoo.config.provision.NodeResources; +import com.yahoo.config.provision.NodeType; +import com.yahoo.transaction.Mutex; +import com.yahoo.vespa.applicationmodel.InfrastructureApplication; +import com.yahoo.vespa.hosted.provision.Node; +import com.yahoo.vespa.hosted.provision.Node.State; +import com.yahoo.vespa.hosted.provision.NodeList; +import com.yahoo.vespa.hosted.provision.node.IP; +import org.junit.jupiter.api.Test; + +import java.util.List; +import java.util.Optional; +import java.util.Queue; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.Phaser; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Supplier; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.fail; + +/** + * @author jonmv + */ +class InfraApplicationRedeployerTest { + + private static final ApplicationId cfghost = InfrastructureApplication.CONFIG_SERVER_HOST.id(); + private static final ApplicationId cfg = InfrastructureApplication.CONFIG_SERVER.id(); + private static final ApplicationId tenanthost = InfrastructureApplication.TENANT_HOST.id(); + + @Test + void testMultiTriggering() throws InterruptedException { + TestLocks locks = new TestLocks(); + List<Node> nodes = new CopyOnWriteArrayList<>(); + TestInfraDeployer deployer = new TestInfraDeployer(); + InfraApplicationRedeployer redeployer = new InfraApplicationRedeployer(deployer, locks::get, () -> NodeList.copyOf(nodes)); + Phaser intro = new Phaser(2); + CountDownLatch intermezzo = new CountDownLatch(1), outro = new CountDownLatch(1); + + // First run does nothing, as no nodes are ready after all, but several new runs are triggered as this ends. + locks.expect(tenanthost, () -> () -> { intro.arriveAndAwaitAdvance(); intro.arriveAndAwaitAdvance(); }); + redeployer.readied(NodeType.host); + intro.arriveAndAwaitAdvance(); // Wait for redeployer to start, before setting up more state. + // Before re-triggered events from first tenanthost run, we also trigger for confighost, which should then run before those. + locks.expect(cfghost, () -> () -> { }); + redeployer.readied(NodeType.confighost); + for (int i = 0; i < 10000; i++) redeployer.readied(NodeType.host); + nodes.add(node("host", NodeType.host, State.ready)); + // Re-run for tenanthost clears host from ready, and next run does nothing. + deployer.expect(tenanthost, () -> { + nodes.clear(); + return Optional.empty(); + }); + locks.expect(tenanthost, () -> intermezzo::countDown); + intro.arriveAndAwaitAdvance(); // Let redeployer continue. + intermezzo.await(10, TimeUnit.SECONDS); // Rendezvous with last, no-op tenanthost redeployment. + locks.verify(); + deployer.verify(); + + // Confighost is triggered again with one ready host. Both applications deploy, and a new trigger redeploys neither. + locks.expect(cfghost, () -> () -> { }); + locks.expect(cfg, () -> () -> { }); + nodes.add(node("cfghost", NodeType.confighost, State.ready)); + deployer.expect(cfghost, () -> { + nodes.clear(); + return Optional.empty(); + }); + deployer.expect(cfg, () -> { + redeployer.readied(NodeType.confighost); + return Optional.empty(); + }); + locks.expect(cfghost, () -> outro::countDown); + redeployer.readied(NodeType.confighost); + + outro.await(10, TimeUnit.SECONDS); + redeployer.close(); + locks.verify(); + deployer.verify(); + } + + @Test + void testRetries() throws InterruptedException { + TestLocks locks = new TestLocks(); + List<Node> nodes = new CopyOnWriteArrayList<>(); + TestInfraDeployer deployer = new TestInfraDeployer(); + InfraApplicationRedeployer redeployer = new InfraApplicationRedeployer(deployer, locks::get, () -> NodeList.copyOf(nodes)); + + // Does nothing. + redeployer.readied(NodeType.tenant); + + // Getting lock fails with runtime exception; no deployments, no retries. + locks.expect(tenanthost, () -> { throw new RuntimeException("Failed"); }); + redeployer.readied(NodeType.host); + + // Getting lock times out for configserver application; deployment of configserverapp is retried, but host is done. + CountDownLatch latch = new CountDownLatch(1); + locks.expect(cfghost, () -> () -> { }); + locks.expect(cfg, () -> { throw new UncheckedTimeoutException("Timeout"); }); + locks.expect(cfg, () -> latch::countDown); + nodes.add(node("cfghost", NodeType.confighost, State.ready)); + deployer.expect(cfghost, () -> { + nodes.set(0, node("cfghost", NodeType.confighost, State.active)); + return Optional.empty(); + }); + deployer.expect(cfg, Optional::empty); + redeployer.readied(NodeType.confighost); + latch.await(10, TimeUnit.SECONDS); + redeployer.close(); + locks.verify(); + deployer.verify(); + } + + private static Node node(String name, NodeType type, State state) { + return Node.create(name, name, new Flavor(NodeResources.unspecified()), state, type) + .ipConfig(IP.Config.of(List.of("1.2.3.4"), List.of("1.2.3.4"))) + .build(); + } + + private static class Expectations<T, R> { + + final Queue<T> expected = new ConcurrentLinkedQueue<>(); + final Queue<Throwable> stacks = new ConcurrentLinkedQueue<>(); + final Queue<Supplier<R>> reactions = new ConcurrentLinkedQueue<>(); + final AtomicReference<Throwable> failure = new AtomicReference<>(); + + void expect(T id, Supplier<R> reaction) { + expected.add(id); + stacks.add(new AssertionError("Failed expectation of " + id)); + reactions.add(reaction); + } + + R get(T id) { + Throwable s = stacks.poll(); + if (s == null) s = new AssertionError("Unexpected invocation with " + id); + try { assertEquals(expected.poll(), id); } + catch (Throwable t) { + StackTraceElement[] trace = t.getStackTrace(); + t.setStackTrace(s.getStackTrace()); + s.setStackTrace(trace); + t.addSuppressed(s); + if ( ! failure.compareAndSet(null, t)) failure.get().addSuppressed(t); + throw t; + } + return reactions.poll().get(); + } + + @SuppressWarnings("unchecked") + <E extends Throwable> void verify() throws E { + if (failure.get() != null) throw (E) failure.get(); + assertEquals(List.of(), List.copyOf(expected)); + } + + } + + private static class TestLocks extends Expectations<ApplicationId, Mutex> { } + + private static class TestInfraDeployer extends Expectations<ApplicationId, Optional<Deployment>> implements InfraDeployer { + @Override public Optional<Deployment> getDeployment(ApplicationId application) { return get(application); } + @Override public void activateAllSupportedInfraApplications(boolean propagateException) { fail(); } + } + +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporterTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporterTest.java index cc414cc50c2..691e67d945c 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporterTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporterTest.java @@ -11,7 +11,7 @@ import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.NodeFlavors; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; -import com.yahoo.config.provision.ProvisionLock; +import com.yahoo.config.provision.ApplicationMutex; import com.yahoo.jdisc.Metric; import com.yahoo.transaction.Mutex; import com.yahoo.transaction.NestedTransaction; @@ -216,7 +216,7 @@ public class MetricsReporterTest { NestedTransaction transaction = new NestedTransaction(); nodeRepository.nodes().activate(nodeRepository.nodes().list().nodeType(NodeType.host).asList(), - new ApplicationTransaction(new ProvisionLock(InfrastructureApplication.TENANT_HOST.id(), () -> { }), transaction)); + new ApplicationTransaction(new ApplicationMutex(InfrastructureApplication.TENANT_HOST.id(), () -> { }), transaction)); transaction.commit(); Orchestrator orchestrator = mock(Orchestrator.class); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RebalancerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RebalancerTest.java index 05d5afa2c8a..7e3a9d1ea88 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RebalancerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RebalancerTest.java @@ -10,7 +10,7 @@ import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.Flavor; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; -import com.yahoo.config.provision.ProvisionLock; +import com.yahoo.config.provision.ApplicationMutex; import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.Zone; import com.yahoo.config.provisioning.FlavorsConfig; @@ -85,7 +85,7 @@ public class RebalancerTest { // --- Making the system stable enables rebalancing NestedTransaction tx = new NestedTransaction(); tester.nodeRepository().nodes().deactivate(List.of(cpuSkewedNode), - new ApplicationTransaction(new ProvisionLock(cpuApp, () -> {}), tx)); + new ApplicationTransaction(new ApplicationMutex(cpuApp, () -> {}), tx)); tx.commit(); assertEquals(1, tester.getNodes(Node.State.dirty).size()); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/SpareCapacityMaintainerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/SpareCapacityMaintainerTest.java index 023047f17e6..9fefc9d34e1 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/SpareCapacityMaintainerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/SpareCapacityMaintainerTest.java @@ -11,7 +11,7 @@ import com.yahoo.config.provision.Flavor; import com.yahoo.config.provision.NodeFlavors; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; -import com.yahoo.config.provision.ProvisionLock; +import com.yahoo.config.provision.ApplicationMutex; import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.Zone; import com.yahoo.test.ManualClock; @@ -321,7 +321,7 @@ public class SpareCapacityMaintainerTest { } nodes = nodeRepository.nodes().reserve(nodes); var transaction = new NestedTransaction(); - nodes = nodeRepository.nodes().activate(nodes, new ApplicationTransaction(new ProvisionLock(application, () -> { }), transaction)); + nodes = nodeRepository.nodes().activate(nodes, new ApplicationTransaction(new ApplicationMutex(application, () -> { }), transaction)); transaction.commit(); } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializerTest.java index 4aab8b683b0..0fbbefa39bb 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializerTest.java @@ -483,7 +483,7 @@ public class NodeSerializerTest { ApplicationId provisionedForApp = ApplicationId.from("tenant1", "app1", "instance1"); node = nodeSerializer.fromJson(nodeSerializer.toJson(builder.exclusiveToApplicationId(provisionedForApp).build())); assertEquals(Optional.of(provisionedForApp), node.exclusiveToApplicationId()); - // assertEquals(Optional.empty(), node.provisionedForApplicationId()); TODO: enable once serialisation phase 1 is done + assertEquals(Optional.empty(), node.provisionedForApplicationId()); ClusterSpec.Type exclusiveToCluster = ClusterSpec.Type.admin; node = builder.provisionedForApplicationId(provisionedForApp) @@ -513,7 +513,7 @@ public class NodeSerializerTest { CloudAccount account = CloudAccount.from("012345678912"); Node node = Node.create("id", "host1.example.com", nodeFlavors.getFlavorOrThrow("default"), State.provisioned, NodeType.host) .cloudAccount(account) - .exclusiveToApplicationId(ApplicationId.defaultId()) + .provisionedForApplicationId(ApplicationId.defaultId()) .build(); node = nodeSerializer.fromJson(nodeSerializer.toJson(node)); assertEquals(account, node.cloudAccount()); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicAllocationTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicAllocationTest.java index 72582e47621..ff5ffd82bf1 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicAllocationTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicAllocationTest.java @@ -13,7 +13,7 @@ import com.yahoo.config.provision.HostSpec; import com.yahoo.config.provision.NodeAllocationException; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; -import com.yahoo.config.provision.ProvisionLock; +import com.yahoo.config.provision.ApplicationMutex; import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.Zone; @@ -542,7 +542,7 @@ public class DynamicAllocationTest { tester.nodeRepository().nodes().addNodes(List.of(node1aAllocation), Agent.system); NestedTransaction transaction = new NestedTransaction().add(new CuratorTransaction(tester.getCurator())); - tester.nodeRepository().nodes().activate(List.of(node1aAllocation), new ApplicationTransaction(new ProvisionLock(id, () -> { }), transaction)); + tester.nodeRepository().nodes().activate(List.of(node1aAllocation), new ApplicationTransaction(new ApplicationMutex(id, () -> { }), transaction)); transaction.commit(); } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicProvisioningTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicProvisioningTester.java index 9204e3ddb0b..be2b2ca896a 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicProvisioningTester.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicProvisioningTester.java @@ -200,7 +200,7 @@ public class DynamicProvisioningTester { int nodeCount, int groupCount, double approxCpu, double approxMemory, double approxDisk, Autoscaling autoscaling) { - assertTrue("Resources are present: " + message + " (" + autoscaling + ": " + autoscaling.status() + ")", + assertTrue("Resources should be present: " + message + " (" + autoscaling + ": " + autoscaling.status() + ")", autoscaling.resources().isPresent()); var resources = autoscaling.resources().get(); assertResources(message, nodeCount, groupCount, approxCpu, approxMemory, approxDisk, resources); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/NodeCandidateTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/NodeCandidateTest.java index ba35aa67dac..3f5992b2a64 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/NodeCandidateTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/NodeCandidateTest.java @@ -23,17 +23,17 @@ public class NodeCandidateTest { @Test public void testOrdering() { List<NodeCandidate> expected = List.of( - new NodeCandidate.ConcreteNodeCandidate(node("01", Node.State.ready), false, new NodeResources(2, 2, 2, 2), Optional.empty(), false, true, true, false, false), - new NodeCandidate.ConcreteNodeCandidate(node("02", Node.State.active), false, new NodeResources(2, 2, 2, 2), Optional.empty(), true, true, false, false, false), - new NodeCandidate.ConcreteNodeCandidate(node("04", Node.State.reserved), false, new NodeResources(2, 2, 2, 2), Optional.empty(), true, true, false, false, false), - new NodeCandidate.ConcreteNodeCandidate(node("03", Node.State.inactive), false, new NodeResources(2, 2, 2, 2), Optional.empty(), true, true, false, false, false), - new NodeCandidate.ConcreteNodeCandidate(node("05", Node.State.ready), false, new NodeResources(2, 2, 2, 2), Optional.of(node("host1", Node.State.active)), true, true, false, true, false), - new NodeCandidate.ConcreteNodeCandidate(node("06", Node.State.ready), false, new NodeResources(2, 2, 2, 2), Optional.of(node("host1", Node.State.ready)), true, true, false, true, false), - new NodeCandidate.ConcreteNodeCandidate(node("07", Node.State.ready), false, new NodeResources(2, 2, 2, 2), Optional.of(node("host1", Node.State.provisioned)), true, true, false, true, false), - new NodeCandidate.ConcreteNodeCandidate(node("08", Node.State.ready), false, new NodeResources(2, 2, 2, 2), Optional.of(node("host1", Node.State.failed)), true, true, false, true, false), - new NodeCandidate.ConcreteNodeCandidate(node("09", Node.State.ready), false, new NodeResources(1, 1, 1, 1), Optional.empty(), true, true, false, true, false), - new NodeCandidate.ConcreteNodeCandidate(node("10", Node.State.ready), false, new NodeResources(2, 2, 2, 2), Optional.empty(), true, true, false, true, false), - new NodeCandidate.ConcreteNodeCandidate(node("11", Node.State.ready), false, new NodeResources(2, 2, 2, 2), Optional.empty(), true, true, false, true, false) + new NodeCandidate.ConcreteNodeCandidate(node("01", Node.State.ready), false, new NodeResources(2, 2, 2, 2), Optional.empty(), false, true, false, true, false, false), + new NodeCandidate.ConcreteNodeCandidate(node("02", Node.State.active), false, new NodeResources(2, 2, 2, 2), Optional.empty(), true, true, false, false, false, false), + new NodeCandidate.ConcreteNodeCandidate(node("04", Node.State.reserved), false, new NodeResources(2, 2, 2, 2), Optional.empty(), true, true, false, false, false, false), + new NodeCandidate.ConcreteNodeCandidate(node("03", Node.State.inactive), false, new NodeResources(2, 2, 2, 2), Optional.empty(), true, true, false, false, false, false), + new NodeCandidate.ConcreteNodeCandidate(node("05", Node.State.ready), false, new NodeResources(2, 2, 2, 2), Optional.of(node("host1", Node.State.active)), true, true, false, false, true, false), + new NodeCandidate.ConcreteNodeCandidate(node("06", Node.State.ready), false, new NodeResources(2, 2, 2, 2), Optional.of(node("host1", Node.State.ready)), true, true, false, false, true, false), + new NodeCandidate.ConcreteNodeCandidate(node("07", Node.State.ready), false, new NodeResources(2, 2, 2, 2), Optional.of(node("host1", Node.State.provisioned)), true, true, false, false, true, false), + new NodeCandidate.ConcreteNodeCandidate(node("08", Node.State.ready), false, new NodeResources(2, 2, 2, 2), Optional.of(node("host1", Node.State.failed)), true, true, false, false, true, false), + new NodeCandidate.ConcreteNodeCandidate(node("09", Node.State.ready), false, new NodeResources(1, 1, 1, 1), Optional.empty(), true, true, false, false, true, false), + new NodeCandidate.ConcreteNodeCandidate(node("10", Node.State.ready), false, new NodeResources(2, 2, 2, 2), Optional.empty(), true, true, false, false, true, false), + new NodeCandidate.ConcreteNodeCandidate(node("11", Node.State.ready), false, new NodeResources(2, 2, 2, 2), Optional.empty(), true, true, false, false, true, false) ); assertOrder(expected); } @@ -148,7 +148,7 @@ public class NodeCandidateTest { .ipConfig(IP.Config.of(List.of("::1"), List.of("::2"))) .build(); return new NodeCandidate.ConcreteNodeCandidate(node, false, totalHostResources.subtract(allocatedHostResources), Optional.of(parent), - false, exclusiveSwitch, false, true, false); + false, exclusiveSwitch, false, false, true, false); } private static NodeCandidate node(String hostname, NodeResources nodeResources, diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java index d25dfd44b1e..8ba9fbdf6d2 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java @@ -17,6 +17,9 @@ import com.yahoo.config.provision.HostFilter; import com.yahoo.config.provision.HostSpec; import com.yahoo.config.provision.NodeAllocationException; import com.yahoo.config.provision.NodeResources; +import com.yahoo.config.provision.NodeResources.Architecture; +import com.yahoo.config.provision.NodeResources.DiskSpeed; +import com.yahoo.config.provision.NodeResources.StorageType; import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.ParentHostUnavailableException; import com.yahoo.config.provision.RegionName; @@ -411,8 +414,8 @@ public class ProvisioningTest { tester); assertEquals(6, state.allHosts.size()); tester.activate(application, state.allHosts); - assertTrue(state.allHosts.stream().allMatch(host -> host.requestedResources().get().diskSpeed() == NodeResources.DiskSpeed.any)); - assertTrue(tester.nodeRepository().nodes().list().owner(application).stream().allMatch(node -> node.allocation().get().requestedResources().diskSpeed() == NodeResources.DiskSpeed.any)); + assertTrue(state.allHosts.stream().allMatch(host -> host.requestedResources().get().diskSpeed() == DiskSpeed.any)); + assertTrue(tester.nodeRepository().nodes().list().owner(application).stream().allMatch(node -> node.allocation().get().requestedResources().diskSpeed() == DiskSpeed.any)); } { @@ -423,8 +426,8 @@ public class ProvisioningTest { tester); assertEquals(8, state.allHosts.size()); tester.activate(application, state.allHosts); - assertTrue(state.allHosts.stream().allMatch(host -> host.requestedResources().get().diskSpeed() == NodeResources.DiskSpeed.fast)); - assertTrue(tester.nodeRepository().nodes().list().owner(application).stream().allMatch(node -> node.allocation().get().requestedResources().diskSpeed() == NodeResources.DiskSpeed.fast)); + assertTrue(state.allHosts.stream().allMatch(host -> host.requestedResources().get().diskSpeed() == DiskSpeed.fast)); + assertTrue(tester.nodeRepository().nodes().list().owner(application).stream().allMatch(node -> node.allocation().get().requestedResources().diskSpeed() == DiskSpeed.fast)); } { @@ -434,8 +437,8 @@ public class ProvisioningTest { tester); assertEquals(8, state.allHosts.size()); tester.activate(application, state.allHosts); - assertTrue(state.allHosts.stream().allMatch(host -> host.requestedResources().get().diskSpeed() == NodeResources.DiskSpeed.any)); - assertTrue(tester.nodeRepository().nodes().list().owner(application).stream().allMatch(node -> node.allocation().get().requestedResources().diskSpeed() == NodeResources.DiskSpeed.any)); + assertTrue(state.allHosts.stream().allMatch(host -> host.requestedResources().get().diskSpeed() == DiskSpeed.any)); + assertTrue(tester.nodeRepository().nodes().list().owner(application).stream().allMatch(node -> node.allocation().get().requestedResources().diskSpeed() == DiskSpeed.any)); } } @@ -1074,10 +1077,25 @@ public class ProvisioningTest { } @Test + public void no_arm64_architecture() { + ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.dev, RegionName.from("us-east"))).build(); + + NodeResources nodeResources = new NodeResources(1, 4, 10, 4, DiskSpeed.fast, StorageType.any, Architecture.x86_64); + tester.makeReadyHosts(4, nodeResources); + tester.activateTenantHosts(); + + ApplicationId application = ProvisioningTester.applicationId(); + assertTrue(assertThrows(NodeAllocationException.class, + () -> prepare(application, 1, 1, 1, 1, nodeResources.with(Architecture.arm64), tester)) + .getMessage().startsWith("Could not satisfy request")); + + } + + @Test public void arm64_architecture() { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.dev, RegionName.from("us-east"))).build(); - NodeResources nodeResources = new NodeResources(1, 4, 10, 4, NodeResources.DiskSpeed.any, NodeResources.StorageType.any, NodeResources.Architecture.arm64); + NodeResources nodeResources = new NodeResources(1, 4, 10, 4, DiskSpeed.fast, StorageType.any, Architecture.arm64); tester.makeReadyHosts(4, nodeResources); tester.activateTenantHosts(); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java index 8c35f89234d..b091603aaeb 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java @@ -22,7 +22,7 @@ import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeResources.DiskSpeed; import com.yahoo.config.provision.NodeResources.StorageType; import com.yahoo.config.provision.NodeType; -import com.yahoo.config.provision.ProvisionLock; +import com.yahoo.config.provision.ApplicationMutex; import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.TenantName; @@ -220,7 +220,7 @@ public class ProvisioningTester { NestedTransaction t = new NestedTransaction(); if (parent.ipConfig().primary().isEmpty()) parent = parent.with(IP.Config.of(List.of("::" + 0 + ":0"), List.of("::" + 0 + ":2"))); - nodeRepository.nodes().activate(List.of(parent), new ApplicationTransaction(new ProvisionLock(application, () -> { }), t)); + nodeRepository.nodes().activate(List.of(parent), new ApplicationTransaction(new ApplicationMutex(application, () -> { }), t)); t.commit(); } } @@ -271,7 +271,7 @@ public class ProvisioningTester { public void deactivate(ApplicationId applicationId) { try (var lock = nodeRepository.applications().lock(applicationId)) { NestedTransaction deactivateTransaction = new NestedTransaction(); - nodeRepository.remove(new ApplicationTransaction(new ProvisionLock(applicationId, lock), + nodeRepository.remove(new ApplicationTransaction(new ApplicationMutex(applicationId, lock), deactivateTransaction)); deactivateTransaction.commit(); } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/VirtualNodeProvisioningTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/VirtualNodeProvisioningTest.java index 3edd41049f0..9a6fdedb213 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/VirtualNodeProvisioningTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/VirtualNodeProvisioningTest.java @@ -15,7 +15,7 @@ import com.yahoo.config.provision.InstanceName; import com.yahoo.config.provision.NodeAllocationException; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; -import com.yahoo.config.provision.ProvisionLock; +import com.yahoo.config.provision.ApplicationMutex; import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.TenantName; @@ -594,7 +594,7 @@ public class VirtualNodeProvisioningTest { tester.activate(app1, cluster1, Capacity.from(new ClusterResources(5, 1, r))); tester.activate(app1, cluster1, Capacity.from(new ClusterResources(2, 1, r))); - var tx = new ApplicationTransaction(new ProvisionLock(app1, tester.nodeRepository().applications().lock(app1)), new NestedTransaction()); + var tx = new ApplicationTransaction(new ApplicationMutex(app1, tester.nodeRepository().applications().lock(app1)), new NestedTransaction()); tester.nodeRepository().nodes().deactivate(tester.nodeRepository().nodes().list(Node.State.active).owner(app1).retired().asList(), tx); tx.nested().commit(); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiTest.java index 72c1e2e4ec3..9a7d2252b0e 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiTest.java @@ -1103,7 +1103,7 @@ public class NodesV2ApiTest { createIpAddresses(ipAddress) + "\"flavor\":\"" + flavor + "\"" + (reservedTo.map(tenantName -> ", \"reservedTo\":\"" + tenantName.value() + "\"").orElse("")) + - (exclusiveTo.map(appId -> ", \"exclusiveTo\":\"" + appId.serializedForm() + "\"").orElse("")) + + (exclusiveTo.map(appId -> ", \"provisionedFor\":\"" + appId.serializedForm() + "\"").orElse("")) + (switchHostname.map(s -> ", \"switchHostname\":\"" + s + "\"").orElse("")) + (additionalIpAddresses.isEmpty() ? "" : ", \"additionalIpAddresses\":[\"" + String.join("\",\"", additionalIpAddresses) + "\"]") + (additionalHostnames.isEmpty() ? "" : ", \"additionalHostnames\":[\"" + String.join("\",\"", additionalHostnames) + "\"]") + diff --git a/parent/pom.xml b/parent/pom.xml index aec0f5b88fe..98462a026d3 100644 --- a/parent/pom.xml +++ b/parent/pom.xml @@ -317,7 +317,7 @@ --> <groupId>org.openrewrite.maven</groupId> <artifactId>rewrite-maven-plugin</artifactId> - <version>5.8.1</version> + <version>5.9.1</version> <configuration> <activeRecipes> <recipe>org.openrewrite.java.testing.junit5.JUnit5BestPractices</recipe> @@ -666,7 +666,7 @@ <dependency> <groupId>commons-cli</groupId> <artifactId>commons-cli</artifactId> - <version>1.5.0</version> + <version>${commons-cli.vespa.version}</version> </dependency> <dependency> <groupId>commons-codec</groupId> @@ -933,7 +933,12 @@ <dependency> <groupId>org.apache.velocity</groupId> <artifactId>velocity-engine-core</artifactId> - <version>2.3</version> + <version>${velocity.vespa.version}</version> + </dependency> + <dependency> + <groupId>org.apache.velocity.tools</groupId> + <artifactId>velocity-tools-generic</artifactId> + <version>${velocity.tools.vespa.version}</version> </dependency> <dependency> <groupId>org.apiguardian</groupId> @@ -1116,7 +1121,7 @@ See pluginManagement of rewrite-maven-plugin for more details --> <groupId>org.openrewrite.recipe</groupId> <artifactId>rewrite-recipe-bom</artifactId> - <version>2.3.1</version> + <version>2.4.0</version> <type>pom</type> <scope>import</scope> </dependency> diff --git a/screwdriver.yaml b/screwdriver.yaml index b1844fad97d..ccab9bc4bd0 100644 --- a/screwdriver.yaml +++ b/screwdriver.yaml @@ -34,7 +34,7 @@ shared: du -sh /tmp/vespa/* if [[ -z "$SD_PULL_REQUEST" ]]; then - if [[ -z $VESPA_USE_SANITIZER ]] || [[ $VESPA_USE_SANITIZER == null ]]; then + if [[ -z "$VESPA_USE_SANITIZER" ]] || [[ "$VESPA_USE_SANITIZER" == null ]]; then # Remove what we have produced rm -rf $LOCAL_MVN_REPO/com/yahoo rm -rf $LOCAL_MVN_REPO/ai/vespa @@ -117,6 +117,7 @@ jobs: (got VESPA_VERSION=$VESPA_VERSION, VESPA_REF=$VESPA_REF, SYSTEM_TEST_REF=$SYSTEM_TEST_REF)." exit 1 fi + meta set vespa.version $VESPA_VERSION - install-dependencies: | dnf config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo dnf -y install docker-ce docker-ce-cli containerd.io @@ -180,6 +181,7 @@ jobs: cp -a $LOCAL_MVN_REPO docker/repository cd docker docker build --file Dockerfile.systemtest \ + --progress=plain \ --build-arg VESPA_BASE_IMAGE=vespaengine/vespa-systemtest-base-centos-stream8:latest \ --build-arg SYSTEMTEST_BASE_IMAGE=vespa --build-arg SKIP_M2_POPULATE=false \ --target systemtest \ @@ -237,6 +239,7 @@ jobs: - DOCKER_IMAGE_DEPLOY_KEY - DOCKER_HUB_DEPLOY_KEY - GHCR_DEPLOY_KEY + - ANN_BENCHMARK_DEPLOY_KEY - SVC_OKTA_VESPA_FACTORY_TOKEN environment: @@ -264,6 +267,8 @@ jobs: screwdriver/release-rpms.sh $VESPA_VERSION $VESPA_REF - release-container-image: | screwdriver/release-container-image-docker.sh $VESPA_VERSION + - release-ann-benchmark: | + screwdriver/release-ann-benchmark.sh $VESPA_VERSION - update-sample-apps: | screwdriver/update-vespa-version-in-sample-apps.sh $VESPA_VERSION - update-released-time: | @@ -362,12 +367,13 @@ jobs: screwdriver.cd/buildPeriodically: H 6 1 * * environment: - IMAGE_NAME: "vespaengine/vespa-el9-preview" + BASE_IMAGE: "el9" + IMAGE_NAME: "vespa-el9-preview" secrets: - DOCKER_HUB_DEPLOY_KEY - steps: + steps: &publish-el9-preview-steps - get-vespa-version: | set -x VESPA_VERSION=$(meta get vespa.version --external publish-release) @@ -394,9 +400,10 @@ jobs: --progress plain \ --load \ --platform linux/amd64 \ - --build-arg VESPA_BASE_IMAGE=el9 \ + --build-arg VESPA_BASE_IMAGE=$BASE_IMAGE \ --build-arg VESPA_VERSION=$VESPA_VERSION \ --file Dockerfile \ + --tag vespaengine/vespa:latest \ --tag vespaengine/$IMAGE_NAME:latest \ . - verify-container-image: | @@ -415,7 +422,7 @@ jobs: --progress plain \ --push \ --platform linux/amd64,linux/arm64 \ - --build-arg VESPA_BASE_IMAGE=el9 \ + --build-arg VESPA_BASE_IMAGE=$BASE_IMAGE \ --build-arg VESPA_VERSION=$VESPA_VERSION \ --file Dockerfile \ --tag docker.io/vespaengine/$IMAGE_NAME:$VESPA_VERSION \ @@ -425,6 +432,24 @@ jobs: fi fi + publish-el8-preview: + image: docker.io/vespaengine/vespa-build-centos-stream8:latest + annotations: + screwdriver.cd/cpu: 7 + screwdriver.cd/ram: 16 + screwdriver.cd/disk: HIGH + screwdriver.cd/timeout: 300 + screwdriver.cd/dockerEnabled: true + screwdriver.cd/dockerCpu: TURBO + screwdriver.cd/dockerRam: HIGH + screwdriver.cd/buildPeriodically: H 6 1 * * + environment: + BASE_IMAGE: "el8" + IMAGE_NAME: "vespa-el8-preview" + secrets: + - DOCKER_HUB_DEPLOY_KEY + steps: *publish-el9-preview-steps + publish-cli-release: requires: [publish-release] image: homebrew/brew:latest @@ -532,6 +557,6 @@ jobs: --typhoeus '{"connecttimeout": 10, "timeout": 30, "followlocation": false}' \ --hydra '{"max_concurrency": 1}' \ --ignore-urls '/slack.vespa.ai/,/localhost:8080/,/127.0.0.1:3000/,/favicon.svg/,/main.jsx/' \ - --ignore-files '/fnet/index.html/' \ + --ignore-files '/fnet/index.html/,/client/js/app/node_modules/,/controller-server/src/test/resources/mail/' \ --swap-urls '(.*).md:\1.html' \ _site diff --git a/screwdriver/build-vespa.sh b/screwdriver/build-vespa.sh index 1f43770b871..72b26e1032e 100755 --- a/screwdriver/build-vespa.sh +++ b/screwdriver/build-vespa.sh @@ -24,8 +24,7 @@ fi build_cpp() { cat /proc/cpuinfo | grep "model name" | head -1 cat /proc/cpuinfo | grep "flags" | head -1 - # TODO This will only build for x86_64 architecture, and is used for pull request builds. - cmake3 -DVESPA_UNPRIVILEGED=no -DDEFAULT_VESPA_CPU_ARCH_FLAGS="-march=skylake" $1 + cmake3 -DVESPA_UNPRIVILEGED=no $1 time make -j ${NUM_THREADS} time ctest3 --output-on-failure -j ${NUM_THREADS} ccache --show-stats diff --git a/screwdriver/release-ann-benchmark.sh b/screwdriver/release-ann-benchmark.sh new file mode 100755 index 00000000000..7ef7e4df68c --- /dev/null +++ b/screwdriver/release-ann-benchmark.sh @@ -0,0 +1,32 @@ +#!/usr/bin/ssh-agent /bin/bash +# Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +set -euo pipefail + +if [[ $# -ne 1 ]]; then + echo "Usage: $0 <Vespa version>" + exit 1 +fi + +readonly VESPA_VERSION=$1 + +if [[ -z "$ANN_BENCHMARK_DEPLOY_KEY" ]]; then + echo "Environment variable ANN_BENCHMARK_DEPLOY_KEY must be set, but is empty." + exit 1 +fi + +BUILD_DIR=$(mktemp -d) +trap "rm -rf $BUILD_DIR" EXIT +cd $BUILD_DIR + +ssh-add -D +ssh-add <(echo $ANN_BENCHMARK_DEPLOY_KEY | base64 -d) +git clone git@github.com:vespa-engine/vespa-ann-benchmark +cd vespa-ann-benchmark + +RELEASE_TAG="v$VESPA_VERSION" +if ! git rev-parse $RELEASE_TAG &> /dev/null; then + git tag -a "$RELEASE_TAG" -m "Release version $VESPA_VERSION" + git push origin "$RELEASE_TAG" +fi + diff --git a/searchcore/src/tests/proton/common/timer/timer_test.cpp b/searchcore/src/tests/proton/common/timer/timer_test.cpp index 1efae97c6e0..4419275230a 100644 --- a/searchcore/src/tests/proton/common/timer/timer_test.cpp +++ b/searchcore/src/tests/proton/common/timer/timer_test.cpp @@ -84,7 +84,7 @@ TYPED_TEST(ScheduledExecutorTest, test_drop_handle) { } TYPED_TEST(ScheduledExecutorTest, test_only_one_instance_running) { - vespalib::TimeBomb time_bomb(60s); + vespalib::TimeBomb time_bomb(120s); vespalib::Gate latch; std::atomic<uint64_t> counter = 0; auto handleA = this->timer->scheduleAtFixedRate(makeLambdaTask([&]() { counter++; latch.await();}), 0ms, 1ms); @@ -96,7 +96,7 @@ TYPED_TEST(ScheduledExecutorTest, test_only_one_instance_running) { } TYPED_TEST(ScheduledExecutorTest, test_sync_delete) { - vespalib::TimeBomb time_bomb(60s); + vespalib::TimeBomb time_bomb(120s); vespalib::Gate latch; std::atomic<uint64_t> counter = 0; std::atomic<uint64_t> reset_counter = 0; diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.cpp index 8e2b097cd35..f81b47583b9 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.cpp +++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.cpp @@ -549,6 +549,7 @@ public: for (auto lidToRemove : _lidsToRemove) { applyRemoveToAttribute(_serialNum, lidToRemove, attr, _onWriteDone); } + attr.commitIfChangeVectorTooLarge(); } } } diff --git a/searchcore/src/vespa/searchcore/proton/flushengine/CMakeLists.txt b/searchcore/src/vespa/searchcore/proton/flushengine/CMakeLists.txt index 64cbde68416..bcf9848ff66 100644 --- a/searchcore/src/vespa/searchcore/proton/flushengine/CMakeLists.txt +++ b/searchcore/src/vespa/searchcore/proton/flushengine/CMakeLists.txt @@ -13,6 +13,7 @@ vespa_add_library(searchcore_flushengine STATIC flushtargetproxy.cpp flushtask.cpp prepare_restart_flush_strategy.cpp + priority_flush_token.cpp threadedflushtarget.cpp tls_stats_factory.cpp tls_stats_map.cpp diff --git a/searchcore/src/vespa/searchcore/proton/flushengine/flushengine.cpp b/searchcore/src/vespa/searchcore/proton/flushengine/flushengine.cpp index fc08d4d8a17..768800ee781 100644 --- a/searchcore/src/vespa/searchcore/proton/flushengine/flushengine.cpp +++ b/searchcore/src/vespa/searchcore/proton/flushengine/flushengine.cpp @@ -66,16 +66,18 @@ FlushEngine::FlushMeta::~FlushMeta() = default; FlushEngine::FlushInfo::FlushInfo() : FlushMeta("", "", 0), - _target() + _target(), + _priority_flush_token() { } FlushEngine::FlushInfo::~FlushInfo() = default; -FlushEngine::FlushInfo::FlushInfo(uint32_t taskId, const vespalib::string& handler_name, const IFlushTarget::SP& target) +FlushEngine::FlushInfo::FlushInfo(uint32_t taskId, const vespalib::string& handler_name, const IFlushTarget::SP& target, std::shared_ptr<PriorityFlushToken> priority_flush_token) : FlushMeta(handler_name, target->getName(), taskId), - _target(target) + _target(target), + _priority_flush_token(std::move(priority_flush_token)) { } @@ -89,6 +91,7 @@ FlushEngine::FlushEngine(std::shared_ptr<flushengine::ITlsStatsFactory> tlsStats _has_thread(false), _strategy(std::move(strategy)), _priorityStrategy(), + _priority_flush_token(), _executor(maxConcurrentTotal(), CpuUsage::wrap(flush_engine_executor, CpuUsage::Category::COMPACT)), _lock(), _cond(), @@ -249,7 +252,7 @@ createName(const IFlushHandler &handler, const vespalib::string &targetName) bool FlushEngine::prune() { - std::set<IFlushHandler::SP> toPrune; + PendingPrunes toPrune; { std::lock_guard<std::mutex> guard(_lock); if (_pendingPrune.empty()) { @@ -257,7 +260,8 @@ FlushEngine::prune() } _pendingPrune.swap(toPrune); } - for (const auto &handler : toPrune) { + for (const auto& kv : toPrune) { + const auto& handler = kv.first; IFlushTarget::List lst = handler->getFlushTargets(); auto oldestFlushed = findOldestFlushedTarget(lst, *handler); if (LOG_WOULD_LOG(event)) { @@ -368,21 +372,21 @@ FlushEngine::initNextFlush(const FlushContext::List &lst) void FlushEngine::flushAll(const FlushContext::List &lst) { + mark_currently_flushing_tasks(_priority_flush_token); LOG(debug, "%ld targets to flush.", lst.size()); for (const FlushContext::SP & ctx : lst) { if (wait_for_slot(IFlushTarget::Priority::NORMAL)) { if (ctx->initFlush(get_flush_token(*ctx))) { logTarget("initiated", *ctx); - _executor.execute(std::make_unique<FlushTask>(initFlush(*ctx), *this, ctx)); + _executor.execute(std::make_unique<FlushTask>(initFlush(*ctx, _priority_flush_token), *this, ctx)); } else { logTarget("failed to initiate", *ctx); } } } - _executor.sync(); - prune(); std::lock_guard<std::mutex> strategyGuard(_strategyLock); _priorityStrategy.reset(); + _priority_flush_token.reset(); _strategyCond.notify_all(); } @@ -404,19 +408,19 @@ FlushEngine::flushNextTarget(const vespalib::string & name, const FlushContext:: name.c_str(), contexts.size()); std::this_thread::sleep_for(100ms); } - _executor.execute(std::make_unique<FlushTask>(initFlush(*ctx), *this, ctx)); + _executor.execute(std::make_unique<FlushTask>(initFlush(*ctx, {}), *this, ctx)); return ctx->getName(); } uint32_t -FlushEngine::initFlush(const FlushContext &ctx) +FlushEngine::initFlush(const FlushContext &ctx, std::shared_ptr<PriorityFlushToken> priority_flush_token) { if (LOG_WOULD_LOG(event)) { IFlushTarget::MemoryGain mgain(ctx.getTarget()->getApproxMemoryGain()); EventLogger::flushStart(ctx.getName(), mgain.getBefore(), mgain.getAfter(), mgain.gain(), ctx.getTarget()->getFlushedSerialNum() + 1, ctx.getHandler()->getCurrentSerialNumber()); } - return initFlush(ctx.getHandler(), ctx.getTarget()); + return initFlush(ctx.getHandler(), ctx.getTarget(), std::move(priority_flush_token)); } void @@ -434,10 +438,25 @@ FlushEngine::flushDone(const FlushContext &ctx, uint32_t taskId) } LOG(debug, "FlushEngine::flushDone(taskId='%d') took '%f' secs", taskId, vespalib::to_s(duration)); std::lock_guard<std::mutex> guard(_lock); - _flushing.erase(taskId); + /* + * Hand over any priority flush token for completed flush to + * _pendingPrune, to ensure that setStrategy will wait until + * flush engine has called prune(). + */ + std::shared_ptr<PriorityFlushToken> priority_flush_token; + { + auto itr = _flushing.find(taskId); + if (itr != _flushing.end()) { + priority_flush_token = std::move(itr->second._priority_flush_token); + _flushing.erase(itr); + } + } assert(ctx.getHandler()); if (_handlers.hasHandler(ctx.getHandler())) { - _pendingPrune.insert(ctx.getHandler()); + auto ins_res = _pendingPrune.emplace(ctx.getHandler(), PendingPrunes::mapped_type()); + if (priority_flush_token) { + ins_res.first->second = std::move(priority_flush_token); + } } _cond.notify_all(); } @@ -450,7 +469,7 @@ FlushEngine::putFlushHandler(const DocTypeName &docTypeName, const IFlushHandler if (result) { _pendingPrune.erase(result); } - _pendingPrune.insert(flushHandler); + _pendingPrune.emplace(flushHandler, PendingPrunes::mapped_type()); return result; } @@ -475,13 +494,13 @@ FlushEngine::getCurrentlyFlushingSet() const } uint32_t -FlushEngine::initFlush(const IFlushHandler::SP &handler, const IFlushTarget::SP &target) +FlushEngine::initFlush(const IFlushHandler::SP &handler, const IFlushTarget::SP &target, std::shared_ptr<PriorityFlushToken> priority_flush_token) { uint32_t taskId; { std::lock_guard<std::mutex> guard(_lock); taskId = _taskId++; - FlushInfo flush(taskId, handler->getName(), target); + FlushInfo flush(taskId, handler->getName(), target, std::move(priority_flush_token)); _flushing[taskId] = flush; } LOG(debug, "FlushEngine::initFlush(handler='%s', target='%s') => taskId='%d'", @@ -492,6 +511,9 @@ FlushEngine::initFlush(const IFlushHandler::SP &handler, const IFlushTarget::SP void FlushEngine::setStrategy(IFlushStrategy::SP strategy) { + std::promise<void> promise; + auto future = promise.get_future(); + auto priority_flush_token = std::make_shared<PriorityFlushToken>(std::move(promise)); std::lock_guard<std::mutex> setStrategyGuard(_setStrategyLock); std::unique_lock<std::mutex> strategyGuard(_strategyLock); if (_closed.load(std::memory_order_relaxed)) { @@ -499,6 +521,7 @@ FlushEngine::setStrategy(IFlushStrategy::SP strategy) } assert(!_priorityStrategy); _priorityStrategy = std::move(strategy); + _priority_flush_token = std::move(priority_flush_token); { std::lock_guard<std::mutex> guard(_lock); _cond.notify_all(); @@ -506,6 +529,22 @@ FlushEngine::setStrategy(IFlushStrategy::SP strategy) while (_priorityStrategy) { _strategyCond.wait(strategyGuard); } + strategyGuard.unlock(); + /* + * Wait for flushes started before the strategy change, for + * flushes initiated by the strategy, and for flush engine to call + * prune() afterwards. + */ + future.wait(); +} + +void +FlushEngine::mark_currently_flushing_tasks(std::shared_ptr<PriorityFlushToken> priority_flush_token) +{ + std::lock_guard<std::mutex> guard(_lock); + for (auto& kv : _flushing) { + kv.second._priority_flush_token = priority_flush_token; + } } } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/flushengine/flushengine.h b/searchcore/src/vespa/searchcore/proton/flushengine/flushengine.h index ec0f019f6e4..302c4a2499e 100644 --- a/searchcore/src/vespa/searchcore/proton/flushengine/flushengine.h +++ b/searchcore/src/vespa/searchcore/proton/flushengine/flushengine.h @@ -3,6 +3,7 @@ #include "flushcontext.h" #include "iflushstrategy.h" +#include "priority_flush_token.h" #include <vespa/searchcore/proton/common/handlermap.hpp> #include <vespa/searchcore/proton/common/doctypename.h> #include <vespa/vespalib/util/threadstackexecutor.h> @@ -43,13 +44,15 @@ private: struct FlushInfo : public FlushMeta { FlushInfo(); - FlushInfo(uint32_t taskId, const vespalib::string& handler_name, const IFlushTarget::SP &target); + FlushInfo(uint32_t taskId, const vespalib::string& handler_name, const IFlushTarget::SP &target, std::shared_ptr<PriorityFlushToken> priority_flush_token); ~FlushInfo(); IFlushTarget::SP _target; + std::shared_ptr<PriorityFlushToken> _priority_flush_token; }; using FlushMap = std::map<uint32_t, FlushInfo>; using FlushHandlerMap = HandlerMap<IFlushHandler>; + using PendingPrunes = std::map<std::shared_ptr<IFlushHandler>, std::shared_ptr<PriorityFlushToken>>; std::atomic<bool> _closed; const uint32_t _maxConcurrentNormal; const vespalib::duration _idleInterval; @@ -58,6 +61,7 @@ private: std::atomic<bool> _has_thread; IFlushStrategy::SP _strategy; mutable IFlushStrategy::SP _priorityStrategy; + mutable std::shared_ptr<PriorityFlushToken> _priority_flush_token; vespalib::ThreadStackExecutor _executor; mutable std::mutex _lock; std::condition_variable _cond; @@ -67,7 +71,7 @@ private: std::mutex _strategyLock; std::condition_variable _strategyCond; std::shared_ptr<flushengine::ITlsStatsFactory> _tlsStatsFactory; - std::set<IFlushHandler::SP> _pendingPrune; + PendingPrunes _pendingPrune; std::shared_ptr<search::FlushToken> _normal_flush_token; std::shared_ptr<search::FlushToken> _gc_flush_token; @@ -78,8 +82,8 @@ private: vespalib::string flushNextTarget(const vespalib::string & name, const FlushContext::List & contexts); void flushAll(const FlushContext::List &lst); bool prune(); - uint32_t initFlush(const FlushContext &ctx); - uint32_t initFlush(const IFlushHandler::SP &handler, const IFlushTarget::SP &target); + uint32_t initFlush(const FlushContext &ctx, std::shared_ptr<PriorityFlushToken> priority_flush_token); + uint32_t initFlush(const IFlushHandler::SP &handler, const IFlushTarget::SP &target, std::shared_ptr<PriorityFlushToken> priority_flush_token); void flushDone(const FlushContext &ctx, uint32_t taskId); bool canFlushMore(const std::unique_lock<std::mutex> &guard, IFlushTarget::Priority priority) const; void wait_for_slot_or_pending_prune(IFlushTarget::Priority priority); @@ -179,6 +183,7 @@ public: FlushMetaSet getCurrentlyFlushingSet() const; void setStrategy(IFlushStrategy::SP strategy); + void mark_currently_flushing_tasks(std::shared_ptr<PriorityFlushToken> priority_flush_token); uint32_t maxConcurrentTotal() const { return _maxConcurrentNormal + 1; } uint32_t maxConcurrentNormal() const { return _maxConcurrentNormal; } }; diff --git a/searchcore/src/vespa/searchcore/proton/flushengine/priority_flush_token.cpp b/searchcore/src/vespa/searchcore/proton/flushengine/priority_flush_token.cpp new file mode 100644 index 00000000000..f031c19d1d8 --- /dev/null +++ b/searchcore/src/vespa/searchcore/proton/flushengine/priority_flush_token.cpp @@ -0,0 +1,17 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "priority_flush_token.h" + +namespace proton { + +PriorityFlushToken::PriorityFlushToken(std::promise<void> promise) + : _promise(std::move(promise)) +{ +} + +PriorityFlushToken::~PriorityFlushToken() +{ + _promise.set_value(); +} + +} diff --git a/searchcore/src/vespa/searchcore/proton/flushengine/priority_flush_token.h b/searchcore/src/vespa/searchcore/proton/flushengine/priority_flush_token.h new file mode 100644 index 00000000000..82048b3eb3f --- /dev/null +++ b/searchcore/src/vespa/searchcore/proton/flushengine/priority_flush_token.h @@ -0,0 +1,21 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <vespa/vespalib/util/idestructorcallback.h> +#include <future> + +namespace proton { + +/* + * This token is shared between flushes initiated from a priority flush + * strategy (cf. Proton::triggerFLush and Proton::prepareRestart). + */ +class PriorityFlushToken : public vespalib::IDestructorCallback { + std::promise<void> _promise; +public: + PriorityFlushToken(std::promise<void> promise); + ~PriorityFlushToken() override; +}; + +} diff --git a/searchcore/src/vespa/searchcore/proton/server/feedhandler.cpp b/searchcore/src/vespa/searchcore/proton/server/feedhandler.cpp index 06d201a4bda..f5774734949 100644 --- a/searchcore/src/vespa/searchcore/proton/server/feedhandler.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/feedhandler.cpp @@ -288,7 +288,6 @@ FeedHandler::performDeleteBucket(FeedToken token, DeleteBucketOperation &op) { _activeFeedView->handleDeleteBucket(op, token); // Delete bucket itself, should no longer have documents. _bucketDBHandler->handleDeleteBucket(op.getBucketId()); - initiateCommit(vespalib::steady_clock::now()); } void diff --git a/searchcore/src/vespa/searchcorespi/index/indexmaintainer.cpp b/searchcore/src/vespa/searchcorespi/index/indexmaintainer.cpp index 50bbbaec355..bbd17be9b5a 100644 --- a/searchcore/src/vespa/searchcorespi/index/indexmaintainer.cpp +++ b/searchcore/src/vespa/searchcorespi/index/indexmaintainer.cpp @@ -1362,7 +1362,6 @@ void IndexMaintainer::consider_initial_urgent_flush() { const Schema *prev_schema = nullptr; - std::optional<uint32_t> urgent_source_id; auto coll = getSourceCollection(); uint32_t count = coll->getSourceCount(); for (uint32_t i = 0; i < count; ++i) { diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/ExpressionFunction.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/ExpressionFunction.java index 7df08d7d356..093e65b2e4d 100755 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/ExpressionFunction.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/ExpressionFunction.java @@ -212,7 +212,7 @@ public class ExpressionFunction { public String toString() { return "function '" + name + "'"; } - + /** * An instance of a serialization of this function, using a particular serialization context (by {@link * ExpressionFunction#expand}) diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/SerializationContext.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/SerializationContext.java index d88bd03b7d4..d1cb77fb1b4 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/SerializationContext.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/SerializationContext.java @@ -21,7 +21,7 @@ import java.util.Optional; * @author bratseth */ public class SerializationContext extends FunctionReferenceContext { - + /** Serialized form of functions indexed by name */ private final Map<String, String> serializedFunctions; diff --git a/searchlib/src/main/java/com/yahoo/searchlib/tensor/EvaluateTensorConformance.java b/searchlib/src/main/java/com/yahoo/searchlib/tensor/EvaluateTensorConformance.java index dcdf2f532e4..69c8a091cce 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/tensor/EvaluateTensorConformance.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/tensor/EvaluateTensorConformance.java @@ -71,7 +71,7 @@ public class EvaluateTensorConformance { System.exit(1); } } - + private boolean testCase(String test, int count) { boolean wasOk = false; try { diff --git a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/RankingExpressionTestCase.java b/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/RankingExpressionTestCase.java index e9b1e3bda0d..8af77ec1cdd 100755 --- a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/RankingExpressionTestCase.java +++ b/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/RankingExpressionTestCase.java @@ -61,7 +61,7 @@ public class RankingExpressionTestCase { assertParse("query(var1) + query(var2) - query(var3) * (query(var4) / query(var5))", " $var1 + $var2 - $var3 *($var4 / $var5)"); assertParse("if (if (f1.out < query(p1), 0, 1) < if (f2.out < query(p2), 0, 1), f3.out, query(p3))", "if(if(f1.out<$p1,0,1)<if(f2.out<$p2,0,1),f3.out,$p3)"); } - + @Test public void testProgrammaticBuilding() throws ParseException { ReferenceNode input = new ReferenceNode("input"); @@ -147,14 +147,14 @@ public class RankingExpressionTestCase { "10 + 8 * 1977"), "cox", functions ); } - + @Test public void testTensorSerialization() { - assertSerialization("map(constant(tensor0), f(a)(cos(a)))", + assertSerialization("map(constant(tensor0), f(a)(cos(a)))", "map(constant(tensor0), f(a)(cos(a)))"); - assertSerialization("map(constant(tensor0), f(a)(cos(a))) + join(attribute(tensor1), map(reduce(map(attribute(tensor1), f(a)(a * a)), sum, x), f(a)(sqrt(a))), f(a,b)(a / b))", + assertSerialization("map(constant(tensor0), f(a)(cos(a))) + join(attribute(tensor1), map(reduce(map(attribute(tensor1), f(a)(a * a)), sum, x), f(a)(sqrt(a))), f(a,b)(a / b))", "map(constant(tensor0), f(a)(cos(a))) + l2_normalize(attribute(tensor1), x)"); - assertSerialization("join(reduce(join(reduce(join(constant(tensor0), attribute(tensor1), f(a,b)(a * b)), sum, x), attribute(tensor1), f(a,b)(a * b)), sum, y), query(tensor2), f(a,b)(a + b))", + assertSerialization("join(reduce(join(reduce(join(constant(tensor0), attribute(tensor1), f(a,b)(a * b)), sum, x), attribute(tensor1), f(a,b)(a * b)), sum, y), query(tensor2), f(a,b)(a + b))", "xw_plus_b(matmul(constant(tensor0), attribute(tensor1), x), attribute(tensor1), query(tensor2), y)"); assertSerialization("tensor(x{}):{{x:a}:(1 + 2 + 3),{x:b}:(if (1 > 2, 3, 4)),{x:c}:(reduce(tensor0 * tensor1, sum))}", "tensor(x{}):{ {x:a}:1+2+3, {x:b}:if(1>2,3,4), {x:c}:sum(tensor0*tensor1) }"); @@ -378,7 +378,7 @@ public class RankingExpressionTestCase { // (but not the same one due to primitivization) RankingExpression reparsedExpression = new RankingExpression(serializedExpression); // Serializing the primitivized expression should yield the same expression again - String reserializedExpression = + String reserializedExpression = reparsedExpression.getRankProperties(new SerializationContext()).values().iterator().next(); assertEquals(expectedSerialization, reserializedExpression); } diff --git a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/evaluation/EvaluationBenchmark.java b/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/evaluation/EvaluationBenchmark.java index 955ca05ce37..f0ce613e27f 100644 --- a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/evaluation/EvaluationBenchmark.java +++ b/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/evaluation/EvaluationBenchmark.java @@ -209,7 +209,7 @@ public class EvaluationBenchmark { if (Math.abs(a-b) >= Math.abs((a+b)/100000000) ) throw new RuntimeException("Expected value " + a + " but optimized evaluation produced " + b); } - + private final String gbdt = "if (LW_NEWS_SEARCHES_RATIO < 1.72971, 0.0697159, if (LW_USERS < 0.10496, if (SEARCHES < 0.0329127, 0.151257, 0.117501), if (SUGG_OVERLAP < 18.5, 0.0897622, 0.0756903))) + \n" + "if (LW_NEWS_SEARCHES_RATIO < 1.73156, if (NEWS_USERS < 0.0737993, -0.00481646, 0.00110018), if (LW_USERS < 0.0844616, 0.0488919, if (SUGG_OVERLAP < 32.5, 0.0136917, 9.85328E-4))) + \n" + diff --git a/searchlib/src/tests/docstore/file_chunk/file_chunk_test.cpp b/searchlib/src/tests/docstore/file_chunk/file_chunk_test.cpp index 54d7770e271..8f506b7ca2d 100644 --- a/searchlib/src/tests/docstore/file_chunk/file_chunk_test.cpp +++ b/searchlib/src/tests/docstore/file_chunk/file_chunk_test.cpp @@ -202,7 +202,7 @@ assertUpdateLidMap(FixtureType &f) f.assertBucketizer(expLids); size_t entrySize = 10 + 8; EXPECT_EQUAL(9 * entrySize, f.chunk.getAddedBytes()); - EXPECT_EQUAL(3u, f.chunk.getBloatCount()); + EXPECT_EQUAL(3u, f.chunk.getErasedCount()); EXPECT_EQUAL(3 * entrySize, f.chunk.getErasedBytes()); } diff --git a/searchlib/src/tests/tensor/dense_tensor_store/dense_tensor_store_test.cpp b/searchlib/src/tests/tensor/dense_tensor_store/dense_tensor_store_test.cpp index 29242e2cb90..0e173691f99 100644 --- a/searchlib/src/tests/tensor/dense_tensor_store/dense_tensor_store_test.cpp +++ b/searchlib/src/tests/tensor/dense_tensor_store/dense_tensor_store_test.cpp @@ -1,8 +1,6 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/log/log.h> -LOG_SETUP("dense_tensor_store_test"); + #include <vespa/vespalib/testkit/test_kit.h> -#include <vespa/vespalib/util/memory_allocator.h> #include <vespa/searchlib/tensor/dense_tensor_store.h> #include <vespa/eval/eval/simple_value.h> #include <vespa/eval/eval/tensor_spec.h> @@ -11,6 +9,9 @@ LOG_SETUP("dense_tensor_store_test"); #include <vespa/eval/eval/test/value_compare.h> #include <vespa/vespalib/util/size_literals.h> +#include <vespa/log/log.h> +LOG_SETUP("dense_tensor_store_test"); + using search::tensor::DenseTensorStore; using vespalib::eval::SimpleValue; using vespalib::eval::TensorSpec; @@ -28,7 +29,7 @@ makeTensor(const TensorSpec &spec) struct Fixture { DenseTensorStore store; - Fixture(const vespalib::string &tensorType) + explicit Fixture(const vespalib::string &tensorType) : store(ValueType::from_spec(tensorType), {}) {} void assertSetAndGetTensor(const TensorSpec &tensorSpec) { @@ -38,14 +39,14 @@ struct Fixture EXPECT_EQUAL(*expTensor, *actTensor); assertTensorView(ref, *expTensor); } - void assertEmptyTensor(const TensorSpec &tensorSpec) { + void assertEmptyTensor(const TensorSpec &tensorSpec) const { Value::UP expTensor = makeTensor(tensorSpec); EntryRef ref; Value::UP actTensor = store.get_tensor(ref); EXPECT_TRUE(actTensor.get() == nullptr); assertTensorView(ref, *expTensor); } - void assertTensorView(EntryRef ref, const Value &expTensor) { + void assertTensorView(EntryRef ref, const Value &expTensor) const { auto cells = store.get_typed_cells(ref); vespalib::eval::DenseValueView actTensor(store.type(), cells); EXPECT_EQUAL(expTensor, actTensor); @@ -101,6 +102,7 @@ assert_max_buffer_entries(const vespalib::string& tensor_type, uint32_t exp_entr TEST("require that max entries is calculated correctly") { TEST_DO(assert_max_buffer_entries("tensor(x[1])", 1_Mi)); + TEST_DO(assert_max_buffer_entries("tensor(x[32])", 1_Mi)); TEST_DO(assert_max_buffer_entries("tensor(x[64])", 512_Ki)); TEST_DO(assert_max_buffer_entries("tensor(x[1024])", 32_Ki)); @@ -109,7 +111,6 @@ TEST("require that max entries is calculated correctly") TEST_DO(assert_max_buffer_entries("tensor(x[33554428])", 2)); TEST_DO(assert_max_buffer_entries("tensor(x[33554429])", 1)); TEST_DO(assert_max_buffer_entries("tensor(x[33554432])", 1)); - TEST_DO(assert_max_buffer_entries("tensor(x[303554432])", 1)); } TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchlib/src/vespa/searchcommon/common/schema.h b/searchlib/src/vespa/searchcommon/common/schema.h index e5c27d22e08..bdd9f3d981f 100644 --- a/searchlib/src/vespa/searchcommon/common/schema.h +++ b/searchlib/src/vespa/searchcommon/common/schema.h @@ -74,7 +74,6 @@ public: class IndexField : public Field { private: uint32_t _avgElemLen; - // TODO: Remove when posting list format with interleaved features is made default bool _interleaved_features; public: diff --git a/searchlib/src/vespa/searchlib/attribute/postinglistsearchcontext.cpp b/searchlib/src/vespa/searchlib/attribute/postinglistsearchcontext.cpp index 5f9a44f691c..c0d7cb207bf 100644 --- a/searchlib/src/vespa/searchlib/attribute/postinglistsearchcontext.cpp +++ b/searchlib/src/vespa/searchlib/attribute/postinglistsearchcontext.cpp @@ -24,11 +24,9 @@ PostingListSearchContext(const IEnumStoreDictionary& dictionary, bool has_btree_ _dictSize(_frozenDictionary.size()), _pidx(), _frozenRoot(), - _FSTC(0.0), - _PLSTC(0.0), _hasWeight(hasWeight), _useBitVector(useBitVector), - _counted_hits() + _estimated_hits() { } @@ -72,6 +70,17 @@ PostingListSearchContext::lookupSingle() } } +size_t +PostingListSearchContext::estimated_hits_in_range() const +{ + if (_estimated_hits.has_value()) { + return _estimated_hits.value(); + } + size_t result = calc_estimated_hits_in_range(); + _estimated_hits = result; + return result; +} + template class PostingListSearchContextT<vespalib::btree::BTreeNoLeafData>; template class PostingListSearchContextT<int32_t>; template class PostingListFoldedSearchContextT<vespalib::btree::BTreeNoLeafData>; diff --git a/searchlib/src/vespa/searchlib/attribute/postinglistsearchcontext.h b/searchlib/src/vespa/searchlib/attribute/postinglistsearchcontext.h index 91383bfe5f9..562c15e94d5 100644 --- a/searchlib/src/vespa/searchlib/attribute/postinglistsearchcontext.h +++ b/searchlib/src/vespa/searchlib/attribute/postinglistsearchcontext.h @@ -35,10 +35,6 @@ protected: using EntryRef = vespalib::datastore::EntryRef; using EnumIndex = IEnumStore::Index; - static constexpr long MIN_UNIQUE_VALUES_BEFORE_APPROXIMATION = 100; - static constexpr long MIN_UNIQUE_VALUES_TO_NUMDOCS_RATIO_BEFORE_APPROXIMATION = 20; - static constexpr long MIN_APPROXHITS_TO_NUMDOCS_RATIO_BEFORE_APPROXIMATION = 10; - const IEnumStoreDictionary& _dictionary; const ISearchContext& _baseSearchCtx; const BitVector* _bv; // bitvector if _useBitVector has been set @@ -51,11 +47,9 @@ protected: uint32_t _dictSize; EntryRef _pidx; EntryRef _frozenRoot; // Posting list in tree form - float _FSTC; // Filtering Search Time Constant - float _PLSTC; // Posting List Search Time Constant bool _hasWeight; bool _useBitVector; - mutable std::optional<size_t> _counted_hits; // Snapshot of size of posting lists in range + mutable std::optional<size_t> _estimated_hits; // Snapshot of size of posting lists in range PostingListSearchContext(const IEnumStoreDictionary& dictionary, bool has_btree_dictionary, uint32_t docIdLimit, uint64_t numValues, bool hasWeight, bool useBitVector, const ISearchContext &baseSearchCtx); @@ -65,48 +59,18 @@ protected: void lookupTerm(const vespalib::datastore::EntryComparator &comp); void lookupRange(const vespalib::datastore::EntryComparator &low, const vespalib::datastore::EntryComparator &high); void lookupSingle(); + size_t estimated_hits_in_range() const; virtual bool use_dictionary_entry(DictionaryConstIterator& it) const { (void) it; return true; } + virtual bool use_posting_lists_when_non_strict(const queryeval::ExecuteInfo& info) const = 0; - float calculateFilteringCost() const { - // filtering search time (ms) ~ FSTC * numValues; (FSTC = - // Filtering Search Time Constant) - return _FSTC * _numValues; - } - - float calculatePostingListCost(uint32_t approxNumHits) const { - // search time (ms) ~ PLSTC * numHits * log(numHits); (PLSTC = - // Posting List Search Time Constant) - return _PLSTC * approxNumHits; - } - - uint32_t calculateApproxNumHits() const { - float docsPerUniqueValue = static_cast<float>(_docIdLimit) / - static_cast<float>(_dictSize); - return static_cast<uint32_t>(docsPerUniqueValue * _uniqueValues); - } - - virtual bool fallbackToFiltering() const { - if (_uniqueValues >= 2 && !_dictionary.get_has_btree_dictionary()) { - return true; // force filtering for range search - } - uint32_t numHits = calculateApproxNumHits(); - // numHits > 1000: make sure that posting lists are unit tested. - return (numHits > 1000) && - (calculateFilteringCost() < calculatePostingListCost(numHits)); - } - virtual bool use_posting_list_when_non_strict(const queryeval::ExecuteInfo&) const { - return false; - } - virtual bool fallback_to_approx_num_hits() const { - return ((_uniqueValues > MIN_UNIQUE_VALUES_BEFORE_APPROXIMATION) && - ((_uniqueValues * MIN_UNIQUE_VALUES_TO_NUMDOCS_RATIO_BEFORE_APPROXIMATION > static_cast<int>(_docIdLimit)) || - (calculateApproxNumHits() * MIN_APPROXHITS_TO_NUMDOCS_RATIO_BEFORE_APPROXIMATION > _docIdLimit) || - (_uniqueValues > MIN_UNIQUE_VALUES_BEFORE_APPROXIMATION*10))); - } - virtual size_t countHits() const = 0; + /** + * Calculates the estimated number of hits when _uniqueValues >= 2, + * by looking at the posting lists in the range [lower, upper>. + */ + virtual size_t calc_estimated_hits_in_range() const = 0; virtual void fillArray() = 0; virtual void fillBitVector() = 0; }; @@ -136,7 +100,6 @@ protected: ~PostingListSearchContextT() override; void lookupSingle(); - size_t countHits() const override; void fillArray() override; void fillBitVector() override; @@ -166,7 +129,6 @@ protected: using DictionaryConstIterator = Dictionary::ConstIterator; using EntryRef = vespalib::datastore::EntryRef; using PostingList = typename Parent::PostingList; - using Parent::_counted_hits; using Parent::_docIdLimit; using Parent::_lowerDictItr; using Parent::_merger; @@ -184,8 +146,7 @@ protected: bool useBitVector, const ISearchContext &baseSearchCtx); ~PostingListFoldedSearchContextT() override; - bool fallback_to_approx_num_hits() const override; - size_t countHits() const override; + size_t calc_estimated_hits_in_range() const override; template <bool fill_array> void fill_array_or_bitvector_helper(EntryRef pidx); template <bool fill_array> @@ -217,13 +178,13 @@ private: using Parent = PostingSearchContext<BaseSC, PostingListFoldedSearchContextT<DataT>, AttrT>; using RegexpUtil = vespalib::RegexpUtil; using Parent::_enumStore; - // Note: steps iterator one ore more steps when not using dictionary entry + // Note: Steps iterator one or more steps when not using dictionary entry bool use_dictionary_entry(PostingListSearchContext::DictionaryConstIterator& it) const override; // Note: Uses copy of dictionary iterator to avoid stepping original. bool use_single_dictionary_entry(PostingListSearchContext::DictionaryConstIterator it) const { return use_dictionary_entry(it); } - bool use_posting_list_when_non_strict(const queryeval::ExecuteInfo&) const override; + bool use_posting_lists_when_non_strict(const queryeval::ExecuteInfo& info) const override; public: StringPostingSearchContext(BaseSC&& base_sc, bool useBitVector, const AttrT &toBeSearched); }; @@ -245,11 +206,6 @@ private: void getIterators(bool shouldApplyRangeLimit); bool valid() const override { return this->isValid(); } - bool fallbackToFiltering() const override { - return (this->getRangeLimit() != 0) - ? (this->_uniqueValues >= 2 && !this->_dictionary.get_has_btree_dictionary()) - : Parent::fallbackToFiltering(); - } unsigned int approximateHits() const override { const unsigned int estimate = PostingListSearchContextT<DataT>::approximateHits(); const unsigned int limit = std::abs(this->getRangeLimit()); @@ -269,6 +225,9 @@ private: } } + bool use_posting_lists_when_non_strict(const queryeval::ExecuteInfo& info) const override; + size_t calc_estimated_hits_in_range() const override; + public: NumericPostingSearchContext(BaseSC&& base_sc, const Params & params, const AttrT &toBeSearched); const Params ¶ms() const { return _params; } @@ -301,14 +260,6 @@ StringPostingSearchContext<BaseSC, AttrT, DataT>:: StringPostingSearchContext(BaseSC&& base_sc, bool useBitVector, const AttrT &toBeSearched) : Parent(std::move(base_sc), useBitVector, toBeSearched) { - // after benchmarking prefix search performance on single, array, and weighted set fast-aggregate string attributes - // with 1M values the following constant has been derived: - this->_FSTC = 0.000028; - - // after benchmarking prefix search performance on single, array, and weighted set fast-search string attributes - // with 1M values the following constant has been derived: - this->_PLSTC = 0.000000; - if (this->valid()) { if (this->isPrefix()) { auto comp = _enumStore.make_folded_comparator_prefix(this->queryTerm()->getTerm()); @@ -363,11 +314,11 @@ StringPostingSearchContext<BaseSC, AttrT, DataT>::use_dictionary_entry(PostingLi template <typename BaseSC, typename AttrT, typename DataT> bool -StringPostingSearchContext<BaseSC, AttrT, DataT>::use_posting_list_when_non_strict(const queryeval::ExecuteInfo& info) const +StringPostingSearchContext<BaseSC, AttrT, DataT>::use_posting_lists_when_non_strict(const queryeval::ExecuteInfo& info) const { if (this->isFuzzy()) { uint32_t exp_doc_hits = this->_docIdLimit * info.hitRate(); - constexpr uint32_t fuzzy_use_posting_list_doc_limit = 10000; + constexpr uint32_t fuzzy_use_posting_lists_doc_limit = 10000; /** * The above constant was derived after a query latency experiment with fuzzy matching * on 2M documents with a dictionary size of 292070. @@ -390,7 +341,7 @@ StringPostingSearchContext<BaseSC, AttrT, DataT>::use_posting_list_when_non_stri * is already performed at this point. * The only work remaining if returning true is merging the posting lists. */ - if (exp_doc_hits > fuzzy_use_posting_list_doc_limit) { + if (exp_doc_hits > fuzzy_use_posting_lists_doc_limit) { return true; } } @@ -403,11 +354,6 @@ NumericPostingSearchContext(BaseSC&& base_sc, const Params & params_in, const At : Parent(std::move(base_sc), params_in.useBitVector(), toBeSearched), _params(params_in) { - // after simplyfying the formula and simple benchmarking and thumbs in the air - // a ratio of 8 between numvalues and estimated number of hits has been found. - this->_FSTC = 1; - - this->_PLSTC = 8; if (valid()) { if (_low == _high) { auto comp = _enumStore.make_comparator(_low); @@ -455,7 +401,68 @@ getIterators(bool shouldApplyRangeLimit) } } +template <typename BaseSC, typename AttrT, typename DataT> +bool +NumericPostingSearchContext<BaseSC, AttrT, DataT>::use_posting_lists_when_non_strict(const queryeval::ExecuteInfo& info) const +{ + // The following initial constants are derived after running parts of + // the range search performance test with 10M documents on an Apple M1 Pro with 32 GB memory. + // This code was compiled with two different behaviors: + // 1) 'filter matching' (never use posting lists). + // 2) 'posting list matching' (always use posting lists). + // https://github.com/vespa-engine/system-test/tree/master/tests/performance/range_search + // + // The following test case was used to establish the baseline cost of producing different number of hits as cheap as possible: + // range_hits_ratio=[1, 10, 50, 100, 200, 500], values_in_range=1, fast_search=true, filter_hits_ratio=0. + // The 6 range queries end up using a single posting list that produces the following number of hits: [10k, 100k, 500k, 1M, 2M, 5M] + // Avg query latency (ms) results: [5.43, 8.56, 11.68, 14.68, 22.77, 42.88] + // + // Then the following test case was executed for both 1) 'filter matching' and 2) 'posting list matching': + // range_hits_ratio=[1, 10, 50, 100, 200, 500], values_in_range=100, fast_search=true, filter_hits_ratio=0. + // Avg query latency (ms) results: + // 1) 'filter matching': [47.52, 51.06, 59.68, 79.3, 118.7, 145.26] + // 2) 'posting list matching': [4.79, 11.6, 13.54, 20.24, 32.65, 67.28] + // + // For 1) 'filter matching' we use the result from range_hits_ratio=1 (10k hits) compared to the baseline + // to calculate the cost per document (in ns) to do filter matching: 1M*(47.52-5.43)/10M = 4.2 + // + // For 2) 'posting list matching' we use the results from range_hits_ratio=[50, 100, 200, 500] compared to the baseline + // to calculate the average cost per hit (in ns) when merging the 100 posting lists: + // 1M*(13.54-11.68)/500k = 3.7 + // 1M*(20.24-14.68)/1M = 5.6 + // 1M*(32.65-22.77)/2M = 4.9 + // 1M*(67.28-42.88)/5M = 4.9 + // + // The average is 4.8. + + constexpr float filtering_match_constant = 4.2; + constexpr float posting_list_merge_constant = 4.8; + + uint32_t exp_doc_hits = this->_docIdLimit * info.hitRate(); + float avg_values_per_document = static_cast<float>(this->_numValues) / static_cast<float>(this->_docIdLimit); + float filtering_cost = exp_doc_hits * avg_values_per_document * filtering_match_constant; + float posting_list_cost = this->estimated_hits_in_range() * posting_list_merge_constant; + return posting_list_cost < filtering_cost; +} +template <typename BaseSC, typename AttrT, typename DataT> +size_t +NumericPostingSearchContext<BaseSC, AttrT, DataT>::calc_estimated_hits_in_range() const +{ + size_t exact_sum = 0; + size_t estimated_sum = 0; + constexpr uint32_t max_posting_lists_to_count = 1000; + auto it = this->_lowerDictItr; + for (uint32_t count = 0; (it != this->_upperDictItr) && (count < max_posting_lists_to_count); ++it, ++count) { + exact_sum += this->_postingList.frozenSize(it.getData().load_acquire()); + } + if (it != this->_upperDictItr) { + uint32_t remaining_posting_lists = this->_upperDictItr - it; + float hits_per_posting_list = static_cast<float>(exact_sum) / static_cast<float>(max_posting_lists_to_count); + estimated_sum = remaining_posting_lists * hits_per_posting_list; + } + return exact_sum + estimated_sum; +} extern template class PostingListSearchContextT<vespalib::btree::BTreeNoLeafData>; extern template class PostingListSearchContextT<int32_t>; diff --git a/searchlib/src/vespa/searchlib/attribute/postinglistsearchcontext.hpp b/searchlib/src/vespa/searchlib/attribute/postinglistsearchcontext.hpp index 7c7b9117a30..964101ed3a6 100644 --- a/searchlib/src/vespa/searchlib/attribute/postinglistsearchcontext.hpp +++ b/searchlib/src/vespa/searchlib/attribute/postinglistsearchcontext.hpp @@ -58,21 +58,6 @@ PostingListSearchContextT<DataT>::lookupSingle() } template <typename DataT> -size_t -PostingListSearchContextT<DataT>::countHits() const -{ - if (_counted_hits.has_value()) { - return _counted_hits.value(); - } - size_t sum(0); - for (auto it(_lowerDictItr); it != _upperDictItr; ++it) { - sum += _postingList.frozenSize(it.getData().load_acquire()); - } - _counted_hits = sum; - return sum; -} - -template <typename DataT> void PostingListSearchContextT<DataT>::fillArray() { @@ -97,9 +82,9 @@ template <typename DataT> void PostingListSearchContextT<DataT>::fetchPostings(const queryeval::ExecuteInfo & execInfo) { - if (!_merger.merge_done() && _uniqueValues >= 2u) { - if ((execInfo.isStrict() || use_posting_list_when_non_strict(execInfo)) && !fallbackToFiltering()) { - size_t sum(countHits()); + if (!_merger.merge_done() && _uniqueValues >= 2u && this->_dictionary.get_has_btree_dictionary()) { + if (execInfo.isStrict() || use_posting_lists_when_non_strict(execInfo)) { + size_t sum = estimated_hits_in_range(); if (sum < _docIdLimit / 64) { _merger.reserveArray(_uniqueValues, sum); fillArray(); @@ -221,19 +206,14 @@ PostingListSearchContextT<DataT>::approximateHits() const if (_uniqueValues == 0u) { } else if (_uniqueValues == 1u) { numHits = singleHits(); + } else if (_dictionary.get_has_btree_dictionary()) { + numHits = estimated_hits_in_range(); } else { - if (this->fallbackToFiltering()) { - numHits = _docIdLimit; - } else if (this->fallback_to_approx_num_hits()) { - numHits = this->calculateApproxNumHits(); - } else { - numHits = countHits(); - } + numHits = _docIdLimit; } return std::min(numHits, size_t(std::numeric_limits<uint32_t>::max())); } - template <typename DataT> void PostingListSearchContextT<DataT>::applyRangeLimit(int rangeLimit) @@ -273,20 +253,10 @@ template <typename DataT> PostingListFoldedSearchContextT<DataT>::~PostingListFoldedSearchContextT() = default; template <typename DataT> -bool -PostingListFoldedSearchContextT<DataT>::fallback_to_approx_num_hits() const -{ - return false; -} - -template <typename DataT> size_t -PostingListFoldedSearchContextT<DataT>::countHits() const +PostingListFoldedSearchContextT<DataT>::calc_estimated_hits_in_range() const { - if (_counted_hits.has_value()) { - return _counted_hits.value(); - } - size_t sum(0); + size_t sum = 0; bool overflow = false; for (auto it(_lowerDictItr); it != _upperDictItr;) { if (use_dictionary_entry(it)) { @@ -305,7 +275,6 @@ PostingListFoldedSearchContextT<DataT>::countHits() const ++it; } } - _counted_hits = sum; return sum; } diff --git a/searchlib/src/vespa/searchlib/bitcompression/README.md b/searchlib/src/vespa/searchlib/bitcompression/README.md new file mode 100644 index 00000000000..e387ec099fb --- /dev/null +++ b/searchlib/src/vespa/searchlib/bitcompression/README.md @@ -0,0 +1,56 @@ +<!-- Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> + +## About the disk dictionary format + +The designs of the disk index dictionary formats were incremental, due +to changing requirements over the years. + +### 1st generation + +Patricia tree in memory. + +### 2nd generation, 1998-09-04 + +Problem: Dictionary did not fit in memory (machine had 512 MB ram) +when indexing 5 million web documents on a single machine. + +Changed format to variable length records on disk, with a sparse +version of the dictionary in memory (each 256th word) to limit disk +access for binary search. + +### 3rd generation, 2000-03-09 + +Problem: Too many disk read operations and too many bytes read from disk +(limited PCI bandwidth). + +Changed format to a "paged" dictionary where a dictionary lookup would +use 1 disk read, reading 4 kiB of data. Data was not compressed. Could +not memory map whole dictionary. The sparse files were read into +memory and used to determine the page to use for further lookup. +Binary search within the pages read from disk. + +### 4th generation, 2002-08-16 + +Problem: Dictionary used too much disk space. + +Changed format to compressed format. Decompression could not contain +much state, thus delta values were compressed using exp golomb coding. + +Two levels of skip lists within each page, where skip list on a level +contained enough information to skip on all levels below within the +same page. + +Start of word was replaced by a byte telling how many bytes is +ommitted due to the prefix being common with previous words (word +before in dictionary and word before in the lookup order). + +### 5th generation, 2010-08-21 + +Payload ("value") changed when skip information was added for large +posting lists. Added overflow handling for long words / huge payloads. +Added another level of pages ("sparse pages") to improve compression. + +### 6th generation, 2015-05-12 + +Started using a separate dictionary for each index field instead of a +shared dictionary across all index fields. Minor changes. diff --git a/searchlib/src/vespa/searchlib/docstore/filechunk.cpp b/searchlib/src/vespa/searchlib/docstore/filechunk.cpp index 66f6cd38bce..c57650bb16f 100644 --- a/searchlib/src/vespa/searchlib/docstore/filechunk.cpp +++ b/searchlib/src/vespa/searchlib/docstore/filechunk.cpp @@ -208,7 +208,7 @@ FileChunk::handleChunk(const unique_lock &guard, ISetLid &ds, uint32_t docIdLimi } else { remove(lidMeta.getLid(), lidMeta.size()); } - _addedBytes += adjustSize(lidMeta.size()); + _addedBytes.store(getAddedBytes() + adjustSize(lidMeta.size()), std::memory_order_relaxed); } uint64_t serialNum = chunkMeta.getLastSerial(); addNumBuckets(bucketMap.getNumBuckets()); @@ -250,8 +250,8 @@ void FileChunk::remove(uint32_t lid, uint32_t size) { (void) lid; - _erasedCount++; - _erasedBytes += adjustSize(size); + _erasedCount.store(getErasedCount() + 1, std::memory_order_relaxed); + _erasedBytes.store(getErasedBytes() + adjustSize(size), std::memory_order_relaxed); } uint64_t @@ -451,7 +451,7 @@ FileChunk::verify(bool reportOnly) const (void) reportOnly; LOG(info, "Verifying file '%s' with fileid '%u'. erased-count='%zu' and erased-bytes='%zu'. diskFootprint='%zu'", - _name.c_str(), _fileId.getId(), _erasedCount, _erasedBytes, _diskFootprint.load(std::memory_order_relaxed)); + _name.c_str(), _fileId.getId(), getErasedCount(), getErasedBytes(), _diskFootprint.load(std::memory_order_relaxed)); uint64_t lastSerial(0); size_t chunkId(0); bool errorInPrev(false); diff --git a/searchlib/src/vespa/searchlib/docstore/filechunk.h b/searchlib/src/vespa/searchlib/docstore/filechunk.h index 4d6d7c1d332..1668d030141 100644 --- a/searchlib/src/vespa/searchlib/docstore/filechunk.h +++ b/searchlib/src/vespa/searchlib/docstore/filechunk.h @@ -120,9 +120,10 @@ public: virtual size_t getDiskHeaderFootprint() const { return _dataHeaderLen + _idxHeaderLen; } size_t getDiskBloat() const { - return (_addedBytes == 0) + size_t addedBytes = getAddedBytes(); + return (addedBytes == 0) ? getDiskFootprint() - : size_t(getDiskFootprint() * double(_erasedBytes)/_addedBytes); + : size_t(getDiskFootprint() * double(getErasedBytes())/addedBytes); } /** * Get a metric for unorder of data in the file relative to when @@ -154,9 +155,9 @@ public: FileId getFileId() const { return _fileId; } NameId getNameId() const { return _nameId; } uint32_t getNumLids() const { return _numLids; } - size_t getBloatCount() const { return _erasedCount; } - size_t getAddedBytes() const { return _addedBytes; } - size_t getErasedBytes() const { return _erasedBytes; } + size_t getErasedCount() const { return _erasedCount.load(std::memory_order_relaxed); } + size_t getAddedBytes() const { return _addedBytes.load(std::memory_order_relaxed); } + size_t getErasedBytes() const { return _erasedBytes.load(std::memory_order_relaxed); } uint64_t getLastPersistedSerialNum() const; uint32_t getDocIdLimit() const { return _docIdLimit; } virtual vespalib::system_time getModificationTime() const; @@ -213,8 +214,8 @@ private: const FileId _fileId; const NameId _nameId; const vespalib::string _name; - size_t _erasedCount; - size_t _erasedBytes; + std::atomic<size_t> _erasedCount; + std::atomic<size_t> _erasedBytes; std::atomic<size_t> _diskFootprint; size_t _sumNumBuckets; size_t _numChunksWithBuckets; @@ -247,17 +248,17 @@ protected: static void writeDocIdLimit(vespalib::GenericHeader &header, uint32_t docIdLimit); using ChunkInfoVector = std::vector<ChunkInfo, vespalib::allocator_large<ChunkInfo>>; - const IBucketizer * _bucketizer; - size_t _addedBytes; - TuneFileSummary _tune; - vespalib::string _dataFileName; - vespalib::string _idxFileName; - ChunkInfoVector _chunkInfo; - std::atomic<uint64_t> _lastPersistedSerialNum; - uint32_t _dataHeaderLen; - uint32_t _idxHeaderLen; - uint32_t _numLids; - uint32_t _docIdLimit; // Limit when the file was created. Stored in idx file header. + const IBucketizer * _bucketizer; + std::atomic<size_t> _addedBytes; + TuneFileSummary _tune; + vespalib::string _dataFileName; + vespalib::string _idxFileName; + ChunkInfoVector _chunkInfo; + std::atomic<uint64_t> _lastPersistedSerialNum; + uint32_t _dataHeaderLen; + uint32_t _idxHeaderLen; + uint32_t _numLids; + uint32_t _docIdLimit; // Limit when the file was created. Stored in idx file header. vespalib::system_time _modificationTime; }; diff --git a/searchlib/src/vespa/searchlib/docstore/logdatastore.cpp b/searchlib/src/vespa/searchlib/docstore/logdatastore.cpp index 96d30479f4c..45028e124f1 100644 --- a/searchlib/src/vespa/searchlib/docstore/logdatastore.cpp +++ b/searchlib/src/vespa/searchlib/docstore/logdatastore.cpp @@ -453,9 +453,13 @@ void LogDataStore::compactFile(FileId fileId) std::unique_ptr<IWriteData> compacter; FileId destinationFileId = FileId::active(); if (_bucketizer) { - size_t disk_footprint = fc->getDiskFootprint(); - size_t disk_bloat = fc->getDiskBloat(); - size_t compacted_size = (disk_footprint <= disk_bloat) ? 0u : (disk_footprint - disk_bloat); + size_t compacted_size; + { + MonitorGuard guard(_updateLock); + size_t disk_footprint = fc->getDiskFootprint(); + size_t disk_bloat = fc->getDiskBloat(); + compacted_size = (disk_footprint <= disk_bloat) ? 0u : (disk_footprint - disk_bloat); + } if ( ! shouldCompactToActiveFile(compacted_size)) { MonitorGuard guard(_updateLock); destinationFileId = allocateFileId(guard); diff --git a/searchlib/src/vespa/searchlib/docstore/writeablefilechunk.cpp b/searchlib/src/vespa/searchlib/docstore/writeablefilechunk.cpp index 19816617448..297b8f66099 100644 --- a/searchlib/src/vespa/searchlib/docstore/writeablefilechunk.cpp +++ b/searchlib/src/vespa/searchlib/docstore/writeablefilechunk.cpp @@ -730,7 +730,7 @@ WriteableFileChunk::append(uint64_t serialNum, uint32_t lid, vespalib::ConstBuff } assert(serialNum >= _serialNum); _serialNum = serialNum; - _addedBytes += adjustSize(data.size()); + _addedBytes.store(getAddedBytes() + adjustSize(data.size()), std::memory_order_relaxed); _numLids++; size_t oldSz(_active->size()); LidMeta lm = _active->append(lid, data); diff --git a/searchsummary/CMakeLists.txt b/searchsummary/CMakeLists.txt index e82ffa8d2b8..f771a8e4494 100644 --- a/searchsummary/CMakeLists.txt +++ b/searchsummary/CMakeLists.txt @@ -20,6 +20,7 @@ vespa_define_module( src/tests/docsummary/attribute_combiner src/tests/docsummary/attributedfw src/tests/docsummary/document_id_dfw + src/tests/docsummary/tokens_converter src/tests/docsummary/matched_elements_filter src/tests/docsummary/query_term_filter_factory src/tests/docsummary/result_class diff --git a/searchsummary/src/tests/docsummary/slime_filler/slime_filler_test.cpp b/searchsummary/src/tests/docsummary/slime_filler/slime_filler_test.cpp index 9468cc30a3d..c20f9570ef8 100644 --- a/searchsummary/src/tests/docsummary/slime_filler/slime_filler_test.cpp +++ b/searchsummary/src/tests/docsummary/slime_filler/slime_filler_test.cpp @@ -68,6 +68,7 @@ using search::docsummary::IStringFieldConverter; using search::docsummary::ResultConfig; using search::docsummary::SlimeFiller; using search::docsummary::SlimeFillerFilter; +using vespalib::Memory; using vespalib::SimpleBuffer; using vespalib::Slime; using vespalib::eval::SimpleValue; @@ -146,17 +147,27 @@ get_document_types_config() class MockStringFieldConverter : public IStringFieldConverter { std::vector<vespalib::string> _result; + bool _render_wset_as_array; + bool _insert; public: - MockStringFieldConverter() + MockStringFieldConverter(bool render_wset_as_array, bool insert) : IStringFieldConverter(), - _result() + _result(), + _render_wset_as_array(render_wset_as_array), + _insert(insert) { } ~MockStringFieldConverter() override = default; - void convert(const document::StringFieldValue& input, vespalib::slime::Inserter&) override { + void convert(const document::StringFieldValue& input, vespalib::slime::Inserter& inserter) override { _result.emplace_back(input.getValueRef()); + if (_insert) { + inserter.insertString(Memory(input.getValueRef())); + } } const std::vector<vespalib::string>& get_result() const noexcept { return _result; } + bool render_weighted_set_as_array() const override { + return _render_wset_as_array; + } }; } @@ -188,6 +199,7 @@ protected: void expect_insert_summary_field_with_filter(const vespalib::string& exp, const FieldValue& fv, const std::vector<uint32_t>& matching_elems); void expect_insert_summary_field_with_field_filter(const vespalib::string& exp, const FieldValue& fv, const SlimeFillerFilter* filter); void expect_insert_juniper_field(const std::vector<vespalib::string>& exp, const vespalib::string& exp_slime, const FieldValue& fv); + void expect_insert_summary_field_with_converter(const std::vector<vespalib::string>& exp, const vespalib::string& exp_slime, const FieldValue& fv, MockStringFieldConverter& converter); }; SlimeFillerTest::SlimeFillerTest() @@ -317,7 +329,7 @@ SlimeFillerTest::expect_insert_callback(const std::vector<vespalib::string>& exp { Slime slime; SlimeInserter inserter(slime); - MockStringFieldConverter converter; + MockStringFieldConverter converter(false, false); SlimeFiller filler(inserter, &converter, SlimeFillerFilter::all()); fv.accept(filler); auto act_null = slime_to_string(slime); @@ -351,7 +363,7 @@ SlimeFillerTest::expect_insert_summary_field_with_field_filter(const vespalib::s { Slime slime; SlimeInserter inserter(slime); - SlimeFiller::insert_summary_field_with_field_filter(fv, inserter, filter); + SlimeFiller::insert_summary_field_with_field_filter(fv, inserter, nullptr, filter); auto act = slime_to_string(slime); EXPECT_EQ(exp, act); } @@ -361,7 +373,7 @@ SlimeFillerTest::expect_insert_juniper_field(const std::vector<vespalib::string> { Slime slime; SlimeInserter inserter(slime); - MockStringFieldConverter converter; + MockStringFieldConverter converter(false, false); SlimeFiller::insert_juniper_field(fv, inserter, converter); auto act_slime = slime_to_string(slime); EXPECT_EQ(exp_slime, act_slime); @@ -369,6 +381,18 @@ SlimeFillerTest::expect_insert_juniper_field(const std::vector<vespalib::string> EXPECT_EQ(exp, act); } +void +SlimeFillerTest::expect_insert_summary_field_with_converter(const std::vector<vespalib::string>& exp, const vespalib::string& exp_slime, const FieldValue& fv, MockStringFieldConverter& converter) +{ + Slime slime; + SlimeInserter inserter(slime); + SlimeFiller::insert_summary_field(fv, inserter, &converter); + auto act_slime = slime_to_string(slime); + EXPECT_EQ(exp_slime, act_slime); + auto act = converter.get_result(); + EXPECT_EQ(exp, act); +} + TEST_F(SlimeFillerTest, insert_primitive_values) { { @@ -625,4 +649,16 @@ TEST_F(SlimeFillerTest, insert_juniper_field) expect_insert_juniper_field({}, "null", make_empty_array()); } +TEST_F(SlimeFillerTest, string_field_is_not_converted_for_weighted_set_rendering) +{ + MockStringFieldConverter cvt_as_wset(false, true); + expect_insert_summary_field_with_converter({}, R"([{"item":"foo","weight":2},{"item":"bar","weight":4},{"item":"baz","weight":6}])", make_weighted_set(), cvt_as_wset); +} + +TEST_F(SlimeFillerTest, weighted_set_can_be_rendered_as_array) +{ + MockStringFieldConverter cvt_as_array(true, true); + expect_insert_summary_field_with_converter({"foo","bar","baz"}, R"(["foo","bar","baz"])", make_weighted_set(), cvt_as_array); +} + GTEST_MAIN_RUN_ALL_TESTS() diff --git a/searchsummary/src/tests/docsummary/tokens_converter/CMakeLists.txt b/searchsummary/src/tests/docsummary/tokens_converter/CMakeLists.txt new file mode 100644 index 00000000000..68885a74b1b --- /dev/null +++ b/searchsummary/src/tests/docsummary/tokens_converter/CMakeLists.txt @@ -0,0 +1,10 @@ +# Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(searchsummary_tokens_converter_test_app TEST + SOURCES + tokens_converter_test.cpp + DEPENDS + searchsummary + GTest::gtest +) + +vespa_add_test(NAME searchsummary_tokens_converter_test_app COMMAND searchsummary_tokens_converter_test_app) diff --git a/searchsummary/src/tests/docsummary/tokens_converter/tokens_converter_test.cpp b/searchsummary/src/tests/docsummary/tokens_converter/tokens_converter_test.cpp new file mode 100644 index 00000000000..493cbe0ecba --- /dev/null +++ b/searchsummary/src/tests/docsummary/tokens_converter/tokens_converter_test.cpp @@ -0,0 +1,178 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/document/annotation/annotation.h> +#include <vespa/document/annotation/span.h> +#include <vespa/document/annotation/spanlist.h> +#include <vespa/document/annotation/spantree.h> +#include <vespa/document/datatype/annotationtype.h> +#include <vespa/document/fieldvalue/stringfieldvalue.h> +#include <vespa/document/repo/configbuilder.h> +#include <vespa/document/repo/fixedtyperepo.h> +#include <vespa/searchlib/util/linguisticsannotation.h> +#include <vespa/searchlib/util/token_extractor.h> +#include <vespa/searchsummary/docsummary/tokens_converter.h> +#include <vespa/vespalib/data/simple_buffer.h> +#include <vespa/vespalib/data/slime/json_format.h> +#include <vespa/vespalib/data/slime/slime.h> +#include <vespa/vespalib/gtest/gtest.h> + +using document::Annotation; +using document::AnnotationType; +using document::DocumentType; +using document::DocumentTypeRepo; +using document::Span; +using document::SpanList; +using document::SpanTree; +using document::StringFieldValue; +using search::docsummary::TokensConverter; +using search::linguistics::SPANTREE_NAME; +using search::linguistics::TokenExtractor; +using vespalib::SimpleBuffer; +using vespalib::Slime; +using vespalib::slime::JsonFormat; +using vespalib::slime::SlimeInserter; + +namespace { + +vespalib::string +slime_to_string(const Slime& slime) +{ + SimpleBuffer buf; + JsonFormat::encode(slime, buf, true); + return buf.get().make_string(); +} + +DocumenttypesConfig +get_document_types_config() +{ + using namespace document::config_builder; + DocumenttypesConfigBuilderHelper builder; + builder.document(42, "indexingdocument", + Struct("indexingdocument.header"), + Struct("indexingdocument.body")); + return builder.config(); +} + +} + +class TokensConverterTest : public testing::Test +{ +protected: + std::shared_ptr<const DocumentTypeRepo> _repo; + const DocumentType* _document_type; + document::FixedTypeRepo _fixed_repo; + vespalib::string _dummy_field_name; + TokenExtractor _token_extractor; + + TokensConverterTest(); + ~TokensConverterTest() override; + void set_span_tree(StringFieldValue& value, std::unique_ptr<SpanTree> tree); + StringFieldValue make_annotated_string(bool alt_tokens); + StringFieldValue make_annotated_chinese_string(); + vespalib::string make_exp_annotated_chinese_string_tokens(); + vespalib::string convert(const StringFieldValue& fv); +}; + +TokensConverterTest::TokensConverterTest() + : testing::Test(), + _repo(std::make_unique<DocumentTypeRepo>(get_document_types_config())), + _document_type(_repo->getDocumentType("indexingdocument")), + _fixed_repo(*_repo, *_document_type), + _dummy_field_name(), + _token_extractor(_dummy_field_name, 100) +{ +} + +TokensConverterTest::~TokensConverterTest() = default; + +void +TokensConverterTest::set_span_tree(StringFieldValue & value, std::unique_ptr<SpanTree> tree) +{ + StringFieldValue::SpanTrees trees; + trees.push_back(std::move(tree)); + value.setSpanTrees(trees, _fixed_repo); +} + +StringFieldValue +TokensConverterTest::make_annotated_string(bool alt_tokens) +{ + auto span_list_up = std::make_unique<SpanList>(); + auto span_list = span_list_up.get(); + auto tree = std::make_unique<SpanTree>(SPANTREE_NAME, std::move(span_list_up)); + tree->annotate(span_list->add(std::make_unique<Span>(0, 3)), *AnnotationType::TERM); + if (alt_tokens) { + tree->annotate(span_list->add(std::make_unique<Span>(4, 3)), *AnnotationType::TERM); + } + tree->annotate(span_list->add(std::make_unique<Span>(4, 3)), + Annotation(*AnnotationType::TERM, std::make_unique<StringFieldValue>("baz"))); + StringFieldValue value("foo bar"); + set_span_tree(value, std::move(tree)); + return value; +} + +StringFieldValue +TokensConverterTest::make_annotated_chinese_string() +{ + auto span_list_up = std::make_unique<SpanList>(); + auto span_list = span_list_up.get(); + auto tree = std::make_unique<SpanTree>(SPANTREE_NAME, std::move(span_list_up)); + // These chinese characters each use 3 bytes in their UTF8 encoding. + tree->annotate(span_list->add(std::make_unique<Span>(0, 15)), *AnnotationType::TERM); + tree->annotate(span_list->add(std::make_unique<Span>(15, 9)), *AnnotationType::TERM); + StringFieldValue value("我就是那个大ç°ç‹¼"); + set_span_tree(value, std::move(tree)); + return value; +} + +vespalib::string +TokensConverterTest::make_exp_annotated_chinese_string_tokens() +{ + return R"(["我就是那个","大ç°ç‹¼"])"; +} + +vespalib::string +TokensConverterTest::convert(const StringFieldValue& fv) +{ + TokensConverter converter(_token_extractor); + Slime slime; + SlimeInserter inserter(slime); + converter.convert(fv, inserter); + return slime_to_string(slime); +} + +TEST_F(TokensConverterTest, convert_empty_string) +{ + vespalib::string exp(R"([])"); + StringFieldValue plain_string(""); + EXPECT_EQ(exp, convert(plain_string)); +} + +TEST_F(TokensConverterTest, convert_plain_string) +{ + vespalib::string exp(R"(["Foo Bar Baz"])"); + StringFieldValue plain_string("Foo Bar Baz"); + EXPECT_EQ(exp, convert(plain_string)); +} + +TEST_F(TokensConverterTest, convert_annotated_string) +{ + vespalib::string exp(R"(["foo","baz"])"); + auto annotated_string = make_annotated_string(false); + EXPECT_EQ(exp, convert(annotated_string)); +} + +TEST_F(TokensConverterTest, convert_annotated_string_with_alternatives) +{ + vespalib::string exp(R"(["foo",["bar","baz"]])"); + auto annotated_string = make_annotated_string(true); + EXPECT_EQ(exp, convert(annotated_string)); +} + +TEST_F(TokensConverterTest, convert_annotated_chinese_string) +{ + auto exp = make_exp_annotated_chinese_string_tokens(); + auto annotated_chinese_string = make_annotated_chinese_string(); + EXPECT_EQ(exp, convert(annotated_chinese_string)); +} + +GTEST_MAIN_RUN_ALL_TESTS() diff --git a/searchsummary/src/vespa/searchsummary/docsummary/CMakeLists.txt b/searchsummary/src/vespa/searchsummary/docsummary/CMakeLists.txt index 32df047c27f..0287517f830 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/CMakeLists.txt +++ b/searchsummary/src/vespa/searchsummary/docsummary/CMakeLists.txt @@ -37,4 +37,6 @@ vespa_add_library(searchsummary_docsummary OBJECT struct_fields_resolver.cpp struct_map_attribute_combiner_dfw.cpp summaryfeaturesdfw.cpp + tokens_converter.cpp + tokens_dfw.cpp ) diff --git a/searchsummary/src/vespa/searchsummary/docsummary/annotation_converter.cpp b/searchsummary/src/vespa/searchsummary/docsummary/annotation_converter.cpp index bf267ab9e27..77724305220 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/annotation_converter.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/annotation_converter.cpp @@ -109,4 +109,10 @@ AnnotationConverter::convert(const StringFieldValue &input, vespalib::slime::Ins _juniper_converter.convert(_out.str(), inserter); } +bool +AnnotationConverter::render_weighted_set_as_array() const +{ + return false; +} + } diff --git a/searchsummary/src/vespa/searchsummary/docsummary/annotation_converter.h b/searchsummary/src/vespa/searchsummary/docsummary/annotation_converter.h index b6430b35f29..b082269eb7e 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/annotation_converter.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/annotation_converter.h @@ -33,6 +33,7 @@ public: AnnotationConverter(IJuniperConverter& juniper_converter); ~AnnotationConverter() override; void convert(const document::StringFieldValue &input, vespalib::slime::Inserter& inserter) override; + bool render_weighted_set_as_array() const override; }; } diff --git a/searchsummary/src/vespa/searchsummary/docsummary/docsum_field_writer_commands.cpp b/searchsummary/src/vespa/searchsummary/docsummary/docsum_field_writer_commands.cpp index 2ce809e1cbe..2ac5d1babbf 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/docsum_field_writer_commands.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/docsum_field_writer_commands.cpp @@ -17,6 +17,7 @@ const vespalib::string matched_elements_filter("matchedelementsfilter"); const vespalib::string positions("positions"); const vespalib::string rank_features("rankfeatures"); const vespalib::string summary_features("summaryfeatures"); +const vespalib::string tokens("tokens"); } diff --git a/searchsummary/src/vespa/searchsummary/docsummary/docsum_field_writer_commands.h b/searchsummary/src/vespa/searchsummary/docsummary/docsum_field_writer_commands.h index 26bc33e7e3c..d53351d8b04 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/docsum_field_writer_commands.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/docsum_field_writer_commands.h @@ -23,5 +23,6 @@ extern const vespalib::string matched_elements_filter; extern const vespalib::string positions; extern const vespalib::string rank_features; extern const vespalib::string summary_features; +extern const vespalib::string tokens; } diff --git a/searchsummary/src/vespa/searchsummary/docsummary/docsum_field_writer_factory.cpp b/searchsummary/src/vespa/searchsummary/docsummary/docsum_field_writer_factory.cpp index 9b7391dd1ab..2f7d9acdb65 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/docsum_field_writer_factory.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/docsum_field_writer_factory.cpp @@ -13,6 +13,7 @@ #include "positionsdfw.h" #include "rankfeaturesdfw.h" #include "summaryfeaturesdfw.h" +#include "tokens_dfw.h" #include <vespa/searchlib/common/matching_elements_fields.h> #include <vespa/vespalib/util/exceptions.h> @@ -84,6 +85,12 @@ DocsumFieldWriterFactory::create_docsum_field_writer(const vespalib::string& fie } else { throw_missing_source(command); } + } else if (command == command::tokens) { + if (!source.empty()) { + fieldWriter = std::make_unique<TokensDFW>(source); + } else { + throw_missing_source(command); + } } else if (command == command::abs_distance) { if (has_attribute_manager()) { fieldWriter = AbsDistanceDFW::create(source.c_str(), getEnvironment().getAttributeManager()); diff --git a/searchsummary/src/vespa/searchsummary/docsummary/docsum_store_document.cpp b/searchsummary/src/vespa/searchsummary/docsummary/docsum_store_document.cpp index c6756a6fd1a..63b71929d63 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/docsum_store_document.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/docsum_store_document.cpp @@ -37,11 +37,11 @@ DocsumStoreDocument::get_field_value(const vespalib::string& field_name) const } void -DocsumStoreDocument::insert_summary_field(const vespalib::string& field_name, vespalib::slime::Inserter& inserter) const +DocsumStoreDocument::insert_summary_field(const vespalib::string& field_name, vespalib::slime::Inserter& inserter, IStringFieldConverter* converter) const { auto field_value = get_field_value(field_name); if (field_value) { - SlimeFiller::insert_summary_field(*field_value, inserter); + SlimeFiller::insert_summary_field(*field_value, inserter, converter); } } diff --git a/searchsummary/src/vespa/searchsummary/docsummary/docsum_store_document.h b/searchsummary/src/vespa/searchsummary/docsummary/docsum_store_document.h index f6e4e7e1244..3d8ca8abcc1 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/docsum_store_document.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/docsum_store_document.h @@ -18,7 +18,7 @@ public: explicit DocsumStoreDocument(std::unique_ptr<document::Document> document); ~DocsumStoreDocument() override; DocsumStoreFieldValue get_field_value(const vespalib::string& field_name) const override; - void insert_summary_field(const vespalib::string& field_name, vespalib::slime::Inserter& inserter) const override; + void insert_summary_field(const vespalib::string& field_name, vespalib::slime::Inserter& inserter, IStringFieldConverter* converter) const override; void insert_juniper_field(const vespalib::string& field_name, vespalib::slime::Inserter& inserter, IJuniperConverter& converter) const override; void insert_document_id(vespalib::slime::Inserter& inserter) const override; }; diff --git a/searchsummary/src/vespa/searchsummary/docsummary/i_docsum_store_document.h b/searchsummary/src/vespa/searchsummary/docsummary/i_docsum_store_document.h index ffd37da4026..a229e98bb4b 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/i_docsum_store_document.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/i_docsum_store_document.h @@ -10,6 +10,7 @@ namespace vespalib::slime { struct Inserter; } namespace search::docsummary { class IJuniperConverter; +class IStringFieldConverter; /** * Interface class providing access to a document retrieved from an IDocsumStore. @@ -21,7 +22,7 @@ class IDocsumStoreDocument public: virtual ~IDocsumStoreDocument() = default; virtual DocsumStoreFieldValue get_field_value(const vespalib::string& field_name) const = 0; - virtual void insert_summary_field(const vespalib::string& field_name, vespalib::slime::Inserter& inserter) const = 0; + virtual void insert_summary_field(const vespalib::string& field_name, vespalib::slime::Inserter& inserter, IStringFieldConverter* converter = nullptr) const = 0; virtual void insert_juniper_field(const vespalib::string& field_name, vespalib::slime::Inserter& inserter, IJuniperConverter& converter) const = 0; virtual void insert_document_id(vespalib::slime::Inserter& inserter) const = 0; }; diff --git a/searchsummary/src/vespa/searchsummary/docsummary/i_string_field_converter.h b/searchsummary/src/vespa/searchsummary/docsummary/i_string_field_converter.h index 3b36455d09d..805b5cf3508 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/i_string_field_converter.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/i_string_field_converter.h @@ -17,6 +17,7 @@ class IStringFieldConverter public: virtual ~IStringFieldConverter() = default; virtual void convert(const document::StringFieldValue &input, vespalib::slime::Inserter& inserter) = 0; + virtual bool render_weighted_set_as_array() const = 0; }; } diff --git a/searchsummary/src/vespa/searchsummary/docsummary/slime_filler.cpp b/searchsummary/src/vespa/searchsummary/docsummary/slime_filler.cpp index b3678a94ca7..080129fe780 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/slime_filler.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/slime_filler.cpp @@ -285,6 +285,7 @@ SlimeFiller::visit(const WeightedSetFieldValue& value) if (empty_or_empty_after_filtering(value)) { return; } + bool render_as_array = _string_converter != nullptr && _string_converter->render_weighted_set_as_array(); Cursor& a = _inserter.insertArray(); Symbol isym = a.resolve("item"); Symbol wsym = a.resolve("weight"); @@ -305,12 +306,18 @@ SlimeFiller::visit(const WeightedSetFieldValue& value) } ++matching_elements_itr; } - Cursor& o = a.addObject(); - ObjectSymbolInserter ki(o, isym); - SlimeFiller conv(ki); - entry.first->accept(conv); - int weight = static_cast<const IntFieldValue&>(*entry.second).getValue(); - o.setLong(wsym, weight); + if (render_as_array) { + ArrayInserter ai(a); + SlimeFiller conv(ai, _string_converter, SlimeFillerFilter::all()); + entry.first->accept(conv); + } else { + Cursor& o = a.addObject(); + ObjectSymbolInserter ki(o, isym); + SlimeFiller conv(ki); + entry.first->accept(conv); + int weight = static_cast<const IntFieldValue&>(*entry.second).getValue(); + o.setLong(wsym, weight); + } ++idx; } } @@ -335,12 +342,12 @@ SlimeFiller::visit(const ReferenceFieldValue& value) } void -SlimeFiller::insert_summary_field(const FieldValue& value, vespalib::slime::Inserter& inserter) +SlimeFiller::insert_summary_field(const FieldValue& value, vespalib::slime::Inserter& inserter, IStringFieldConverter* converter) { CheckUndefinedValueVisitor check_undefined; value.accept(check_undefined); if (!check_undefined.is_undefined()) { - SlimeFiller visitor(inserter); + SlimeFiller visitor(inserter, converter, SlimeFillerFilter::all()); value.accept(visitor); } } @@ -357,12 +364,12 @@ SlimeFiller::insert_summary_field_with_filter(const FieldValue& value, vespalib: } void -SlimeFiller::insert_summary_field_with_field_filter(const document::FieldValue& value, vespalib::slime::Inserter& inserter, const SlimeFillerFilter* filter) +SlimeFiller::insert_summary_field_with_field_filter(const document::FieldValue& value, vespalib::slime::Inserter& inserter, IStringFieldConverter* converter, const SlimeFillerFilter* filter) { CheckUndefinedValueVisitor check_undefined; value.accept(check_undefined); if (!check_undefined.is_undefined()) { - SlimeFiller visitor(inserter, nullptr, (filter != nullptr) ? filter->begin() : SlimeFillerFilter::all()); + SlimeFiller visitor(inserter, converter, (filter != nullptr) ? filter->begin() : SlimeFillerFilter::all()); value.accept(visitor); } } diff --git a/searchsummary/src/vespa/searchsummary/docsummary/slime_filler.h b/searchsummary/src/vespa/searchsummary/docsummary/slime_filler.h index ff71cb7239c..547d2a13ec3 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/slime_filler.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/slime_filler.h @@ -59,13 +59,13 @@ public: SlimeFillerFilter::Iterator filter); ~SlimeFiller() override; - static void insert_summary_field(const document::FieldValue& value, vespalib::slime::Inserter& inserter); + static void insert_summary_field(const document::FieldValue& value, vespalib::slime::Inserter& inserter, IStringFieldConverter* converter = nullptr); /** * Insert the given field value, but only the elements that are contained in the matching_elems vector. */ static void insert_summary_field_with_filter(const document::FieldValue& value, vespalib::slime::Inserter& inserter, const std::vector<uint32_t>& matching_elems); - static void insert_summary_field_with_field_filter(const document::FieldValue& value, vespalib::slime::Inserter& inserter, const SlimeFillerFilter* filter); + static void insert_summary_field_with_field_filter(const document::FieldValue& value, vespalib::slime::Inserter& inserter, IStringFieldConverter* converter, const SlimeFillerFilter* filter); static void insert_juniper_field(const document::FieldValue& value, vespalib::slime::Inserter& inserter, IStringFieldConverter& converter); }; diff --git a/searchsummary/src/vespa/searchsummary/docsummary/tokens_converter.cpp b/searchsummary/src/vespa/searchsummary/docsummary/tokens_converter.cpp new file mode 100644 index 00000000000..e2849fe793e --- /dev/null +++ b/searchsummary/src/vespa/searchsummary/docsummary/tokens_converter.cpp @@ -0,0 +1,78 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "tokens_converter.h" +#include <vespa/document/fieldvalue/stringfieldvalue.h> +#include <vespa/searchlib/util/token_extractor.h> +#include <vespa/vespalib/data/slime/slime.h> + +using document::StringFieldValue; +using search::linguistics::TokenExtractor; +using vespalib::Memory; +using vespalib::slime::ArrayInserter; +using vespalib::slime::Cursor; +using vespalib::slime::Inserter; + +namespace search::docsummary { + +TokensConverter::TokensConverter(const TokenExtractor& token_extractor) + : IStringFieldConverter(), + _token_extractor(token_extractor), + _text() +{ +} + +TokensConverter::~TokensConverter() = default; + +template <typename ForwardIt> +void +TokensConverter::handle_alternative_index_terms(ForwardIt it, ForwardIt last, Inserter& inserter) +{ + Cursor& a = inserter.insertArray(); + ArrayInserter ai(a); + for (;it != last; ++it) { + handle_index_term(it->word, ai); + } +} + +void +TokensConverter::handle_index_term(vespalib::stringref word, Inserter& inserter) +{ + inserter.insertString(Memory(word)); +} + +void +TokensConverter::handle_indexing_terms(const StringFieldValue& value, vespalib::slime::Inserter& inserter) +{ + Cursor& a = inserter.insertArray(); + ArrayInserter ai(a); + using SpanTerm = TokenExtractor::SpanTerm; + std::vector<SpanTerm> terms; + auto span_trees = value.getSpanTrees(); + _token_extractor.extract(terms, span_trees, _text, nullptr); + auto it = terms.begin(); + auto ite = terms.end(); + auto itn = it; + for (; it != ite; it = itn) { + for (; itn != ite && itn->span == it->span; ++itn); + if ((itn - it) > 1) { + handle_alternative_index_terms(it, itn, ai); + } else { + handle_index_term(it->word, ai); + } + } +} + +void +TokensConverter::convert(const StringFieldValue &input, vespalib::slime::Inserter& inserter) +{ + _text = input.getValueRef(); + handle_indexing_terms(input, inserter); +} + +bool +TokensConverter::render_weighted_set_as_array() const +{ + return true; +} + +} diff --git a/searchsummary/src/vespa/searchsummary/docsummary/tokens_converter.h b/searchsummary/src/vespa/searchsummary/docsummary/tokens_converter.h new file mode 100644 index 00000000000..1798abac203 --- /dev/null +++ b/searchsummary/src/vespa/searchsummary/docsummary/tokens_converter.h @@ -0,0 +1,32 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "i_string_field_converter.h" + +namespace search::linguistics { class TokenExtractor; } + +namespace search::docsummary { + +/* + * Class converting a string field value with annotations into an array + * containing the tokens. Multiple tokens at same position are + * placed in a nested array. + */ +class TokensConverter : public IStringFieldConverter +{ + const linguistics::TokenExtractor& _token_extractor; + vespalib::stringref _text; + + template <typename ForwardIt> + void handle_alternative_index_terms(ForwardIt it, ForwardIt last, vespalib::slime::Inserter& inserter); + void handle_index_term(vespalib::stringref word, vespalib::slime::Inserter& inserter); + void handle_indexing_terms(const document::StringFieldValue& value, vespalib::slime::Inserter& inserter); +public: + TokensConverter(const linguistics::TokenExtractor& token_extractor); + ~TokensConverter() override; + void convert(const document::StringFieldValue &input, vespalib::slime::Inserter& inserter) override; + bool render_weighted_set_as_array() const override; +}; + +} diff --git a/searchsummary/src/vespa/searchsummary/docsummary/tokens_dfw.cpp b/searchsummary/src/vespa/searchsummary/docsummary/tokens_dfw.cpp new file mode 100644 index 00000000000..0741e5cc352 --- /dev/null +++ b/searchsummary/src/vespa/searchsummary/docsummary/tokens_dfw.cpp @@ -0,0 +1,36 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "tokens_dfw.h" +#include "i_docsum_store_document.h" +#include "tokens_converter.h" +#include <vespa/searchlib/memoryindex/field_inverter.h> + +using search::memoryindex::FieldInverter; + +namespace search::docsummary { + +TokensDFW::TokensDFW(const vespalib::string& input_field_name) + : DocsumFieldWriter(), + _input_field_name(input_field_name), + _token_extractor(_input_field_name, FieldInverter::max_word_len) +{ +} + +TokensDFW::~TokensDFW() = default; + +bool +TokensDFW::isGenerated() const +{ + return false; +} + +void +TokensDFW::insertField(uint32_t, const IDocsumStoreDocument* doc, GetDocsumsState&, vespalib::slime::Inserter& target) const +{ + if (doc != nullptr) { + TokensConverter converter(_token_extractor); + doc->insert_summary_field(_input_field_name, target, &converter); + } +} + +} diff --git a/searchsummary/src/vespa/searchsummary/docsummary/tokens_dfw.h b/searchsummary/src/vespa/searchsummary/docsummary/tokens_dfw.h new file mode 100644 index 00000000000..e9f91ab683a --- /dev/null +++ b/searchsummary/src/vespa/searchsummary/docsummary/tokens_dfw.h @@ -0,0 +1,28 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "docsum_field_writer.h" +#include <vespa/searchlib/util/token_extractor.h> +#include <memory> + +namespace search::docsummary { + +/* + * Class for writing annotated string field values from document as + * arrays containing the tokens. + */ +class TokensDFW : public DocsumFieldWriter +{ +private: + vespalib::string _input_field_name; + linguistics::TokenExtractor _token_extractor; + +public: + explicit TokensDFW(const vespalib::string& input_field_name); + ~TokensDFW() override; + bool isGenerated() const override; + void insertField(uint32_t docid, const IDocsumStoreDocument* doc, GetDocsumsState& state, vespalib::slime::Inserter& target) const override; +}; + +} diff --git a/security-utils/src/main/java/com/yahoo/security/X509CertificateUtils.java b/security-utils/src/main/java/com/yahoo/security/X509CertificateUtils.java index 9bcc6e7b8c6..171a8e890d0 100644 --- a/security-utils/src/main/java/com/yahoo/security/X509CertificateUtils.java +++ b/security-utils/src/main/java/com/yahoo/security/X509CertificateUtils.java @@ -4,6 +4,7 @@ package com.yahoo.security; import org.bouncycastle.asn1.ASN1Encodable; import org.bouncycastle.asn1.ASN1OctetString; import org.bouncycastle.asn1.ASN1Primitive; +import org.bouncycastle.asn1.pkcs.PrivateKeyInfo; import org.bouncycastle.asn1.x509.GeneralNames; import org.bouncycastle.cert.X509CertificateHolder; import org.bouncycastle.cert.jcajce.JcaX509CertificateConverter; @@ -73,15 +74,18 @@ public class X509CertificateUtils { } private static X509Certificate toX509Certificate(Object pemObject) throws CertificateException { - if (pemObject instanceof X509Certificate) { - return (X509Certificate) pemObject; + if (pemObject instanceof X509Certificate certificate) { + return certificate; } - if (pemObject instanceof X509CertificateHolder) { + if (pemObject instanceof X509CertificateHolder certificateHolder) { return new JcaX509CertificateConverter() .setProvider(BouncyCastleProviderHolder.getInstance()) - .getCertificate((X509CertificateHolder) pemObject); + .getCertificate(certificateHolder); } - throw new IllegalArgumentException("Invalid type of PEM object: " + pemObject); + if (pemObject instanceof PrivateKeyInfo) { + throw new IllegalArgumentException("Expected X509 certificate, but got private key"); + } + throw new IllegalArgumentException("Invalid type of PEM object, got " + pemObject.getClass().getName()); } public static String toPem(X509Certificate certificate) { diff --git a/standalone-container/src/main/java/com/yahoo/container/standalone/CloudConfigInstallVariables.java b/standalone-container/src/main/java/com/yahoo/container/standalone/CloudConfigInstallVariables.java index 3bdba2e6bcd..535d0ae7d78 100644 --- a/standalone-container/src/main/java/com/yahoo/container/standalone/CloudConfigInstallVariables.java +++ b/standalone-container/src/main/java/com/yahoo/container/standalone/CloudConfigInstallVariables.java @@ -46,7 +46,9 @@ public class CloudConfigInstallVariables implements CloudConfigOptions { @Override public Optional<Long> zookeeperBarrierTimeout() { - return getInstallVariable("zookeeper_barrier_timeout", Long::parseLong); + return Optional.ofNullable(System.getenv("VESPA_CONFIGSERVER_ZOOKEEPER_BARRIER_TIMEOUT")) + .map(Long::parseLong) + .or(() -> getInstallVariable("zookeeper_barrier_timeout", Long::parseLong)); } @Override diff --git a/storage/src/tests/bucketdb/bucketmanagertest.cpp b/storage/src/tests/bucketdb/bucketmanagertest.cpp index f46ff867fcc..91e901c7254 100644 --- a/storage/src/tests/bucketdb/bucketmanagertest.cpp +++ b/storage/src/tests/bucketdb/bucketmanagertest.cpp @@ -1,29 +1,29 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <tests/common/dummystoragelink.h> +#include <tests/common/testhelper.h> +#include <tests/common/teststorageapp.h> #include <vespa/config/helper/configgetter.hpp> -#include <vespa/document/config/documenttypes_config_fwd.h> #include <vespa/document/config/config-documenttypes.h> +#include <vespa/document/config/documenttypes_config_fwd.h> #include <vespa/document/datatype/documenttype.h> #include <vespa/document/fieldvalue/document.h> -#include <vespa/document/update/documentupdate.h> #include <vespa/document/repo/documenttyperepo.h> -#include <vespa/document/test/make_document_bucket.h> #include <vespa/document/test/make_bucket_space.h> +#include <vespa/document/test/make_document_bucket.h> +#include <vespa/document/update/documentupdate.h> +#include <vespa/metrics/updatehook.h> #include <vespa/storage/bucketdb/bucketmanager.h> #include <vespa/storage/common/global_bucket_space_distribution_converter.h> #include <vespa/storage/persistence/filestorage/filestormanager.h> +#include <vespa/storageapi/message/bucketsplitting.h> #include <vespa/storageapi/message/persistence.h> #include <vespa/storageapi/message/state.h> -#include <vespa/storageapi/message/bucketsplitting.h> -#include <vespa/metrics/updatehook.h> -#include <tests/common/teststorageapp.h> -#include <tests/common/dummystoragelink.h> -#include <tests/common/testhelper.h> -#include <vespa/vdslib/state/random.h> #include <vespa/vdslib/distribution/distribution.h> #include <vespa/vdslib/state/clusterstate.h> -#include <vespa/vespalib/stllike/asciistream.h> +#include <vespa/vdslib/state/random.h> #include <vespa/vespalib/gtest/gtest.h> +#include <vespa/vespalib/stllike/asciistream.h> #include <future> #include <vespa/log/log.h> @@ -148,7 +148,9 @@ void BucketManagerTest::setupTestEnvironment(bool fakePersistenceLayer, bool noD _node->setTypeRepo(repo); _node->setupDummyPersistence(); // Set up the 3 links - auto manager = std::make_unique<BucketManager>(config::ConfigUri(config.getConfigId()), _node->getComponentRegister()); + auto config_uri = config::ConfigUri(config.getConfigId()); + using vespa::config::content::core::StorServerConfig; + auto manager = std::make_unique<BucketManager>(*config_from<StorServerConfig>(config_uri), _node->getComponentRegister()); _manager = manager.get(); _top->push_back(std::move(manager)); if (fakePersistenceLayer) { @@ -156,7 +158,8 @@ void BucketManagerTest::setupTestEnvironment(bool fakePersistenceLayer, bool noD _bottom = bottom.get(); _top->push_back(std::move(bottom)); } else { - auto bottom = std::make_unique<FileStorManager>(config::ConfigUri(config.getConfigId()), + using vespa::config::content::StorFilestorConfig; + auto bottom = std::make_unique<FileStorManager>(*config_from<StorFilestorConfig>(config_uri), _node->getPersistenceProvider(), _node->getComponentRegister(), *_node, _node->get_host_info()); _top->push_back(std::move(bottom)); diff --git a/storage/src/tests/common/testhelper.h b/storage/src/tests/common/testhelper.h index bfc5c7679e1..1f83e938409 100644 --- a/storage/src/tests/common/testhelper.h +++ b/storage/src/tests/common/testhelper.h @@ -1,5 +1,6 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once +#include <vespa/config/helper/configgetter.h> #include <vespa/messagebus/testlib/slobrok.h> #include <vespa/vdstestlib/config/dirconfig.h> #include <fstream> @@ -21,6 +22,11 @@ std::string getRootFolder(vdstestlib::DirConfig & dc); void addSlobrokConfig(vdstestlib::DirConfig& dc, const mbus::Slobrok& slobrok); +template <typename ConfigT> +std::unique_ptr<ConfigT> config_from(const ::config::ConfigUri& cfg_uri) { + return ::config::ConfigGetter<ConfigT>::getConfig(cfg_uri.getConfigId(), cfg_uri.getContext()); +} + // Class used to print start and end of test. Enable debug when you want to see // which test creates what output or where we get stuck struct TestName { diff --git a/storage/src/tests/distributor/mergelimitertest.cpp b/storage/src/tests/distributor/mergelimitertest.cpp index fc115b28c9a..7313c464a37 100644 --- a/storage/src/tests/distributor/mergelimitertest.cpp +++ b/storage/src/tests/distributor/mergelimitertest.cpp @@ -42,24 +42,30 @@ struct NodeFactory { operator const MergeLimiter::NodeArray&() const { return _nodes; } }; -#define ASSERT_LIMIT(maxNodes, nodes, result) \ -{ \ - MergeLimiter limiter(maxNodes); \ - auto nodesCopy = nodes; \ - limiter.limitMergeToMaxNodes(nodesCopy); \ - std::ostringstream actual; \ - for (uint32_t i = 0; i < nodesCopy.size(); ++i) { \ - if (i != 0) actual << ","; \ - actual << nodesCopy[i]._nodeIndex; \ - if (nodesCopy[i]._sourceOnly) actual << 's'; \ - } \ - ASSERT_EQ(result, actual.str()); \ } -} +struct MergeLimiterTest : Test { + + static std::string limit(uint32_t max_nodes, std::vector<MergeMetaData> nodes) { + MergeLimiter limiter(max_nodes); + limiter.limitMergeToMaxNodes(nodes); + std::ostringstream actual; + for (uint32_t i = 0; i < nodes.size(); ++i) { + if (i != 0) { + actual << ","; + } + actual << nodes[i]._nodeIndex; + if (nodes[i]._sourceOnly) { + actual << 's'; + } + } + return actual.str(); + } + +}; // If there is <= max nodes, then none should be removed. -TEST(MergeLimiterTest, keeps_all_below_limit) { +TEST_F(MergeLimiterTest, keeps_all_below_limit) { MergeLimiter::NodeArray nodes(NodeFactory() .addTrusted(3, 0x4) .addTrusted(5, 0x4) @@ -67,22 +73,22 @@ TEST(MergeLimiterTest, keeps_all_below_limit) { .add(2, 0x6) .add(4, 0x5)); - ASSERT_LIMIT(8, nodes, "3,5,9,2,4"); + ASSERT_EQ(limit(8, nodes), "3,5,9,2,4"); } // If less than max nodes is untrusted, merge all untrusted copies with a // trusted one. (Optionally with extra trusted copies if there is space) -TEST(MergeLimiterTest, less_than_max_untrusted) { +TEST_F(MergeLimiterTest, less_than_max_untrusted) { MergeLimiter::NodeArray nodes(NodeFactory() .addTrusted(3, 0x4) .addTrusted(5, 0x4) .add(9, 0x6) .add(2, 0x6) .add(4, 0x5)); - ASSERT_LIMIT(4, nodes, "2,4,9,5"); + ASSERT_EQ(limit(4, nodes), "2,4,9,5"); } // With more than max untrusted, just merge one trusted with as many untrusted // that fits. -TEST(MergeLimiterTest, more_than_max_untrusted) { +TEST_F(MergeLimiterTest, more_than_max_untrusted) { MergeLimiter::NodeArray nodes(NodeFactory() .addTrusted(3, 0x4) .addTrusted(5, 0x4) @@ -91,12 +97,12 @@ TEST(MergeLimiterTest, more_than_max_untrusted) { .add(13, 0x9) .add(1, 0x7) .add(4, 0x5)); - ASSERT_LIMIT(4, nodes, "2,13,1,5"); + ASSERT_EQ(limit(4, nodes), "2,13,1,5"); } // With nothing trusted. If there is <= max different variants (checksums), // merge one of each variant. After this merge, all these nodes can be set // trusted. (Except for any source only ones) -TEST(MergeLimiterTest, all_untrusted_less_than_max_variants) { +TEST_F(MergeLimiterTest, all_untrusted_less_than_max_variants) { MergeLimiter::NodeArray nodes(NodeFactory() .add(3, 0x4) .add(5, 0x4) @@ -105,11 +111,11 @@ TEST(MergeLimiterTest, all_untrusted_less_than_max_variants) { .add(13, 0x3) .add(1, 0x3) .add(4, 0x3)); - ASSERT_LIMIT(4, nodes, "5,2,4,3"); + ASSERT_EQ(limit(4, nodes), "5,2,4,3"); } // With nothing trusted and more than max variants, we just have to merge one // of each variant until we end up with less than max variants. -TEST(MergeLimiterTest, all_untrusted_more_than_max_variants) { +TEST_F(MergeLimiterTest, all_untrusted_more_than_max_variants) { MergeLimiter::NodeArray nodes(NodeFactory() .add(3, 0x4) .add(5, 0x5) @@ -118,12 +124,12 @@ TEST(MergeLimiterTest, all_untrusted_more_than_max_variants) { .add(13, 0x3) .add(1, 0x9) .add(4, 0x8)); - ASSERT_LIMIT(4, nodes, "3,5,2,13"); + ASSERT_EQ(limit(4, nodes), "3,5,2,13"); } // With more than max untrusted, just merge one trusted with as many untrusted // that fits. -TEST(MergeLimiterTest, source_only_last) { +TEST_F(MergeLimiterTest, source_only_last) { MergeLimiter::NodeArray nodes(NodeFactory() .addTrusted(3, 0x4) .addTrusted(5, 0x4).setSourceOnly() @@ -132,20 +138,20 @@ TEST(MergeLimiterTest, source_only_last) { .add(13, 0x9) .add(1, 0x7) .add(4, 0x5)); - ASSERT_LIMIT(4, nodes, "9,3,5s,2s"); + ASSERT_EQ(limit(4, nodes), "9,3,5s,2s"); } -TEST(MergeLimiterTest, limited_set_cannot_be_just_source_only) { +TEST_F(MergeLimiterTest, limited_set_cannot_be_just_source_only) { MergeLimiter::NodeArray nodes(NodeFactory() .addTrusted(9, 0x6) .addTrusted(2, 0x6) .addTrusted(13, 0x6).setSourceOnly() .add(1, 0x7).setSourceOnly()); - ASSERT_LIMIT(2, nodes, "2,13s"); - ASSERT_LIMIT(3, nodes, "2,13s,1s"); + ASSERT_EQ(limit(2, nodes), "2,13s"); + ASSERT_EQ(limit(3, nodes), "2,13s,1s"); } -TEST(MergeLimiterTest, non_source_only_replica_chosen_from_in_sync_group) { +TEST_F(MergeLimiterTest, non_source_only_replica_chosen_from_in_sync_group) { // nodes 9, 2, 13 are all in sync. Merge limiter will currently by default // pop the _last_ node of an in-sync replica "group" when outputting a limited // set. Unless we special-case source-only replicas here, we'd end up with an @@ -155,38 +161,38 @@ TEST(MergeLimiterTest, non_source_only_replica_chosen_from_in_sync_group) { .add(2, 0x6) .add(13, 0x6).setSourceOnly() .add(1, 0x7).setSourceOnly()); - ASSERT_LIMIT(2, nodes, "2,13s"); - ASSERT_LIMIT(3, nodes, "2,13s,1s"); + ASSERT_EQ(limit(2, nodes), "2,13s"); + ASSERT_EQ(limit(3, nodes), "2,13s,1s"); } -TEST(MergeLimiterTest, non_source_only_replicas_preferred_when_replicas_not_in_sync) { +TEST_F(MergeLimiterTest, non_source_only_replicas_preferred_when_replicas_not_in_sync) { MergeLimiter::NodeArray nodes(NodeFactory() .add(9, 0x4) .add(2, 0x5) .add(13, 0x6).setSourceOnly() .add(1, 0x7).setSourceOnly()); - ASSERT_LIMIT(2, nodes, "9,2"); - ASSERT_LIMIT(3, nodes, "9,2,13s"); + ASSERT_EQ(limit(2, nodes), "9,2"); + ASSERT_EQ(limit(3, nodes), "9,2,13s"); } -TEST(MergeLimiterTest, at_least_one_non_source_only_replica_chosen_when_all_trusted) { +TEST_F(MergeLimiterTest, at_least_one_non_source_only_replica_chosen_when_all_trusted) { MergeLimiter::NodeArray nodes(NodeFactory() .addTrusted(9, 0x6) .addTrusted(2, 0x6) .addTrusted(13, 0x6).setSourceOnly() .addTrusted(1, 0x6).setSourceOnly()); - ASSERT_LIMIT(2, nodes, "2,13s"); - ASSERT_LIMIT(3, nodes, "2,13s,1s"); + ASSERT_EQ(limit(2, nodes), "2,13s"); + ASSERT_EQ(limit(3, nodes), "2,13s,1s"); } -TEST(MergeLimiterTest, missing_replica_distinct_from_empty_replica) { +TEST_F(MergeLimiterTest, missing_replica_distinct_from_empty_replica) { MergeLimiter::NodeArray nodes(NodeFactory() .addEmpty(3) .addEmpty(5) .addMissing(1) .addMissing(2)); - ASSERT_LIMIT(2, nodes, "5,2"); - ASSERT_LIMIT(3, nodes, "5,2,3"); + ASSERT_EQ(limit(2, nodes), "5,2"); + ASSERT_EQ(limit(3, nodes), "5,2,3"); } } // storage::distributor diff --git a/storage/src/tests/persistence/common/filestortestfixture.cpp b/storage/src/tests/persistence/common/filestortestfixture.cpp index c989acd5228..86015bee4e7 100644 --- a/storage/src/tests/persistence/common/filestortestfixture.cpp +++ b/storage/src/tests/persistence/common/filestortestfixture.cpp @@ -1,15 +1,17 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/storage/persistence/messages.h> -#include <vespa/storage/persistence/filestorage/filestormanager.h> -#include <vespa/storageapi/message/bucket.h> -#include <vespa/persistence/dummyimpl/dummypersistence.h> +#include <tests/common/testhelper.h> #include <tests/persistence/common/filestortestfixture.h> -#include <vespa/document/repo/documenttyperepo.h> +#include <vespa/config/helper/configgetter.hpp> #include <vespa/document/fieldset/fieldsets.h> +#include <vespa/document/repo/documenttyperepo.h> #include <vespa/document/test/make_document_bucket.h> -#include <vespa/vdslib/state/clusterstate.h> +#include <vespa/persistence/dummyimpl/dummypersistence.h> #include <vespa/persistence/spi/test.h> +#include <vespa/storage/persistence/filestorage/filestormanager.h> +#include <vespa/storage/persistence/messages.h> +#include <vespa/storageapi/message/bucket.h> +#include <vespa/vdslib/state/clusterstate.h> #include <sstream> using storage::spi::test::makeSpiBucket; @@ -73,18 +75,20 @@ FileStorTestFixture::TestFileStorComponents::TestFileStorComponents( manager(nullptr) { injector.inject(top); - auto fsm = std::make_unique<FileStorManager>(config::ConfigUri(fixture._config->getConfigId()), fixture._node->getPersistenceProvider(), + using vespa::config::content::StorFilestorConfig; + auto config = config_from<StorFilestorConfig>(config::ConfigUri(fixture._config->getConfigId())); + auto fsm = std::make_unique<FileStorManager>(*config, fixture._node->getPersistenceProvider(), fixture._node->getComponentRegister(), *fixture._node, fixture._node->get_host_info()); manager = fsm.get(); top.push_back(std::move(fsm)); top.open(); } -vespalib::string _Storage("storage"); +vespalib::string _storage("storage"); api::StorageMessageAddress FileStorTestFixture::makeSelfAddress() { - return api::StorageMessageAddress(&_Storage, lib::NodeType::STORAGE, 0); + return api::StorageMessageAddress(&_storage, lib::NodeType::STORAGE, 0); } void diff --git a/storage/src/tests/persistence/filestorage/filestormanagertest.cpp b/storage/src/tests/persistence/filestorage/filestormanagertest.cpp index b09febce408..2b0218bf20c 100644 --- a/storage/src/tests/persistence/filestorage/filestormanagertest.cpp +++ b/storage/src/tests/persistence/filestorage/filestormanagertest.cpp @@ -5,6 +5,7 @@ #include <tests/common/teststorageapp.h> #include <tests/persistence/filestorage/forwardingmessagesender.h> #include <vespa/config/common/exceptions.h> +#include <vespa/config/helper/configgetter.hpp> #include <vespa/document/fieldset/fieldsets.h> #include <vespa/document/repo/documenttyperepo.h> #include <vespa/document/select/parser.h> @@ -65,11 +66,11 @@ namespace storage { namespace { -vespalib::string _Cluster("cluster"); -vespalib::string _Storage("storage"); -api::StorageMessageAddress _Storage2(&_Storage, lib::NodeType::STORAGE, 2); -api::StorageMessageAddress _Storage3(&_Storage, lib::NodeType::STORAGE, 3); -api::StorageMessageAddress _Cluster1(&_Cluster, lib::NodeType::STORAGE, 1); +vespalib::string _cluster("cluster"); +vespalib::string _storage("storage"); +api::StorageMessageAddress _storage2(&_storage, lib::NodeType::STORAGE, 2); +api::StorageMessageAddress _storage3(&_storage, lib::NodeType::STORAGE, 3); +api::StorageMessageAddress _cluster1(&_cluster, lib::NodeType::STORAGE, 1); struct TestFileStorComponents; @@ -93,7 +94,7 @@ struct FileStorTestBase : Test { const document::DocumentType* _testdoctype1; FileStorTestBase() : _node(), _waitTime(LONG_WAITTIME) {} - ~FileStorTestBase(); + ~FileStorTestBase() override; void SetUp() override; void TearDown() override; @@ -223,8 +224,10 @@ struct TestFileStorComponents { explicit TestFileStorComponents(FileStorTestBase& test, bool use_small_config = false) : manager(nullptr) { - auto fsm = std::make_unique<FileStorManager>(config::ConfigUri((use_small_config ? test.smallConfig : test.config)->getConfigId()), - test._node->getPersistenceProvider(), + using vespa::config::content::StorFilestorConfig; + auto config_uri = config::ConfigUri((use_small_config ? test.smallConfig : test.config)->getConfigId()); + auto config = config_from<StorFilestorConfig>(config_uri); + auto fsm = std::make_unique<FileStorManager>(*config, test._node->getPersistenceProvider(), test._node->getComponentRegister(), *test._node, test._node->get_host_info()); manager = fsm.get(); top.push_back(std::move(fsm)); @@ -324,7 +327,7 @@ TEST_F(FileStorManagerTest, header_only_put) { // Putting it { auto cmd = std::make_shared<api::PutCommand>(makeDocumentBucket(bid), doc, 105); - cmd->setAddress(_Storage3); + cmd->setAddress(_storage3); top.sendDown(cmd); top.waitForMessages(1, _waitTime); ASSERT_EQ(1, top.getNumReplies()); @@ -339,7 +342,7 @@ TEST_F(FileStorManagerTest, header_only_put) { { auto cmd = std::make_shared<api::PutCommand>(makeDocumentBucket(bid), doc, 124); cmd->setUpdateTimestamp(105); - cmd->setAddress(_Storage3); + cmd->setAddress(_storage3); top.sendDown(cmd); top.waitForMessages(1, _waitTime); ASSERT_EQ(1, top.getNumReplies()); @@ -351,7 +354,7 @@ TEST_F(FileStorManagerTest, header_only_put) { // Getting it { auto cmd = std::make_shared<api::GetCommand>(makeDocumentBucket(bid), doc->getId(), document::AllFields::NAME); - cmd->setAddress(_Storage3); + cmd->setAddress(_storage3); top.sendDown(cmd); top.waitForMessages(1, _waitTime); ASSERT_EQ(1, top.getNumReplies()); @@ -386,7 +389,7 @@ TEST_F(FileStorManagerTest, put) { // Putting it { auto cmd = std::make_shared<api::PutCommand>(makeDocumentBucket(bid), doc, 105); - cmd->setAddress(_Storage3); + cmd->setAddress(_storage3); top.sendDown(cmd); top.waitForMessages(1, _waitTime); ASSERT_EQ(1, top.getNumReplies()); @@ -467,7 +470,7 @@ TEST_F(FileStorManagerTest, flush) { std::vector<std::shared_ptr<api::StorageCommand> > _commands; for (uint32_t i=0; i<msgCount; ++i) { auto cmd = std::make_shared<api::PutCommand>(makeDocumentBucket(bid), doc, i+1); - cmd->setAddress(_Storage3); + cmd->setAddress(_storage3); _commands.push_back(cmd); } for (uint32_t i=0; i<msgCount; ++i) { @@ -494,7 +497,7 @@ TEST_F(FileStorManagerTest, handler_priority) { // Populate bucket with the given data for (uint32_t i = 1; i < 6; i++) { auto cmd = std::make_shared<api::PutCommand>(makeDocumentBucket(bucket), doc, 100); - cmd->setAddress(_Storage3); + cmd->setAddress(_storage3); cmd->setPriority(i * 15); filestorHandler.schedule(cmd); } @@ -629,7 +632,7 @@ TEST_F(FileStorManagerTest, handler_pause) { // Populate bucket with the given data for (uint32_t i = 1; i < 6; i++) { auto cmd = std::make_shared<api::PutCommand>(makeDocumentBucket(bucket), doc, 100); - cmd->setAddress(_Storage3); + cmd->setAddress(_storage3); cmd->setPriority(i * 15); filestorHandler.schedule(cmd); } @@ -705,7 +708,7 @@ TEST_F(FileStorManagerTest, handler_timeout) { // Populate bucket with the given data { auto cmd = std::make_shared<api::PutCommand>(makeDocumentBucket(bucket), doc, 100); - cmd->setAddress(_Storage3); + cmd->setAddress(_storage3); cmd->setPriority(0); cmd->setTimeout(50ms); filestorHandler.schedule(cmd); @@ -713,7 +716,7 @@ TEST_F(FileStorManagerTest, handler_timeout) { { auto cmd = std::make_shared<api::PutCommand>(makeDocumentBucket(bucket), doc, 100); - cmd->setAddress(_Storage3); + cmd->setAddress(_storage3); cmd->setPriority(200); cmd->setTimeout(10000ms); filestorHandler.schedule(cmd); @@ -773,7 +776,7 @@ TEST_F(FileStorManagerTest, priority) { document::BucketId bucket(16, factory.getBucketId(documents[i]->getId()).getRawId()); auto cmd = std::make_shared<api::PutCommand>(makeDocumentBucket(bucket), documents[i], 100 + i); - cmd->setAddress(_Storage3); + cmd->setAddress(_storage3); cmd->setPriority(i * 2); filestorHandler.schedule(cmd); } @@ -832,7 +835,7 @@ TEST_F(FileStorManagerTest, split1) { _node->getPersistenceProvider().createBucket(makeSpiBucket(bucket)); auto cmd = std::make_shared<api::PutCommand>(makeDocumentBucket(bucket), documents[i], 100 + i); - cmd->setAddress(_Storage3); + cmd->setAddress(_storage3); cmd->setSourceIndex(0); filestorHandler.schedule(cmd); @@ -847,7 +850,7 @@ TEST_F(FileStorManagerTest, split1) { // Delete every 5th document to have delete entries in file too if (i % 5 == 0) { auto rcmd = std::make_shared<api::RemoveCommand>(makeDocumentBucket(bucket), documents[i]->getId(), 1000000 + 100 + i); - rcmd->setAddress(_Storage3); + rcmd->setAddress(_storage3); filestorHandler.schedule(rcmd); filestorHandler.flush(true); ASSERT_EQ(1, top.getNumReplies()); @@ -875,7 +878,7 @@ TEST_F(FileStorManagerTest, split1) { for (uint32_t i=0; i<documents.size(); ++i) { document::BucketId bucket(17, i % 3 == 0 ? 0x10001 : 0x0100001); auto cmd = std::make_shared<api::GetCommand>(makeDocumentBucket(bucket), documents[i]->getId(), document::AllFields::NAME); - cmd->setAddress(_Storage3); + cmd->setAddress(_storage3); filestorHandler.schedule(cmd); filestorHandler.flush(true); ASSERT_EQ(1, top.getNumReplies()); @@ -907,7 +910,7 @@ TEST_F(FileStorManagerTest, split1) { bucket = document::BucketId(33, factory.getBucketId(documents[i]->getId()).getRawId()); } auto cmd = std::make_shared<api::GetCommand>(makeDocumentBucket(bucket), documents[i]->getId(), document::AllFields::NAME); - cmd->setAddress(_Storage3); + cmd->setAddress(_storage3); filestorHandler.schedule(cmd); filestorHandler.flush(true); ASSERT_EQ(1, top.getNumReplies()); @@ -953,7 +956,7 @@ TEST_F(FileStorManagerTest, split_single_group) { _node->getPersistenceProvider().createBucket(makeSpiBucket(bucket)); auto cmd = std::make_shared<api::PutCommand>(makeDocumentBucket(bucket), documents[i], 100 + i); - cmd->setAddress(_Storage3); + cmd->setAddress(_storage3); filestorHandler.schedule(cmd); filestorHandler.flush(true); ASSERT_EQ(1, top.getNumReplies()); @@ -979,7 +982,7 @@ TEST_F(FileStorManagerTest, split_single_group) { for (uint32_t i=0; i<documents.size(); ++i) { document::BucketId bucket(17, state ? 0x10001 : 0x00001); auto cmd = std::make_shared<api::GetCommand>(makeDocumentBucket(bucket), documents[i]->getId(), document::AllFields::NAME); - cmd->setAddress(_Storage3); + cmd->setAddress(_storage3); filestorHandler.schedule(cmd); filestorHandler.flush(true); ASSERT_EQ(1, top.getNumReplies()); @@ -1007,7 +1010,7 @@ FileStorTestBase::putDoc(DummyStorageLink& top, _node->getPersistenceProvider().createBucket(makeSpiBucket(target)); auto doc = std::make_shared<Document>(*_node->getTypeRepo(), *_testdoctype1, docId); auto cmd = std::make_shared<api::PutCommand>(makeDocumentBucket(target), doc, docNum+1); - cmd->setAddress(_Storage3); + cmd->setAddress(_storage3); cmd->setPriority(120); filestorHandler.schedule(cmd); filestorHandler.flush(true); @@ -1048,7 +1051,7 @@ TEST_F(FileStorManagerTest, split_empty_target_with_remapped_ops) { vespalib::make_string("id:ns:testdoctype1:n=%d:1234", 0x100001)); auto doc = std::make_shared<Document>(*_node->getTypeRepo(), *_testdoctype1, docId); auto putCmd = std::make_shared<api::PutCommand>(makeDocumentBucket(source), doc, 1001); - putCmd->setAddress(_Storage3); + putCmd->setAddress(_storage3); putCmd->setPriority(120); filestorHandler.schedule(splitCmd); @@ -1129,7 +1132,7 @@ TEST_F(FileStorManagerTest, join) { for (uint32_t i=0; i<documents.size(); ++i) { document::BucketId bucket(17, factory.getBucketId(documents[i]->getId()).getRawId()); auto cmd = std::make_shared<api::PutCommand>(makeDocumentBucket(bucket), documents[i], 100 + i); - cmd->setAddress(_Storage3); + cmd->setAddress(_storage3); filestorHandler.schedule(cmd); filestorHandler.flush(true); ASSERT_EQ(1, top.getNumReplies()); @@ -1141,7 +1144,7 @@ TEST_F(FileStorManagerTest, join) { if ((i % 5) == 0) { auto rcmd = std::make_shared<api::RemoveCommand>( makeDocumentBucket(bucket), documents[i]->getId(), 1000000 + 100 + i); - rcmd->setAddress(_Storage3); + rcmd->setAddress(_storage3); filestorHandler.schedule(rcmd); filestorHandler.flush(true); ASSERT_EQ(1, top.getNumReplies()); @@ -1170,7 +1173,7 @@ TEST_F(FileStorManagerTest, join) { document::BucketId bucket(16, 1); auto cmd = std::make_shared<api::GetCommand>( makeDocumentBucket(bucket), documents[i]->getId(), document::AllFields::NAME); - cmd->setAddress(_Storage3); + cmd->setAddress(_storage3); filestorHandler.schedule(cmd); filestorHandler.flush(true); ASSERT_EQ(1, top.getNumReplies()); @@ -1342,7 +1345,7 @@ TEST_F(FileStorManagerTest, remove_location) { docid << "id:ns:testdoctype1:n=" << (i << 8) << ":foo"; Document::SP doc(createDocument("some content", docid.str())); auto cmd = std::make_shared<api::PutCommand>(makeDocumentBucket(bid), doc, 1000 + i); - cmd->setAddress(_Storage3); + cmd->setAddress(_storage3); top.sendDown(cmd); top.waitForMessages(1, _waitTime); ASSERT_EQ(1, top.getNumReplies()); @@ -1355,7 +1358,7 @@ TEST_F(FileStorManagerTest, remove_location) { // Issuing remove location command { auto cmd = std::make_shared<api::RemoveLocationCommand>("id.user % 512 == 0", makeDocumentBucket(bid)); - cmd->setAddress(_Storage3); + cmd->setAddress(_storage3); top.sendDown(cmd); top.waitForMessages(1, _waitTime); ASSERT_EQ(1, top.getNumReplies()); @@ -1381,7 +1384,7 @@ TEST_F(FileStorManagerTest, delete_bucket) { // Putting it { auto cmd = std::make_shared<api::PutCommand>(makeDocumentBucket(bid), doc, 105); - cmd->setAddress(_Storage2); + cmd->setAddress(_storage2); top.sendDown(cmd); top.waitForMessages(1, _waitTime); ASSERT_EQ(1, top.getNumReplies()); @@ -1397,7 +1400,7 @@ TEST_F(FileStorManagerTest, delete_bucket) { // Delete bucket { auto cmd = std::make_shared<api::DeleteBucketCommand>(makeDocumentBucket(bid)); - cmd->setAddress(_Storage2); + cmd->setAddress(_storage2); cmd->setBucketInfo(bucketInfo); top.sendDown(cmd); top.waitForMessages(1, _waitTime); @@ -1423,7 +1426,7 @@ TEST_F(FileStorManagerTest, delete_bucket_rejects_outdated_bucket_info) { // Putting it { auto cmd = std::make_shared<api::PutCommand>(makeDocumentBucket(bid), doc, 105); - cmd->setAddress(_Storage2); + cmd->setAddress(_storage2); top.sendDown(cmd); top.waitForMessages(1, _waitTime); ASSERT_EQ(1, top.getNumReplies()); @@ -1440,7 +1443,7 @@ TEST_F(FileStorManagerTest, delete_bucket_rejects_outdated_bucket_info) { { auto cmd = std::make_shared<api::DeleteBucketCommand>(makeDocumentBucket(bid)); cmd->setBucketInfo(api::BucketInfo(0xf000baaa, 1, 123, 1, 456)); - cmd->setAddress(_Storage2); + cmd->setAddress(_storage2); top.sendDown(cmd); top.waitForMessages(1, _waitTime); ASSERT_EQ(1, top.getNumReplies()); @@ -1468,7 +1471,7 @@ TEST_F(FileStorManagerTest, delete_bucket_with_invalid_bucket_info){ // Putting it { auto cmd = std::make_shared<api::PutCommand>(makeDocumentBucket(bid), doc, 105); - cmd->setAddress(_Storage2); + cmd->setAddress(_storage2); top.sendDown(cmd); top.waitForMessages(1, _waitTime); ASSERT_EQ(1, top.getNumReplies()); @@ -1482,7 +1485,7 @@ TEST_F(FileStorManagerTest, delete_bucket_with_invalid_bucket_info){ // Attempt to delete bucket with invalid bucketinfo { auto cmd = std::make_shared<api::DeleteBucketCommand>(makeDocumentBucket(bid)); - cmd->setAddress(_Storage2); + cmd->setAddress(_storage2); top.sendDown(cmd); top.waitForMessages(1, _waitTime); ASSERT_EQ(1, top.getNumReplies()); @@ -1506,7 +1509,7 @@ TEST_F(FileStorManagerTest, no_timestamps) { // Putting it { auto cmd = std::make_shared<api::PutCommand>(makeDocumentBucket(bid), doc, 0); - cmd->setAddress(_Storage3); + cmd->setAddress(_storage3); EXPECT_EQ(api::Timestamp(0), cmd->getTimestamp()); top.sendDown(cmd); top.waitForMessages(1, _waitTime); @@ -1519,7 +1522,7 @@ TEST_F(FileStorManagerTest, no_timestamps) { // Removing it { auto cmd = std::make_shared<api::RemoveCommand>(makeDocumentBucket(bid), doc->getId(), 0); - cmd->setAddress(_Storage3); + cmd->setAddress(_storage3); EXPECT_EQ(api::Timestamp(0), cmd->getTimestamp()); top.sendDown(cmd); top.waitForMessages(1, _waitTime); @@ -1544,7 +1547,7 @@ TEST_F(FileStorManagerTest, equal_timestamps) { Document::SP doc(createDocument( "some content", "id:crawler:testdoctype1:n=4000:http://www.ntnu.no/")); auto cmd = std::make_shared<api::PutCommand>(makeDocumentBucket(bid), doc, 100); - cmd->setAddress(_Storage3); + cmd->setAddress(_storage3); top.sendDown(cmd); top.waitForMessages(1, _waitTime); ASSERT_EQ(1, top.getNumReplies()); @@ -1561,7 +1564,7 @@ TEST_F(FileStorManagerTest, equal_timestamps) { Document::SP doc(createDocument( "some content", "id:crawler:testdoctype1:n=4000:http://www.ntnu.no/")); auto cmd = std::make_shared<api::PutCommand>(makeDocumentBucket(bid), doc, 100); - cmd->setAddress(_Storage3); + cmd->setAddress(_storage3); top.sendDown(cmd); top.waitForMessages(1, _waitTime); ASSERT_EQ(1, top.getNumReplies()); @@ -1576,7 +1579,7 @@ TEST_F(FileStorManagerTest, equal_timestamps) { Document::SP doc(createDocument( "some content", "id:crawler:testdoctype1:n=4000:http://www.ntnu.nu/")); auto cmd = std::make_shared<api::PutCommand>(makeDocumentBucket(bid), doc, 100); - cmd->setAddress(_Storage3); + cmd->setAddress(_storage3); top.sendDown(cmd); top.waitForMessages(1, _waitTime); ASSERT_EQ(1, top.getNumReplies()); @@ -1608,7 +1611,7 @@ TEST_F(FileStorManagerTest, get_iter) { // Putting all docs to have something to visit for (uint32_t i=0; i<docs.size(); ++i) { auto cmd = std::make_shared<api::PutCommand>(makeDocumentBucket(bid), docs[i], 100 + i); - cmd->setAddress(_Storage3); + cmd->setAddress(_storage3); top.sendDown(cmd); top.waitForMessages(1, _waitTime); ASSERT_EQ(1, top.getNumReplies()); @@ -1671,7 +1674,7 @@ TEST_F(FileStorManagerTest, set_bucket_active_state) { { auto cmd = std::make_shared<api::SetBucketStateCommand>(makeDocumentBucket(bid), api::SetBucketStateCommand::ACTIVE); - cmd->setAddress(_Storage3); + cmd->setAddress(_storage3); top.sendDown(cmd); top.waitForMessages(1, _waitTime); ASSERT_EQ(1, top.getNumReplies()); @@ -1690,7 +1693,7 @@ TEST_F(FileStorManagerTest, set_bucket_active_state) { { auto cmd = std::make_shared<api::SetBucketStateCommand>( makeDocumentBucket(bid), api::SetBucketStateCommand::INACTIVE); - cmd->setAddress(_Storage3); + cmd->setAddress(_storage3); top.sendDown(cmd); top.waitForMessages(1, _waitTime); ASSERT_EQ(1, top.getNumReplies()); @@ -1717,7 +1720,7 @@ TEST_F(FileStorManagerTest, notify_owner_distributor_on_outdated_set_bucket_stat createBucket(bid); auto cmd = std::make_shared<api::SetBucketStateCommand>(makeDocumentBucket(bid), api::SetBucketStateCommand::ACTIVE); - cmd->setAddress(_Cluster1); + cmd->setAddress(_cluster1); cmd->setSourceIndex(0); top.sendDown(cmd); @@ -1752,7 +1755,7 @@ TEST_F(FileStorManagerTest, GetBucketDiff_implicitly_creates_bucket) { std::vector<api::MergeBucketCommand::Node> nodes = {1, 0}; auto cmd = std::make_shared<api::GetBucketDiffCommand>(makeDocumentBucket(bid), nodes, Timestamp(1000)); - cmd->setAddress(_Cluster1); + cmd->setAddress(_cluster1); cmd->setSourceIndex(0); top.sendDown(cmd); @@ -1776,7 +1779,7 @@ TEST_F(FileStorManagerTest, merge_bucket_implicitly_creates_bucket) { std::vector<api::MergeBucketCommand::Node> nodes = {1, 2}; auto cmd = std::make_shared<api::MergeBucketCommand>(makeDocumentBucket(bid), nodes, Timestamp(1000)); - cmd->setAddress(_Cluster1); + cmd->setAddress(_cluster1); cmd->setSourceIndex(0); top.sendDown(cmd); @@ -1797,7 +1800,7 @@ TEST_F(FileStorManagerTest, newly_created_bucket_is_ready) { document::BucketId bid(16, 4000); auto cmd = std::make_shared<api::CreateBucketCommand>(makeDocumentBucket(bid)); - cmd->setAddress(_Cluster1); + cmd->setAddress(_cluster1); cmd->setSourceIndex(0); top.sendDown(cmd); @@ -1818,7 +1821,7 @@ TEST_F(FileStorManagerTest, create_bucket_sets_active_flag_in_database_and_reply document::BucketId bid(16, 4000); auto cmd = std::make_shared<api::CreateBucketCommand>(makeDocumentBucket(bid)); - cmd->setAddress(_Cluster1); + cmd->setAddress(_cluster1); cmd->setSourceIndex(0); cmd->setActive(true); c.top.sendDown(cmd); @@ -1837,7 +1840,7 @@ TEST_F(FileStorManagerTest, create_bucket_sets_active_flag_in_database_and_reply template <typename Metric> void FileStorTestBase::assert_request_size_set(TestFileStorComponents& c, std::shared_ptr<api::StorageMessage> cmd, const Metric& metric) { cmd->setApproxByteSize(54321); - cmd->setAddress(_Storage3); + cmd->setAddress(_storage3); c.top.sendDown(cmd); c.top.waitForMessages(1, _waitTime); EXPECT_EQ(static_cast<int64_t>(cmd->getApproxByteSize()), metric.request_size.getLast()); @@ -1894,7 +1897,7 @@ TEST_F(FileStorManagerTest, test_and_set_condition_mismatch_not_counted_as_failu createBucket(bucket); auto cmd = std::make_shared<api::PutCommand>(makeDocumentBucket(bucket), std::move(doc), api::Timestamp(12345)); cmd->setCondition(TestAndSetCondition("not testdoctype1")); - cmd->setAddress(_Storage3); + cmd->setAddress(_storage3); c.top.sendDown(cmd); api::PutReply* reply; diff --git a/storage/src/tests/persistence/filestorage/filestormodifiedbucketstest.cpp b/storage/src/tests/persistence/filestorage/filestormodifiedbucketstest.cpp index 966382de39b..710da80972f 100644 --- a/storage/src/tests/persistence/filestorage/filestormodifiedbucketstest.cpp +++ b/storage/src/tests/persistence/filestorage/filestormodifiedbucketstest.cpp @@ -1,11 +1,12 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/storageapi/message/bucket.h> -#include <vespa/storage/persistence/filestorage/modifiedbucketchecker.h> -#include <vespa/persistence/spi/test.h> +#include <tests/persistence/common/filestortestfixture.h> #include <tests/persistence/common/persistenceproviderwrapper.h> +#include <vespa/config/helper/configgetter.hpp> #include <vespa/persistence/dummyimpl/dummypersistence.h> -#include <tests/persistence/common/filestortestfixture.h> +#include <vespa/persistence/spi/test.h> +#include <vespa/storage/persistence/filestorage/modifiedbucketchecker.h> +#include <vespa/storageapi/message/bucket.h> using storage::spi::test::makeSpiBucket; using namespace ::testing; @@ -36,10 +37,10 @@ struct BucketCheckerInjector : FileStorTestFixture::StorageLinkInjector _fixture(fixture) {} void inject(DummyStorageLink& link) const override { + using vespa::config::content::core::StorServerConfig; + auto cfg = config_from<StorServerConfig>(config::ConfigUri(_fixture._config->getConfigId())); link.push_back(std::make_unique<ModifiedBucketChecker>( - _node.getComponentRegister(), - _node.getPersistenceProvider(), - config::ConfigUri(_fixture._config->getConfigId()))); + _node.getComponentRegister(), _node.getPersistenceProvider(), *cfg)); } }; diff --git a/storage/src/tests/persistence/filestorage/modifiedbucketcheckertest.cpp b/storage/src/tests/persistence/filestorage/modifiedbucketcheckertest.cpp index 8acaa9a78d3..f96ff9c012e 100644 --- a/storage/src/tests/persistence/filestorage/modifiedbucketcheckertest.cpp +++ b/storage/src/tests/persistence/filestorage/modifiedbucketcheckertest.cpp @@ -1,11 +1,12 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <tests/common/testhelper.h> #include <tests/common/dummystoragelink.h> +#include <tests/common/testhelper.h> #include <tests/common/teststorageapp.h> +#include <vespa/config/common/exceptions.h> +#include <vespa/config/helper/configgetter.hpp> #include <vespa/persistence/dummyimpl/dummypersistence.h> #include <vespa/storage/persistence/filestorage/modifiedbucketchecker.h> -#include <vespa/config/common/exceptions.h> #include <vespa/vespalib/gtest/gtest.h> using namespace ::testing; @@ -45,9 +46,11 @@ ModifiedBucketCheckerTest::SetUp() _node->setupDummyPersistence(); _top.reset(new DummyStorageLink); + using vespa::config::content::core::StorServerConfig; + auto bootstrap_cfg = config_from<StorServerConfig>(config::ConfigUri(_config->getConfigId())); _handler = new ModifiedBucketChecker(_node->getComponentRegister(), _node->getPersistenceProvider(), - config::ConfigUri(_config->getConfigId())); + *bootstrap_cfg); _top->push_back(std::unique_ptr<StorageLink>(_handler)); _bottom = new DummyStorageLink; _handler->push_back(std::unique_ptr<StorageLink>(_bottom)); @@ -136,7 +139,7 @@ TEST_F(ModifiedBucketCheckerTest, recheck_requests_are_chunked) { _top->open(); cfgns::StorServerConfigBuilder cfgBuilder; cfgBuilder.bucketRecheckingChunkSize = 2; - _handler->configure(std::make_unique<cfgns::StorServerConfig>(cfgBuilder)); + _handler->on_configure(*std::make_unique<cfgns::StorServerConfig>(cfgBuilder)); modifyBuckets(5, 0); _handler->tick(); @@ -172,7 +175,7 @@ TEST_F(ModifiedBucketCheckerTest, invalid_chunk_size_config_is_rejected) { _top->open(); cfgns::StorServerConfigBuilder cfgBuilder; cfgBuilder.bucketRecheckingChunkSize = 0; - EXPECT_THROW(_handler->configure(std::make_unique<cfgns::StorServerConfig>(cfgBuilder)), + EXPECT_THROW(_handler->on_configure(*std::make_unique<cfgns::StorServerConfig>(cfgBuilder)), config::InvalidConfigException); } diff --git a/storage/src/tests/storageserver/bouncertest.cpp b/storage/src/tests/storageserver/bouncertest.cpp index c41696e1a02..225b3c94120 100644 --- a/storage/src/tests/storageserver/bouncertest.cpp +++ b/storage/src/tests/storageserver/bouncertest.cpp @@ -1,20 +1,22 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/storageapi/message/bucket.h> -#include <vespa/storageapi/message/state.h> -#include <vespa/storageapi/message/stat.h> -#include <vespa/storage/storageserver/bouncer.h> -#include <vespa/storage/storageserver/bouncer_metrics.h> -#include <tests/common/teststorageapp.h> -#include <tests/common/testhelper.h> #include <tests/common/dummystoragelink.h> +#include <tests/common/testhelper.h> +#include <tests/common/teststorageapp.h> +#include <vespa/config/common/exceptions.h> +#include <vespa/config/helper/configgetter.hpp> #include <vespa/document/bucket/fixed_bucket_spaces.h> -#include <vespa/document/test/make_document_bucket.h> #include <vespa/document/fieldset/fieldsets.h> -#include <vespa/storageapi/message/persistence.h> +#include <vespa/document/test/make_document_bucket.h> #include <vespa/persistence/spi/bucket_limits.h> +#include <vespa/storage/config/config-stor-bouncer.h> +#include <vespa/storage/storageserver/bouncer.h> +#include <vespa/storage/storageserver/bouncer_metrics.h> +#include <vespa/storageapi/message/bucket.h> +#include <vespa/storageapi/message/persistence.h> +#include <vespa/storageapi/message/stat.h> +#include <vespa/storageapi/message/state.h> #include <vespa/vdslib/state/clusterstate.h> -#include <vespa/config/common/exceptions.h> #include <vespa/vespalib/gtest/gtest.h> using document::test::makeDocumentBucket; @@ -51,9 +53,10 @@ struct BouncerTest : public Test { api::Timestamp timestamp, document::BucketSpace bucketSpace); - void expectMessageBouncedWithRejection(); - void expectMessageBouncedWithAbort(); - void expectMessageNotBounced(); + void expectMessageBouncedWithRejection() const; + void expect_message_bounced_with_node_down_abort() const; + void expect_message_bounced_with_shutdown_abort() const; + void expectMessageNotBounced() const; }; BouncerTest::BouncerTest() @@ -72,7 +75,10 @@ void BouncerTest::setUpAsNode(const lib::NodeType& type) { _node.reset(new TestDistributorApp(NodeIndex(2), config.getConfigId())); } _upper.reset(new DummyStorageLink()); - _manager = new Bouncer(_node->getComponentRegister(), config::ConfigUri(config.getConfigId())); + using StorBouncerConfig = vespa::config::content::core::StorBouncerConfig; + auto cfg_uri = config::ConfigUri(config.getConfigId()); + auto cfg = config::ConfigGetter<StorBouncerConfig>::getConfig(cfg_uri.getConfigId(), cfg_uri.getContext()); + _manager = new Bouncer(_node->getComponentRegister(), *cfg); _lower = new DummyStorageLink(); _upper->push_back(std::unique_ptr<StorageLink>(_manager)); _upper->push_back(std::unique_ptr<StorageLink>(_lower)); @@ -181,7 +187,7 @@ TEST_F(BouncerTest, allow_notify_bucket_change_even_when_distributor_down) { } void -BouncerTest::expectMessageBouncedWithRejection() +BouncerTest::expectMessageBouncedWithRejection() const { ASSERT_EQ(1, _upper->getNumReplies()); EXPECT_EQ(0, _upper->getNumCommands()); @@ -191,7 +197,7 @@ BouncerTest::expectMessageBouncedWithRejection() } void -BouncerTest::expectMessageBouncedWithAbort() +BouncerTest::expect_message_bounced_with_node_down_abort() const { ASSERT_EQ(1, _upper->getNumReplies()); EXPECT_EQ(0, _upper->getNumCommands()); @@ -204,7 +210,17 @@ BouncerTest::expectMessageBouncedWithAbort() } void -BouncerTest::expectMessageNotBounced() +BouncerTest::expect_message_bounced_with_shutdown_abort() const +{ + ASSERT_EQ(1, _upper->getNumReplies()); + EXPECT_EQ(0, _upper->getNumCommands()); + auto& reply = dynamic_cast<api::StorageReply&>(*_upper->getReply(0)); + EXPECT_EQ(api::ReturnCode(api::ReturnCode::ABORTED, "Node is shutting down"), reply.getResult()); + EXPECT_EQ(0, _lower->getNumCommands()); +} + +void +BouncerTest::expectMessageNotBounced() const { EXPECT_EQ(size_t(0), _upper->getNumReplies()); EXPECT_EQ(size_t(1), _lower->getNumCommands()); @@ -214,9 +230,9 @@ void BouncerTest::configureRejectionThreshold(int newThreshold) { using Builder = vespa::config::content::core::StorBouncerConfigBuilder; - auto config = std::make_unique<Builder>(); - config->feedRejectionPriorityThreshold = newThreshold; - _manager->configure(std::move(config)); + Builder config; + config.feedRejectionPriorityThreshold = newThreshold; + _manager->on_configure(config); } TEST_F(BouncerTest, reject_lower_prioritized_feed_messages_when_configured) { @@ -296,7 +312,7 @@ TEST_F(BouncerTest, abort_request_when_derived_bucket_space_node_state_is_marked auto state = makeClusterStateBundle("distributor:3 storage:3", {{ document::FixedBucketSpaces::default_space(), "distributor:3 storage:3 .2.s:d" }}); _node->getNodeStateUpdater().setClusterStateBundle(state); _upper->sendDown(createDummyFeedMessage(11 * 1000000, document::FixedBucketSpaces::default_space())); - expectMessageBouncedWithAbort(); + expect_message_bounced_with_node_down_abort(); EXPECT_EQ(1, _manager->metrics().unavailable_node_aborts.getValue()); _upper->reset(); @@ -362,5 +378,23 @@ TEST_F(BouncerTest, operation_with_sufficient_bucket_bits_is_not_rejected) { expectMessageNotBounced(); } +TEST_F(BouncerTest, requests_are_rejected_after_close) { + _manager->close(); + _upper->sendDown(createDummyFeedMessage(11 * 1000000, document::FixedBucketSpaces::default_space())); + expect_message_bounced_with_shutdown_abort(); +} + +TEST_F(BouncerTest, replies_are_swallowed_after_close) { + _manager->close(); + auto req = createDummyFeedMessage(11 * 1000000, document::FixedBucketSpaces::default_space()); + auto reply = req->makeReply(); + _upper->sendDown(std::move(reply)); + + EXPECT_EQ(0, _upper->getNumCommands()); + EXPECT_EQ(0, _upper->getNumReplies()); + EXPECT_EQ(0, _lower->getNumCommands()); + EXPECT_EQ(0, _lower->getNumReplies()); +} + } // storage diff --git a/storage/src/tests/storageserver/changedbucketownershiphandlertest.cpp b/storage/src/tests/storageserver/changedbucketownershiphandlertest.cpp index 639b3231028..50977b5ec8b 100644 --- a/storage/src/tests/storageserver/changedbucketownershiphandlertest.cpp +++ b/storage/src/tests/storageserver/changedbucketownershiphandlertest.cpp @@ -3,6 +3,7 @@ #include <tests/common/teststorageapp.h> #include <tests/common/testhelper.h> #include <tests/common/dummystoragelink.h> +#include <vespa/config/helper/configgetter.hpp> #include <vespa/document/base/testdocman.h> #include <vespa/storage/bucketdb/storbucketdb.h> #include <vespa/storage/persistence/messages.h> @@ -124,11 +125,12 @@ ChangedBucketOwnershipHandlerTest::insertBuckets(uint32_t numBuckets, void ChangedBucketOwnershipHandlerTest::SetUp() { + using vespa::config::content::PersistenceConfig; vdstestlib::DirConfig config(getStandardConfig(true)); _app.reset(new TestServiceLayerApp); _top.reset(new DummyStorageLink); - _handler = new ChangedBucketOwnershipHandler(config::ConfigUri(config.getConfigId()), + _handler = new ChangedBucketOwnershipHandler(*config_from<PersistenceConfig>(config::ConfigUri(config.getConfigId())), _app->getComponentRegister()); _top->push_back(std::unique_ptr<StorageLink>(_handler)); _bottom = new DummyStorageLink; @@ -139,7 +141,7 @@ ChangedBucketOwnershipHandlerTest::SetUp() auto pconfig = std::make_unique<vespa::config::content::PersistenceConfigBuilder>(); pconfig->abortOutdatedMutatingIdealStateOps = true; pconfig->abortOutdatedMutatingExternalLoadOps = true; - _handler->configure(std::move(pconfig)); + _handler->on_configure(*pconfig); } namespace { @@ -466,7 +468,7 @@ TEST_F(ChangedBucketOwnershipHandlerTest, abort_outdated_remove_location) { TEST_F(ChangedBucketOwnershipHandlerTest, ideal_state_aborts_are_configurable) { auto config = std::make_unique<vespa::config::content::PersistenceConfigBuilder>(); config->abortOutdatedMutatingIdealStateOps = false; - _handler->configure(std::move(config)); + _handler->on_configure(*config); // Should not abort operation, even when ownership has changed. expectChangeAbortsMessage<api::CreateBucketCommand>(false, getBucketToAbort()); } @@ -508,7 +510,7 @@ TEST_F(ChangedBucketOwnershipHandlerTest, external_load_op_abort_updates_metric) TEST_F(ChangedBucketOwnershipHandlerTest, external_load_op_aborts_are_configurable) { auto config = std::make_unique<vespa::config::content::PersistenceConfigBuilder>(); config->abortOutdatedMutatingExternalLoadOps = false; - _handler->configure(std::move(config)); + _handler->on_configure(*config); // Should not abort operation, even when ownership has changed. document::DocumentId docId("id:foo:testdoctype1::bar"); expectChangeAbortsMessage<api::RemoveCommand>( diff --git a/storage/src/tests/storageserver/communicationmanagertest.cpp b/storage/src/tests/storageserver/communicationmanagertest.cpp index e86c822e83c..04322562d08 100644 --- a/storage/src/tests/storageserver/communicationmanagertest.cpp +++ b/storage/src/tests/storageserver/communicationmanagertest.cpp @@ -1,31 +1,33 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/storage/storageserver/communicationmanager.h> - -#include <vespa/messagebus/testlib/slobrok.h> -#include <vespa/messagebus/rpcmessagebus.h> -#include <vespa/storageapi/message/persistence.h> -#include <vespa/storage/frameworkimpl/component/storagecomponentregisterimpl.h> -#include <vespa/storage/persistence/messages.h> -#include <vespa/document/bucket/fixed_bucket_spaces.h> -#include <tests/common/teststorageapp.h> #include <tests/common/dummystoragelink.h> #include <tests/common/testhelper.h> -#include <vespa/document/test/make_document_bucket.h> +#include <tests/common/teststorageapp.h> +#include <vespa/config/helper/configgetter.hpp> +#include <vespa/document/bucket/fixed_bucket_spaces.h> #include <vespa/document/fieldset/fieldsets.h> +#include <vespa/document/test/make_document_bucket.h> #include <vespa/documentapi/messagebus/messages/getdocumentmessage.h> -#include <vespa/vespalib/util/stringfmt.h> -#include <vespa/documentapi/messagebus/messages/removedocumentmessage.h> #include <vespa/documentapi/messagebus/messages/getdocumentreply.h> +#include <vespa/documentapi/messagebus/messages/removedocumentmessage.h> +#include <vespa/messagebus/rpcmessagebus.h> +#include <vespa/messagebus/testlib/slobrok.h> +#include <vespa/storage/frameworkimpl/component/storagecomponentregisterimpl.h> +#include <vespa/storage/persistence/messages.h> +#include <vespa/storage/storageserver/communicationmanager.h> +#include <vespa/storageapi/message/persistence.h> +#include <vespa/vespalib/util/stringfmt.h> #include <thread> -#include <vespa/vespalib/gtest/gtest.h> +#include <gtest/gtest.h> using document::test::makeDocumentBucket; using namespace ::testing; namespace storage { -vespalib::string _Storage("storage"); +vespalib::string _storage("storage"); + +using CommunicationManagerConfig = vespa::config::content::core::StorCommunicationmanagerConfig; struct CommunicationManagerTest : Test { @@ -33,13 +35,11 @@ struct CommunicationManagerTest : Test { void doTestConfigPropagation(bool isContentNode); - std::shared_ptr<api::StorageCommand> createDummyCommand( - api::StorageMessage::Priority priority) - { + static std::shared_ptr<api::StorageCommand> createDummyCommand(api::StorageMessage::Priority priority) { auto cmd = std::make_shared<api::GetCommand>(makeDocumentBucket(document::BucketId(0)), document::DocumentId("id:ns:mytype::mydoc"), document::AllFields::NAME); - cmd->setAddress(api::StorageMessageAddress::create(&_Storage, lib::NodeType::STORAGE, 1)); + cmd->setAddress(api::StorageMessageAddress::create(&_storage, lib::NodeType::STORAGE, 1)); cmd->setPriority(priority); return cmd; } @@ -77,19 +77,22 @@ TEST_F(CommunicationManagerTest, simple) { TestServiceLayerApp storNode(storConfig.getConfigId()); TestDistributorApp distNode(distConfig.getConfigId()); - CommunicationManager distributor(distNode.getComponentRegister(), - config::ConfigUri(distConfig.getConfigId())); - CommunicationManager storage(storNode.getComponentRegister(), - config::ConfigUri(storConfig.getConfigId())); - DummyStorageLink *distributorLink = new DummyStorageLink(); - DummyStorageLink *storageLink = new DummyStorageLink(); + auto dist_cfg_uri = config::ConfigUri(distConfig.getConfigId()); + auto stor_cfg_uri = config::ConfigUri(storConfig.getConfigId()); + + CommunicationManager distributor(distNode.getComponentRegister(), dist_cfg_uri, + *config_from<CommunicationManagerConfig>(dist_cfg_uri)); + CommunicationManager storage(storNode.getComponentRegister(), stor_cfg_uri, + *config_from<CommunicationManagerConfig>(stor_cfg_uri)); + auto* distributorLink = new DummyStorageLink(); + auto* storageLink = new DummyStorageLink(); distributor.push_back(std::unique_ptr<StorageLink>(distributorLink)); storage.push_back(std::unique_ptr<StorageLink>(storageLink)); distributor.open(); storage.open(); - auto stor_addr = api::StorageMessageAddress::create(&_Storage, lib::NodeType::STORAGE, 1); - auto distr_addr = api::StorageMessageAddress::create(&_Storage, lib::NodeType::DISTRIBUTOR, 1); + auto stor_addr = api::StorageMessageAddress::create(&_storage, lib::NodeType::STORAGE, 1); + auto distr_addr = api::StorageMessageAddress::create(&_storage, lib::NodeType::DISTRIBUTOR, 1); // It is undefined when the logical nodes will be visible in each others Slobrok // mirrors, so explicitly wait until mutual visibility is ensured. Failure to do this // might cause the below message to be immediately bounced due to failing to map the @@ -136,9 +139,10 @@ CommunicationManagerTest::doTestConfigPropagation(bool isContentNode) node = std::make_unique<TestDistributorApp>(config.getConfigId()); } - CommunicationManager commMgr(node->getComponentRegister(), - config::ConfigUri(config.getConfigId())); - DummyStorageLink *storageLink = new DummyStorageLink(); + auto cfg_uri = config::ConfigUri(config.getConfigId()); + CommunicationManager commMgr(node->getComponentRegister(), cfg_uri, + *config_from<CommunicationManagerConfig>(cfg_uri)); + auto* storageLink = new DummyStorageLink(); commMgr.push_back(std::unique_ptr<StorageLink>(storageLink)); commMgr.open(); @@ -153,13 +157,12 @@ CommunicationManagerTest::doTestConfigPropagation(bool isContentNode) } // Test live reconfig of limits. - using ConfigBuilder - = vespa::config::content::core::StorCommunicationmanagerConfigBuilder; + using ConfigBuilder = vespa::config::content::core::StorCommunicationmanagerConfigBuilder; auto liveCfg = std::make_unique<ConfigBuilder>(); liveCfg->mbusContentNodeMaxPendingCount = 777777; liveCfg->mbusDistributorNodeMaxPendingCount = 999999; - commMgr.configure(std::move(liveCfg)); + commMgr.on_configure(*liveCfg); if (isContentNode) { EXPECT_EQ(777777, mbus.getMaxPendingCount()); } else { @@ -182,9 +185,10 @@ TEST_F(CommunicationManagerTest, commands_are_dequeued_in_fifo_order) { addSlobrokConfig(storConfig, slobrok); TestServiceLayerApp storNode(storConfig.getConfigId()); - CommunicationManager storage(storNode.getComponentRegister(), - config::ConfigUri(storConfig.getConfigId())); - DummyStorageLink *storageLink = new DummyStorageLink(); + auto cfg_uri = config::ConfigUri(storConfig.getConfigId()); + CommunicationManager storage(storNode.getComponentRegister(), cfg_uri, + *config_from<CommunicationManagerConfig>(cfg_uri)); + auto* storageLink = new DummyStorageLink(); storage.push_back(std::unique_ptr<StorageLink>(storageLink)); storage.open(); @@ -215,9 +219,10 @@ TEST_F(CommunicationManagerTest, replies_are_dequeued_in_fifo_order) { addSlobrokConfig(storConfig, slobrok); TestServiceLayerApp storNode(storConfig.getConfigId()); - CommunicationManager storage(storNode.getComponentRegister(), - config::ConfigUri(storConfig.getConfigId())); - DummyStorageLink *storageLink = new DummyStorageLink(); + auto cfg_uri = config::ConfigUri(storConfig.getConfigId()); + CommunicationManager storage(storNode.getComponentRegister(), cfg_uri, + *config_from<CommunicationManagerConfig>(cfg_uri)); + auto* storageLink = new DummyStorageLink(); storage.push_back(std::unique_ptr<StorageLink>(storageLink)); storage.open(); @@ -256,8 +261,9 @@ struct CommunicationManagerFixture { addSlobrokConfig(stor_config, slobrok); node = std::make_unique<TestServiceLayerApp>(stor_config.getConfigId()); - comm_mgr = std::make_unique<CommunicationManager>(node->getComponentRegister(), - config::ConfigUri(stor_config.getConfigId())); + auto cfg_uri = config::ConfigUri(stor_config.getConfigId()); + comm_mgr = std::make_unique<CommunicationManager>(node->getComponentRegister(), cfg_uri, + *config_from<CommunicationManagerConfig>(cfg_uri)); bottom_link = new DummyStorageLink(); comm_mgr->push_back(std::unique_ptr<StorageLink>(bottom_link)); comm_mgr->open(); diff --git a/storage/src/tests/storageserver/documentapiconvertertest.cpp b/storage/src/tests/storageserver/documentapiconvertertest.cpp index 5e70c00cc5f..eb4789b25d4 100644 --- a/storage/src/tests/storageserver/documentapiconvertertest.cpp +++ b/storage/src/tests/storageserver/documentapiconvertertest.cpp @@ -77,7 +77,7 @@ struct DocumentApiConverterTest : Test { } void SetUp() override { - _converter = std::make_unique<DocumentApiConverter>(config::ConfigUri("raw:"), _bucketResolver); + _converter = std::make_unique<DocumentApiConverter>(_bucketResolver); }; template <typename DerivedT, typename BaseT> diff --git a/storage/src/tests/storageserver/mergethrottlertest.cpp b/storage/src/tests/storageserver/mergethrottlertest.cpp index 8e87e07eeff..7a7f2551c2d 100644 --- a/storage/src/tests/storageserver/mergethrottlertest.cpp +++ b/storage/src/tests/storageserver/mergethrottlertest.cpp @@ -2,6 +2,7 @@ #include <tests/common/testhelper.h> #include <tests/common/dummystoragelink.h> #include <tests/common/teststorageapp.h> +#include <vespa/config/helper/configgetter.hpp> #include <vespa/document/test/make_document_bucket.h> #include <vespa/messagebus/dynamicthrottlepolicy.h> #include <vespa/storage/storageserver/mergethrottler.h> @@ -28,7 +29,9 @@ namespace storage { namespace { -vespalib::string _Storage("storage"); +using StorServerConfig = vespa::config::content::core::StorServerConfig; + +vespalib::string _storage("storage"); struct MergeBuilder { document::BucketId _bucket; @@ -108,7 +111,7 @@ struct MergeBuilder { auto cmd = std::make_shared<MergeBucketCommand>( makeDocumentBucket(_bucket), n, _maxTimestamp, _clusterStateVersion, _chain); - cmd->setAddress(StorageMessageAddress::create(&_Storage, lib::NodeType::STORAGE, _nodes[0])); + cmd->setAddress(StorageMessageAddress::create(&_storage, lib::NodeType::STORAGE, _nodes[0])); return cmd; } }; @@ -167,7 +170,9 @@ MergeThrottlerTest::~MergeThrottlerTest() = default; void MergeThrottlerTest::SetUp() { - vdstestlib::DirConfig config(getStandardConfig(true)); + vdstestlib::DirConfig dir_config(getStandardConfig(true)); + auto cfg_uri = ::config::ConfigUri(dir_config.getConfigId()); + auto config = config_from<StorServerConfig>(cfg_uri); for (int i = 0; i < _storageNodeCount; ++i) { auto server = std::make_unique<TestServiceLayerApp>(NodeIndex(i)); @@ -175,7 +180,7 @@ MergeThrottlerTest::SetUp() std::unique_ptr<DummyStorageLink> top; top = std::make_unique<DummyStorageLink>(); - MergeThrottler* throttler = new MergeThrottler(::config::ConfigUri(config.getConfigId()), server->getComponentRegister()); + MergeThrottler* throttler = new MergeThrottler(*config, server->getComponentRegister()); // MergeThrottler will be sandwiched in between two dummy links top->push_back(std::unique_ptr<StorageLink>(throttler)); DummyStorageLink* bottom = new DummyStorageLink; @@ -283,7 +288,7 @@ TEST_F(MergeThrottlerTest, chain) { auto cmd = std::make_shared<MergeBucketCommand>(bucket, nodes, UINT_MAX, 123); cmd->setPriority(7); cmd->setTimeout(54321ms); - cmd->setAddress(StorageMessageAddress::create(&_Storage, lib::NodeType::STORAGE, 0)); + cmd->setAddress(StorageMessageAddress::create(&_storage, lib::NodeType::STORAGE, 0)); const uint16_t distributorIndex = 123; cmd->setSourceIndex(distributorIndex); // Dummy distributor index that must be forwarded @@ -423,7 +428,7 @@ TEST_F(MergeThrottlerTest, with_source_only_node) { std::vector<MergeBucketCommand::Node> nodes({{0}, {2}, {1, true}}); auto cmd = std::make_shared<MergeBucketCommand>(makeDocumentBucket(bid), nodes, UINT_MAX, 123); - cmd->setAddress(StorageMessageAddress::create(&_Storage, lib::NodeType::STORAGE, 0)); + cmd->setAddress(StorageMessageAddress::create(&_storage, lib::NodeType::STORAGE, 0)); _topLinks[0]->sendDown(cmd); _topLinks[0]->waitForMessage(MessageType::MERGEBUCKET, _messageWaitTime); @@ -468,7 +473,7 @@ TEST_F(MergeThrottlerTest, legacy_42_distributor_behavior) { auto cmd = std::make_shared<MergeBucketCommand>(makeDocumentBucket(bid), nodes, 1234); // Send to node 1, which is not the lowest index - cmd->setAddress(StorageMessageAddress::create(&_Storage, lib::NodeType::STORAGE, 1)); + cmd->setAddress(StorageMessageAddress::create(&_storage, lib::NodeType::STORAGE, 1)); _topLinks[1]->sendDown(cmd); _bottomLinks[1]->waitForMessage(MessageType::MERGEBUCKET, _messageWaitTime); @@ -503,7 +508,7 @@ TEST_F(MergeThrottlerTest, legacy_42_distributor_behavior_does_not_take_ownershi auto cmd = std::make_shared<MergeBucketCommand>(makeDocumentBucket(bid), nodes, 1234); // Send to node 1, which is not the lowest index - cmd->setAddress(StorageMessageAddress::create(&_Storage, lib::NodeType::STORAGE, 1)); + cmd->setAddress(StorageMessageAddress::create(&_storage, lib::NodeType::STORAGE, 1)); _topLinks[1]->sendDown(cmd); _bottomLinks[1]->waitForMessage(MessageType::MERGEBUCKET, _messageWaitTime); @@ -550,7 +555,7 @@ TEST_F(MergeThrottlerTest, end_of_chain_execution_does_not_take_ownership) { auto cmd = std::make_shared<MergeBucketCommand>(makeDocumentBucket(bid), nodes, 1234, 1, chain); // Send to last node, which is not the lowest index - cmd->setAddress(StorageMessageAddress::create(&_Storage, lib::NodeType::STORAGE, 3)); + cmd->setAddress(StorageMessageAddress::create(&_storage, lib::NodeType::STORAGE, 3)); _topLinks[2]->sendDown(cmd); _bottomLinks[2]->waitForMessage(MessageType::MERGEBUCKET, _messageWaitTime); @@ -595,7 +600,7 @@ TEST_F(MergeThrottlerTest, resend_handling) { std::vector<MergeBucketCommand::Node> nodes({{0}, {1}, {2}}); auto cmd = std::make_shared<MergeBucketCommand>(makeDocumentBucket(bid), nodes, 1234); - cmd->setAddress(StorageMessageAddress::create(&_Storage, lib::NodeType::STORAGE, 1)); + cmd->setAddress(StorageMessageAddress::create(&_storage, lib::NodeType::STORAGE, 1)); _topLinks[0]->sendDown(cmd); _topLinks[0]->waitForMessage(MessageType::MERGEBUCKET, _messageWaitTime); @@ -962,7 +967,7 @@ TEST_F(MergeThrottlerTest, unseen_merge_with_node_in_chain) { makeDocumentBucket(BucketId(32, 0xdeadbeef)), nodes, 1234, 1, chain); - cmd->setAddress(StorageMessageAddress::create(&_Storage, lib::NodeType::STORAGE, 9)); + cmd->setAddress(StorageMessageAddress::create(&_storage, lib::NodeType::STORAGE, 9)); _topLinks[0]->sendDown(cmd); // First, test that we get rejected when processing merge immediately @@ -1145,7 +1150,7 @@ TEST_F(MergeThrottlerTest, unknown_merge_with_self_in_chain) { std::vector<uint16_t> chain({0}); auto cmd = std::make_shared<MergeBucketCommand>(makeDocumentBucket(bid), nodes, 1234, 1, chain); - cmd->setAddress(StorageMessageAddress::create(&_Storage, lib::NodeType::STORAGE, 1)); + cmd->setAddress(StorageMessageAddress::create(&_storage, lib::NodeType::STORAGE, 1)); _topLinks[0]->sendDown(cmd); _topLinks[0]->waitForMessage(MessageType::MERGEBUCKET_REPLY, _messageWaitTime); diff --git a/storage/src/tests/storageserver/priorityconvertertest.cpp b/storage/src/tests/storageserver/priorityconvertertest.cpp index 5462c83d2a2..69f9d313242 100644 --- a/storage/src/tests/storageserver/priorityconvertertest.cpp +++ b/storage/src/tests/storageserver/priorityconvertertest.cpp @@ -1,7 +1,6 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/storage/storageserver/priorityconverter.h> -#include <tests/common/testhelper.h> #include <vespa/vespalib/gtest/gtest.h> using namespace ::testing; @@ -12,8 +11,7 @@ struct PriorityConverterTest : Test { std::unique_ptr<PriorityConverter> _converter; void SetUp() override { - vdstestlib::DirConfig config(getStandardConfig(true)); - _converter = std::make_unique<PriorityConverter>(config::ConfigUri(config.getConfigId())); + _converter = std::make_unique<PriorityConverter>(); }; }; diff --git a/storage/src/tests/storageserver/service_layer_error_listener_test.cpp b/storage/src/tests/storageserver/service_layer_error_listener_test.cpp index 84a98385962..edb13eea5af 100644 --- a/storage/src/tests/storageserver/service_layer_error_listener_test.cpp +++ b/storage/src/tests/storageserver/service_layer_error_listener_test.cpp @@ -1,11 +1,12 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/storage/storageserver/service_layer_error_listener.h> +#include <tests/common/testhelper.h> +#include <tests/common/teststorageapp.h> +#include <vespa/config/helper/configgetter.hpp> #include <vespa/storage/storageserver/mergethrottler.h> +#include <vespa/storage/storageserver/service_layer_error_listener.h> #include <vespa/storageframework/defaultimplementation/component/componentregisterimpl.h> #include <vespa/vdstestlib/config/dirconfig.h> -#include <tests/common/testhelper.h> -#include <tests/common/teststorageapp.h> #include <vespa/vespalib/gtest/gtest.h> using namespace ::testing; @@ -34,10 +35,12 @@ private: }; struct Fixture { + using StorServerConfig = vespa::config::content::core::StorServerConfig; + vdstestlib::DirConfig config{getStandardConfig(true)}; TestServiceLayerApp app; ServiceLayerComponent component{app.getComponentRegister(), "dummy"}; - MergeThrottler merge_throttler{config::ConfigUri(config.getConfigId()), app.getComponentRegister()}; + MergeThrottler merge_throttler{*config_from<StorServerConfig>(config::ConfigUri(config.getConfigId())), app.getComponentRegister()}; TestShutdownListener shutdown_listener; ServiceLayerErrorListener error_listener{component, merge_throttler}; diff --git a/storage/src/tests/storageserver/testvisitormessagesession.h b/storage/src/tests/storageserver/testvisitormessagesession.h index 86d4dc92a58..cc7dab7ef9e 100644 --- a/storage/src/tests/storageserver/testvisitormessagesession.h +++ b/storage/src/tests/storageserver/testvisitormessagesession.h @@ -49,9 +49,11 @@ struct TestVisitorMessageSessionFactory : public VisitorMessageSessionFactory bool _createAutoReplyVisitorSessions; PriorityConverter _priConverter; - TestVisitorMessageSessionFactory(vespalib::stringref configId = "") + TestVisitorMessageSessionFactory() : _createAutoReplyVisitorSessions(false), - _priConverter(config::ConfigUri(configId)) {} + _priConverter() + { + } VisitorMessageSession::UP createSession(Visitor& v, VisitorThread& vt) override { std::lock_guard lock(_accessLock); diff --git a/storage/src/tests/visiting/visitormanagertest.cpp b/storage/src/tests/visiting/visitormanagertest.cpp index 991b98e5489..5fa6d4a77d8 100644 --- a/storage/src/tests/visiting/visitormanagertest.cpp +++ b/storage/src/tests/visiting/visitormanagertest.cpp @@ -1,5 +1,6 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/config/helper/configgetter.hpp> #include <vespa/document/fieldvalue/intfieldvalue.h> #include <vespa/document/fieldvalue/stringfieldvalue.h> #include <vespa/storageapi/message/datagram.h> @@ -36,8 +37,8 @@ namespace storage { namespace { using msg_ptr_vector = std::vector<api::StorageMessage::SP>; -vespalib::string _Storage("storage"); -api::StorageMessageAddress _Address(&_Storage, lib::NodeType::STORAGE, 0); +vespalib::string _storage("storage"); +api::StorageMessageAddress _address(&_storage, lib::NodeType::STORAGE, 0); } struct VisitorManagerTest : Test { @@ -83,19 +84,23 @@ VisitorManagerTest::initializeTest(bool defer_manager_thread_start) vdstestlib::DirConfig config(getStandardConfig(true)); config.getConfig("stor-visitor").set("visitorthreads", "1"); - _messageSessionFactory = std::make_unique<TestVisitorMessageSessionFactory>(config.getConfigId()); + _messageSessionFactory = std::make_unique<TestVisitorMessageSessionFactory>(); _node = std::make_unique<TestServiceLayerApp>(config.getConfigId()); _node->setupDummyPersistence(); _node->getStateUpdater().setClusterState(std::make_shared<lib::ClusterState>("storage:1 distributor:1")); _top = std::make_unique<DummyStorageLink>(); - auto vm = std::make_unique<VisitorManager>(config::ConfigUri(config.getConfigId()), + using vespa::config::content::core::StorVisitorConfig; + auto bootstrap_cfg = config_from<StorVisitorConfig>(config::ConfigUri(config.getConfigId())); + auto vm = std::make_unique<VisitorManager>(*bootstrap_cfg, _node->getComponentRegister(), *_messageSessionFactory, VisitorFactory::Map(), defer_manager_thread_start); _manager = vm.get(); _top->push_back(std::move(vm)); - _top->push_back(std::make_unique<FileStorManager>(config::ConfigUri(config.getConfigId()), _node->getPersistenceProvider(), + using vespa::config::content::StorFilestorConfig; + auto filestor_cfg = config_from<StorFilestorConfig>(config::ConfigUri(config.getConfigId())); + _top->push_back(std::make_unique<FileStorManager>(*filestor_cfg, _node->getPersistenceProvider(), _node->getComponentRegister(), *_node, _node->get_host_info())); _manager->setTimeBetweenTicks(10); _top->open(); @@ -151,7 +156,7 @@ VisitorManagerTest::initializeTest(bool defer_manager_thread_start) document::BucketId bid(16, i); auto cmd = std::make_shared<api::CreateBucketCommand>(makeDocumentBucket(bid)); - cmd->setAddress(_Address); + cmd->setAddress(_address); cmd->setSourceIndex(0); _top->sendDown(cmd); _top->waitForMessages(1, 60); @@ -166,7 +171,7 @@ VisitorManagerTest::initializeTest(bool defer_manager_thread_start) document::BucketId bid(16, i); auto cmd = std::make_shared<api::PutCommand>(makeDocumentBucket(bid), _documents[i], i+1); - cmd->setAddress(_Address); + cmd->setAddress(_address); _top->sendDown(cmd); _top->waitForMessages(1, 60); const msg_ptr_vector replies = _top->getRepliesOnce(); @@ -186,7 +191,7 @@ VisitorManagerTest::addSomeRemoves(bool removeAll) document::BucketId bid(16, i % 10); auto cmd = std::make_shared<api::RemoveCommand>(makeDocumentBucket(bid), _documents[i]->getId(), vespalib::count_us(clock.getSystemTime().time_since_epoch()) + docCount + i + 1); - cmd->setAddress(_Address); + cmd->setAddress(_address); _top->sendDown(cmd); _top->waitForMessages(1, 60); const msg_ptr_vector replies = _top->getRepliesOnce(); @@ -340,7 +345,7 @@ TEST_F(VisitorManagerTest, normal_usage) { ASSERT_NO_FATAL_FAILURE(initializeTest()); auto cmd = std::make_shared<api::CreateVisitorCommand>(makeBucketSpace(), "DumpVisitor", "testvis", ""); cmd->addBucketToBeVisited(document::BucketId(16, 3)); - cmd->setAddress(_Address); + cmd->setAddress(_address); cmd->setControlDestination("foo/bar"); _top->sendDown(cmd); std::vector<document::Document::SP > docs; @@ -360,7 +365,7 @@ TEST_F(VisitorManagerTest, resending) { ASSERT_NO_FATAL_FAILURE(initializeTest()); auto cmd = std::make_shared<api::CreateVisitorCommand>(makeBucketSpace(), "DumpVisitor", "testvis", ""); cmd->addBucketToBeVisited(document::BucketId(16, 3)); - cmd->setAddress(_Address); + cmd->setAddress(_address); cmd->setControlDestination("foo/bar"); _top->sendDown(cmd); std::vector<document::Document::SP > docs; @@ -407,7 +412,7 @@ TEST_F(VisitorManagerTest, visit_empty_bucket) { auto cmd = std::make_shared<api::CreateVisitorCommand>(makeBucketSpace(), "DumpVisitor", "testvis", ""); cmd->addBucketToBeVisited(document::BucketId(16, 3)); - cmd->setAddress(_Address); + cmd->setAddress(_address); _top->sendDown(cmd); // All data has been replied to, expecting to get a create visitor reply @@ -420,7 +425,7 @@ TEST_F(VisitorManagerTest, multi_bucket_visit) { for (uint32_t i=0; i<10; ++i) { cmd->addBucketToBeVisited(document::BucketId(16, i)); } - cmd->setAddress(_Address); + cmd->setAddress(_address); cmd->setDataDestination("fooclient.0"); _top->sendDown(cmd); std::vector<document::Document::SP> docs; @@ -439,7 +444,7 @@ TEST_F(VisitorManagerTest, no_buckets) { ASSERT_NO_FATAL_FAILURE(initializeTest()); auto cmd = std::make_shared<api::CreateVisitorCommand>(makeBucketSpace(), "DumpVisitor", "testvis", ""); - cmd->setAddress(_Address); + cmd->setAddress(_address); _top->sendDown(cmd); // Should get one reply; a CreateVisitorReply with error since no @@ -458,7 +463,7 @@ TEST_F(VisitorManagerTest, visit_puts_and_removes) { ASSERT_NO_FATAL_FAILURE(initializeTest()); addSomeRemoves(); auto cmd = std::make_shared<api::CreateVisitorCommand>(makeBucketSpace(), "DumpVisitor", "testvis", ""); - cmd->setAddress(_Address); + cmd->setAddress(_address); cmd->setVisitRemoves(); for (uint32_t i=0; i<10; ++i) { cmd->addBucketToBeVisited(document::BucketId(16, i)); @@ -486,7 +491,7 @@ TEST_F(VisitorManagerTest, visit_with_timeframe_and_selection) { for (uint32_t i=0; i<10; ++i) { cmd->addBucketToBeVisited(document::BucketId(16, i)); } - cmd->setAddress(_Address); + cmd->setAddress(_address); _top->sendDown(cmd); std::vector<document::Document::SP> docs; std::vector<document::DocumentId> docIds; @@ -515,7 +520,7 @@ TEST_F(VisitorManagerTest, visit_with_timeframe_and_bogus_selection) { for (uint32_t i=0; i<10; ++i) { cmd->addBucketToBeVisited(document::BucketId(16, i)); } - cmd->setAddress(_Address); + cmd->setAddress(_address); _top->sendDown(cmd); _top->waitForMessages(1, 60); @@ -551,7 +556,7 @@ TEST_F(VisitorManagerTest, visitor_callbacks) { auto cmd = std::make_shared<api::CreateVisitorCommand>(makeBucketSpace(), "TestVisitor", "testvis", ""); cmd->addBucketToBeVisited(document::BucketId(16, 3)); cmd->addBucketToBeVisited(document::BucketId(16, 5)); - cmd->setAddress(_Address); + cmd->setAddress(_address); _top->sendDown(cmd); // Wait until we have started the visitor @@ -595,7 +600,7 @@ TEST_F(VisitorManagerTest, visitor_cleanup) { ost << "testvis" << i; auto cmd = std::make_shared<api::CreateVisitorCommand>(makeBucketSpace(), "InvalidVisitor", ost.str(), ""); cmd->addBucketToBeVisited(document::BucketId(16, 3)); - cmd->setAddress(_Address); + cmd->setAddress(_address); cmd->setQueueTimeout(0ms); _top->sendDown(cmd); _top->waitForMessages(i+1, 60); @@ -607,7 +612,7 @@ TEST_F(VisitorManagerTest, visitor_cleanup) { ost << "testvis" << (i + 10); auto cmd = std::make_shared<api::CreateVisitorCommand>(makeBucketSpace(), "DumpVisitor", ost.str(), ""); cmd->addBucketToBeVisited(document::BucketId(16, 3)); - cmd->setAddress(_Address); + cmd->setAddress(_address); cmd->setQueueTimeout(0ms); _top->sendDown(cmd); } @@ -676,7 +681,7 @@ TEST_F(VisitorManagerTest, visitor_cleanup) { ost << "testvis" << (i + 24); auto cmd = std::make_shared<api::CreateVisitorCommand>(makeBucketSpace(), "DumpVisitor", ost.str(), ""); cmd->addBucketToBeVisited(document::BucketId(16, 3)); - cmd->setAddress(_Address); + cmd->setAddress(_address); cmd->setQueueTimeout(0ms); _top->sendDown(cmd); } @@ -706,7 +711,7 @@ TEST_F(VisitorManagerTest, abort_on_failed_visitor_info) { { auto cmd = std::make_shared<api::CreateVisitorCommand>(makeBucketSpace(), "DumpVisitor", "testvis", ""); cmd->addBucketToBeVisited(document::BucketId(16, 3)); - cmd->setAddress(_Address); + cmd->setAddress(_address); cmd->setQueueTimeout(0ms); _top->sendDown(cmd); } @@ -740,7 +745,7 @@ TEST_F(VisitorManagerTest, abort_on_field_path_error) { auto cmd = std::make_shared<api::CreateVisitorCommand>( makeBucketSpace(), "DumpVisitor", "testvis", "testdoctype1.headerval{bogus} == 1234"); cmd->addBucketToBeVisited(document::BucketId(16, 3)); - cmd->setAddress(_Address); + cmd->setAddress(_address); cmd->setQueueTimeout(0ms); _top->sendDown(cmd); @@ -754,7 +759,7 @@ TEST_F(VisitorManagerTest, visitor_queue_timeout) { { auto cmd = std::make_shared<api::CreateVisitorCommand>(makeBucketSpace(), "DumpVisitor", "testvis", ""); cmd->addBucketToBeVisited(document::BucketId(16, 3)); - cmd->setAddress(_Address); + cmd->setAddress(_address); cmd->setQueueTimeout(1ms); cmd->setTimeout(100 * 1000 * 1000ms); // The manager thread isn't running yet so the visitor stays on the queue @@ -780,7 +785,7 @@ TEST_F(VisitorManagerTest, visitor_processing_timeout) { auto cmd = std::make_shared<api::CreateVisitorCommand>(makeBucketSpace(), "DumpVisitor", "testvis", ""); cmd->addBucketToBeVisited(document::BucketId(16, 3)); - cmd->setAddress(_Address); + cmd->setAddress(_address); cmd->setQueueTimeout(0ms); cmd->setTimeout(100ms); _top->sendDown(cmd); @@ -804,7 +809,7 @@ sendCreateVisitor(vespalib::duration timeout, DummyStorageLink& top, uint8_t pri ost << "testvis" << ++nextVisitor; auto cmd = std::make_shared<api::CreateVisitorCommand>(makeBucketSpace(), "DumpVisitor", ost.str(), ""); cmd->addBucketToBeVisited(document::BucketId(16, 3)); - cmd->setAddress(_Address); + cmd->setAddress(_address); cmd->setQueueTimeout(timeout); cmd->setPriority(priority); top.sendDown(cmd); diff --git a/storage/src/tests/visiting/visitortest.cpp b/storage/src/tests/visiting/visitortest.cpp index 1f1d27ab4cb..f83b6c99d64 100644 --- a/storage/src/tests/visiting/visitortest.cpp +++ b/storage/src/tests/visiting/visitortest.cpp @@ -1,6 +1,7 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/config/common/exceptions.h> +#include <vespa/config/helper/configgetter.hpp> #include <vespa/document/fieldvalue/intfieldvalue.h> #include <vespa/document/fieldvalue/stringfieldvalue.h> #include <vespa/document/datatype/documenttype.h> @@ -161,16 +162,17 @@ VisitorTest::initializeTest(const TestParams& params) std::filesystem::create_directories(std::filesystem::path(vespalib::make_string("%s/disks/d0", rootFolder.c_str()))); std::filesystem::create_directories(std::filesystem::path(vespalib::make_string("%s/disks/d1", rootFolder.c_str()))); - _messageSessionFactory = std::make_unique<TestVisitorMessageSessionFactory>(config.getConfigId()); + _messageSessionFactory = std::make_unique<TestVisitorMessageSessionFactory>(); if (params._autoReplyError.getCode() != mbus::ErrorCode::NONE) { _messageSessionFactory->_autoReplyError = params._autoReplyError; _messageSessionFactory->_createAutoReplyVisitorSessions = true; } _node = std::make_unique<TestServiceLayerApp>(config.getConfigId()); _top = std::make_unique<DummyStorageLink>(); + using vespa::config::content::core::StorVisitorConfig; + auto bootstrap_cfg = config_from<StorVisitorConfig>(config::ConfigUri(config.getConfigId())); _top->push_back(std::unique_ptr<StorageLink>(_manager - = new VisitorManager(config::ConfigUri(config.getConfigId()), - _node->getComponentRegister(), *_messageSessionFactory))); + = new VisitorManager(*bootstrap_cfg, _node->getComponentRegister(), *_messageSessionFactory))); _bottom = new DummyStorageLink(); _top->push_back(std::unique_ptr<StorageLink>(_bottom)); _manager->setTimeBetweenTicks(10); diff --git a/storage/src/vespa/storage/bucketdb/bucketmanager.cpp b/storage/src/vespa/storage/bucketdb/bucketmanager.cpp index 8c24a7dfd4e..d12a9f72ac1 100644 --- a/storage/src/vespa/storage/bucketdb/bucketmanager.cpp +++ b/storage/src/vespa/storage/bucketdb/bucketmanager.cpp @@ -33,10 +33,9 @@ using namespace std::chrono_literals; namespace storage { -BucketManager::BucketManager(const config::ConfigUri & configUri, ServiceLayerComponentRegister& compReg) +BucketManager::BucketManager(const StorServerConfig& bootstrap_config, ServiceLayerComponentRegister& compReg) : StorageLink("Bucket manager"), framework::StatusReporter("bucketdb", "Bucket database"), - _configUri(configUri), _workerLock(), _workerCond(), _clusterStateLock(), @@ -60,8 +59,7 @@ BucketManager::BucketManager(const config::ConfigUri & configUri, ServiceLayerCo ns.setMinUsedBits(58); _component.getStateUpdater().setReportedNodeState(ns); - auto server_config = config::ConfigGetter<vespa::config::content::core::StorServerConfig>::getConfig(configUri.getConfigId(), configUri.getContext()); - _simulated_processing_delay = std::chrono::milliseconds(std::max(0, server_config->simulatedBucketRequestLatencyMsec)); + _simulated_processing_delay = std::chrono::milliseconds(std::max(0, bootstrap_config.simulatedBucketRequestLatencyMsec)); } BucketManager::~BucketManager() @@ -414,9 +412,7 @@ BucketManager::dump(std::ostream& out) const void BucketManager::onOpen() { - if (!_configUri.empty()) { - startWorkerThread(); - } + startWorkerThread(); } void BucketManager::startWorkerThread() diff --git a/storage/src/vespa/storage/bucketdb/bucketmanager.h b/storage/src/vespa/storage/bucketdb/bucketmanager.h index 76d9123a519..35ccab843a9 100644 --- a/storage/src/vespa/storage/bucketdb/bucketmanager.h +++ b/storage/src/vespa/storage/bucketdb/bucketmanager.h @@ -10,10 +10,10 @@ #include "bucketmanagermetrics.h" #include "storbucketdb.h" #include <vespa/config/subscription/configuri.h> -#include <vespa/storage/bucketdb/config-stor-bucketdb.h> #include <vespa/storage/common/servicelayercomponent.h> #include <vespa/storage/common/storagelinkqueued.h> #include <vespa/storage/common/nodestateupdater.h> +#include <vespa/storage/config/config-stor-server.h> #include <vespa/storageapi/message/bucket.h> #include <vespa/storageframework/generic/metric/metricupdatehook.h> #include <vespa/storageframework/generic/status/statusreporter.h> @@ -34,6 +34,7 @@ class BucketManager : public StorageLink, private framework::MetricUpdateHook { public: + using StorServerConfig = vespa::config::content::core::StorServerConfig; /** Type used for message queues */ using BucketInfoRequestList = std::list<std::shared_ptr<api::RequestBucketInfoCommand>>; using BucketInfoRequestMap = std::unordered_map<document::BucketSpace, BucketInfoRequestList, document::BucketSpace::hash>; @@ -41,7 +42,6 @@ public: private: using ReplyQueue = std::vector<api::StorageReply::SP>; using ConflictingBuckets = std::unordered_set<document::BucketId, document::BucketId::hash>; - config::ConfigUri _configUri; BucketInfoRequestMap _bucketInfoRequests; /** @@ -82,7 +82,7 @@ private: }; public: - BucketManager(const config::ConfigUri&, ServiceLayerComponentRegister&); + BucketManager(const StorServerConfig& bootstrap_config, ServiceLayerComponentRegister&); BucketManager(const BucketManager&) = delete; BucketManager& operator=(const BucketManager&) = delete; ~BucketManager() override; diff --git a/storage/src/vespa/storage/common/storagelink.cpp b/storage/src/vespa/storage/common/storagelink.cpp index beccd605650..ec55bc89e90 100644 --- a/storage/src/vespa/storage/common/storagelink.cpp +++ b/storage/src/vespa/storage/common/storagelink.cpp @@ -14,6 +14,23 @@ using namespace storage::api; namespace storage { +StorageLink::StorageLink(const std::string& name, + MsgDownOnFlush allow_msg_down_during_flushing, + MsgUpOnClosed allow_msg_up_during_closed) + : _name(name), + _up(nullptr), + _down(), + _state(CREATED), + _msg_down_during_flushing(allow_msg_down_during_flushing), + _msg_up_during_closed(allow_msg_up_during_closed) +{ +} + +StorageLink::StorageLink(const std::string& name) + : StorageLink(name, MsgDownOnFlush::Disallowed, MsgUpOnClosed::Disallowed) +{ +} + StorageLink::~StorageLink() { LOG(debug, "Destructing link %s.", toString().c_str()); } @@ -129,9 +146,15 @@ void StorageLink::sendDown(const StorageMessage::SP& msg) case CLOSING: case FLUSHINGDOWN: break; + case FLUSHINGUP: + if (_msg_down_during_flushing == MsgDownOnFlush::Allowed) { + break; + } + [[fallthrough]]; default: - LOG(error, "Link %s trying to send %s down while in state %s", - toString().c_str(), msg->toString().c_str(), stateToString(getState())); + LOG(error, "Link %s trying to send %s down while in state %s. Stacktrace: %s", + toString().c_str(), msg->toString().c_str(), stateToString(getState()), + vespalib::getStackTrace(0).c_str()); assert(false); } assert(msg); @@ -171,9 +194,15 @@ void StorageLink::sendUp(const std::shared_ptr<StorageMessage> & msg) case FLUSHINGDOWN: case FLUSHINGUP: break; + case CLOSED: + if (_msg_up_during_closed == MsgUpOnClosed::Allowed) { + break; + } + [[fallthrough]]; default: - LOG(error, "Link %s trying to send %s up while in state %s", - toString().c_str(), msg->toString(true).c_str(), stateToString(getState())); + LOG(error, "Link %s trying to send %s up while in state %s. Stacktrace: %s", + toString().c_str(), msg->toString(true).c_str(), stateToString(getState()), + vespalib::getStackTrace(0).c_str()); assert(false); } assert(msg); @@ -281,15 +310,14 @@ Queue::getNext(std::shared_ptr<api::StorageMessage>& msg, vespalib::duration tim void Queue::enqueue(std::shared_ptr<api::StorageMessage> msg) { - { - std::lock_guard sync(_lock); - _queue.emplace(std::move(msg)); - } + std::lock_guard sync(_lock); + _queue.emplace(std::move(msg)); _cond.notify_one(); } void Queue::signal() { + std::lock_guard sync(_lock); _cond.notify_one(); } diff --git a/storage/src/vespa/storage/common/storagelink.h b/storage/src/vespa/storage/common/storagelink.h index 2b470d029d8..3ff75df9448 100644 --- a/storage/src/vespa/storage/common/storagelink.h +++ b/storage/src/vespa/storage/common/storagelink.h @@ -41,29 +41,41 @@ public: enum State { CREATED, OPENED, CLOSING, FLUSHINGDOWN, FLUSHINGUP, CLOSED }; + enum class MsgDownOnFlush { Allowed, Disallowed }; + enum class MsgUpOnClosed { Allowed, Disallowed }; + private: - std::string _name; - StorageLink* _up; + const std::string _name; + StorageLink* _up; std::unique_ptr<StorageLink> _down; - std::atomic<State> _state; + std::atomic<State> _state; + const MsgDownOnFlush _msg_down_during_flushing; + const MsgUpOnClosed _msg_up_during_closed; public: + StorageLink(const std::string& name, + MsgDownOnFlush allow_msg_down_during_flushing, + MsgUpOnClosed allow_msg_up_during_closed); + explicit StorageLink(const std::string& name); + StorageLink(const StorageLink &) = delete; StorageLink & operator = (const StorageLink &) = delete; - StorageLink(const std::string& name) - : _name(name), _up(0), _down(), _state(CREATED) {} ~StorageLink() override; - const std::string& getName() const { return _name; } - bool isTop() const { return (_up == 0); } - bool isBottom() const { return (_down.get() == 0); } - unsigned int size() const { return (isBottom() ? 1 : _down->size() + 1); } + const std::string& getName() const noexcept { return _name; } + [[nodiscard]] bool isTop() const noexcept { return !_up; } + [[nodiscard]] bool isBottom() const noexcept { return !_down; } + [[nodiscard]] unsigned int size() const noexcept { + return (isBottom() ? 1 : _down->size() + 1); + } /** Adds the link to the end of the chain. */ void push_back(StorageLink::UP); /** Get the current state of the storage link. */ - State getState() const noexcept { return _state.load(std::memory_order_relaxed); } + [[nodiscard]] State getState() const noexcept { + return _state.load(std::memory_order_relaxed); + } /** * Called by storage server after the storage chain have been created. diff --git a/storage/src/vespa/storage/common/visitorfactory.h b/storage/src/vespa/storage/common/visitorfactory.h index 393744c4f59..1f18ace3095 100644 --- a/storage/src/vespa/storage/common/visitorfactory.h +++ b/storage/src/vespa/storage/common/visitorfactory.h @@ -12,6 +12,7 @@ namespace storage { +class StorageComponent; class Visitor; class VisitorEnvironment { diff --git a/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.cpp b/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.cpp index 871cdaddb53..1984d44652a 100644 --- a/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.cpp +++ b/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.cpp @@ -14,7 +14,6 @@ #include <vespa/vespalib/stllike/hash_map.hpp> #include <vespa/vespalib/util/exceptions.h> #include <vespa/vespalib/util/string_escape.h> -#include <xxhash.h> #include <vespa/log/log.h> LOG_SETUP(".persistence.filestor.handler.impl"); @@ -896,8 +895,7 @@ FileStorHandlerImpl::flush() uint64_t FileStorHandlerImpl::dispersed_bucket_bits(const document::Bucket& bucket) noexcept { - const uint64_t raw_id = bucket.getBucketId().getId(); - return XXH3_64bits(&raw_id, sizeof(uint64_t)); + return vespalib::xxhash::xxh3_64(bucket.getBucketId().getId()); } FileStorHandlerImpl::Stripe::Stripe(const FileStorHandlerImpl & owner, MessageSender & messageSender) @@ -1129,7 +1127,7 @@ FileStorHandlerImpl::Stripe::flush() namespace { bool -message_type_is_merge_related(api::MessageType::Id msg_type_id) { +message_type_is_merge_related(api::MessageType::Id msg_type_id) noexcept { switch (msg_type_id) { case api::MessageType::MERGEBUCKET_ID: case api::MessageType::MERGEBUCKET_REPLY_ID: @@ -1137,6 +1135,11 @@ message_type_is_merge_related(api::MessageType::Id msg_type_id) { case api::MessageType::GETBUCKETDIFF_REPLY_ID: case api::MessageType::APPLYBUCKETDIFF_ID: case api::MessageType::APPLYBUCKETDIFF_REPLY_ID: + // DeleteBucket is usually (but not necessarily) executed in the context of a higher-level + // merge operation, but we include it here since we want to enforce that not all threads + // in a stripe can dispatch a bucket delete at the same time. This also provides a strict + // upper bound on the number of in-flight bucket deletes in the persistence core. + case api::MessageType::DELETEBUCKET_ID: return true; default: return false; } diff --git a/storage/src/vespa/storage/persistence/filestorage/filestormanager.cpp b/storage/src/vespa/storage/persistence/filestorage/filestormanager.cpp index 97be1a510c4..e6cd8987c0a 100644 --- a/storage/src/vespa/storage/persistence/filestorage/filestormanager.cpp +++ b/storage/src/vespa/storage/persistence/filestorage/filestormanager.cpp @@ -62,7 +62,7 @@ private: } FileStorManager:: -FileStorManager(const config::ConfigUri & configUri, spi::PersistenceProvider& provider, +FileStorManager(const StorFilestorConfig& bootstrap_config, spi::PersistenceProvider& provider, ServiceLayerComponentRegister& compReg, DoneInitializeHandler& init_handler, HostInfo& hostInfoReporterRegistrar) : StorageLinkQueued("File store manager", compReg), @@ -75,7 +75,6 @@ FileStorManager(const config::ConfigUri & configUri, spi::PersistenceProvider& p _persistenceHandlers(), _threads(), _bucketOwnershipNotifier(std::make_unique<BucketOwnershipNotifier>(_component, *this)), - _configFetcher(std::make_unique<config::ConfigFetcher>(configUri.getContext())), _use_async_message_handling_on_schedule(false), _metrics(std::make_unique<FileStorMetrics>()), _filestorHandler(), @@ -85,8 +84,7 @@ FileStorManager(const config::ConfigUri & configUri, spi::PersistenceProvider& p _host_info_reporter(_component.getStateUpdater()), _resource_usage_listener_registration(provider.register_resource_usage_listener(_host_info_reporter)) { - _configFetcher->subscribe(configUri.getConfigId(), this); - _configFetcher->start(); + on_configure(bootstrap_config); _component.registerMetric(*_metrics); _component.registerStatusPage(*this); _component.getStateUpdater().addStateListener(*this); @@ -172,7 +170,7 @@ dynamic_throttle_params_from_config(const StorFilestorConfig& config, uint32_t n #define TLS_LINKAGE __attribute__((visibility("hidden"), tls_model("local-exec"))) #endif -thread_local PersistenceHandler * _G_threadLocalHandler TLS_LINKAGE = nullptr; +thread_local PersistenceHandler * _g_threadLocalHandler TLS_LINKAGE = nullptr; size_t computeAllPossibleHandlerThreads(const vespa::config::content::StorFilestorConfig & cfg) { @@ -200,26 +198,26 @@ FileStorManager::createRegisteredHandler(const ServiceLayerComponent & component PersistenceHandler & FileStorManager::getThreadLocalHandler() { - if (_G_threadLocalHandler == nullptr) { - _G_threadLocalHandler = & createRegisteredHandler(_component); + if (_g_threadLocalHandler == nullptr) { + _g_threadLocalHandler = & createRegisteredHandler(_component); } - return *_G_threadLocalHandler; + return *_g_threadLocalHandler; } void -FileStorManager::configure(std::unique_ptr<StorFilestorConfig> config) +FileStorManager::on_configure(const StorFilestorConfig& config) { // If true, this is not the first configure. const bool liveUpdate = ! _threads.empty(); - _use_async_message_handling_on_schedule = config->useAsyncMessageHandlingOnSchedule; - _host_info_reporter.set_noise_level(config->resourceUsageReporterNoiseLevel); - const bool use_dynamic_throttling = ((config->asyncOperationThrottlerType == StorFilestorConfig::AsyncOperationThrottlerType::DYNAMIC) || - (config->asyncOperationThrottler.type == StorFilestorConfig::AsyncOperationThrottler::Type::DYNAMIC)); - const bool throttle_merge_feed_ops = config->asyncOperationThrottler.throttleIndividualMergeFeedOps; + _use_async_message_handling_on_schedule = config.useAsyncMessageHandlingOnSchedule; + _host_info_reporter.set_noise_level(config.resourceUsageReporterNoiseLevel); + const bool use_dynamic_throttling = ((config.asyncOperationThrottlerType == StorFilestorConfig::AsyncOperationThrottlerType::DYNAMIC) || + (config.asyncOperationThrottler.type == StorFilestorConfig::AsyncOperationThrottler::Type::DYNAMIC)); + const bool throttle_merge_feed_ops = config.asyncOperationThrottler.throttleIndividualMergeFeedOps; if (!liveUpdate) { - _config = std::move(config); + _config = std::make_unique<StorFilestorConfig>(config); uint32_t numThreads = std::max(1, _config->numThreads); uint32_t numStripes = std::max(1u, numThreads / 2); _metrics->initDiskMetrics(numStripes, computeAllPossibleHandlerThreads(*_config)); @@ -240,7 +238,7 @@ FileStorManager::configure(std::unique_ptr<StorFilestorConfig> config) _bucketExecutorRegistration = _provider->register_executor(std::make_shared<BucketExecutorWrapper>(*this)); } else { assert(_filestorHandler); - auto updated_dyn_throttle_params = dynamic_throttle_params_from_config(*config, _threads.size()); + auto updated_dyn_throttle_params = dynamic_throttle_params_from_config(config, _threads.size()); _filestorHandler->reconfigure_dynamic_throttler(updated_dyn_throttle_params); } // TODO remove once desired dynamic throttling behavior is set in stone @@ -828,8 +826,6 @@ void FileStorManager::onClose() LOG(debug, "Start closing"); _bucketExecutorRegistration.reset(); _resource_usage_listener_registration.reset(); - // Avoid getting config during shutdown - _configFetcher->close(); LOG(debug, "Closed _configFetcher."); _filestorHandler->close(); LOG(debug, "Closed _filestorHandler."); diff --git a/storage/src/vespa/storage/persistence/filestorage/filestormanager.h b/storage/src/vespa/storage/persistence/filestorage/filestormanager.h index 68491ab1e38..96cff8dfeee 100644 --- a/storage/src/vespa/storage/persistence/filestorage/filestormanager.h +++ b/storage/src/vespa/storage/persistence/filestorage/filestormanager.h @@ -1,10 +1,4 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -/** - * @class storage::FileStorManager - * @ingroup filestorage - * - * @version $Id$ - */ #pragma once @@ -42,7 +36,6 @@ namespace spi { struct PersistenceProvider; } class ContentBucketSpace; struct FileStorManagerTest; -class ReadBucketList; class BucketOwnershipNotifier; class AbortBucketOperationsCommand; struct DoneInitializeHandler; @@ -54,10 +47,11 @@ class ProviderErrorWrapper; class FileStorManager : public StorageLinkQueued, public framework::HtmlStatusReporter, public StateListener, - private config::IFetcherCallback<vespa::config::content::StorFilestorConfig>, public MessageSender, public spi::BucketExecutor { + using StorFilestorConfig = vespa::config::content::StorFilestorConfig; + ServiceLayerComponentRegister & _compReg; ServiceLayerComponent _component; std::unique_ptr<spi::PersistenceProvider> _provider; @@ -69,7 +63,6 @@ class FileStorManager : public StorageLinkQueued, std::unique_ptr<BucketOwnershipNotifier> _bucketOwnershipNotifier; std::unique_ptr<vespa::config::content::StorFilestorConfig> _config; - std::unique_ptr<config::ConfigFetcher> _configFetcher; bool _use_async_message_handling_on_schedule; std::shared_ptr<FileStorMetrics> _metrics; std::unique_ptr<FileStorHandler> _filestorHandler; @@ -82,7 +75,7 @@ class FileStorManager : public StorageLinkQueued, std::unique_ptr<vespalib::IDestructorCallback> _resource_usage_listener_registration; public: - FileStorManager(const config::ConfigUri &, spi::PersistenceProvider&, + FileStorManager(const StorFilestorConfig&, spi::PersistenceProvider&, ServiceLayerComponentRegister&, DoneInitializeHandler&, HostInfo&); FileStorManager(const FileStorManager &) = delete; FileStorManager& operator=(const FileStorManager &) = delete; @@ -118,8 +111,9 @@ public: const FileStorMetrics& get_metrics() const { return *_metrics; } + void on_configure(const vespa::config::content::StorFilestorConfig& config); + private: - void configure(std::unique_ptr<vespa::config::content::StorFilestorConfig> config) override; PersistenceHandler & createRegisteredHandler(const ServiceLayerComponent & component); VESPA_DLL_LOCAL PersistenceHandler & getThreadLocalHandler(); diff --git a/storage/src/vespa/storage/persistence/filestorage/modifiedbucketchecker.cpp b/storage/src/vespa/storage/persistence/filestorage/modifiedbucketchecker.cpp index e1c6465b332..9558429bf13 100644 --- a/storage/src/vespa/storage/persistence/filestorage/modifiedbucketchecker.cpp +++ b/storage/src/vespa/storage/persistence/filestorage/modifiedbucketchecker.cpp @@ -46,12 +46,11 @@ ModifiedBucketChecker::BucketIdListResult::reset(document::BucketSpace bucketSpa ModifiedBucketChecker::ModifiedBucketChecker( ServiceLayerComponentRegister& compReg, spi::PersistenceProvider& provider, - const config::ConfigUri& configUri) + const StorServerConfig& bootstrap_config) : StorageLink("Modified bucket checker"), _provider(provider), _component(), _thread(), - _configFetcher(std::make_unique<config::ConfigFetcher>(configUri.getContext())), _monitor(), _stateLock(), _bucketSpaces(), @@ -60,8 +59,7 @@ ModifiedBucketChecker::ModifiedBucketChecker( _maxPendingChunkSize(100), _singleThreadMode(false) { - _configFetcher->subscribe<vespa::config::content::core::StorServerConfig>(configUri.getConfigId(), this); - _configFetcher->start(); + on_configure(bootstrap_config); std::ostringstream threadName; threadName << "Modified bucket checker " << static_cast<void*>(this); @@ -75,15 +73,14 @@ ModifiedBucketChecker::~ModifiedBucketChecker() } void -ModifiedBucketChecker::configure( - std::unique_ptr<vespa::config::content::core::StorServerConfig> newConfig) +ModifiedBucketChecker::on_configure(const vespa::config::content::core::StorServerConfig& newConfig) { std::lock_guard lock(_stateLock); - if (newConfig->bucketRecheckingChunkSize < 1) { + if (newConfig.bucketRecheckingChunkSize < 1) { throw config::InvalidConfigException( "Cannot have bucket rechecking chunk size of less than 1"); } - _maxPendingChunkSize = newConfig->bucketRecheckingChunkSize; + _maxPendingChunkSize = newConfig.bucketRecheckingChunkSize; } diff --git a/storage/src/vespa/storage/persistence/filestorage/modifiedbucketchecker.h b/storage/src/vespa/storage/persistence/filestorage/modifiedbucketchecker.h index 18f03da7469..9f0111b32f9 100644 --- a/storage/src/vespa/storage/persistence/filestorage/modifiedbucketchecker.h +++ b/storage/src/vespa/storage/persistence/filestorage/modifiedbucketchecker.h @@ -23,19 +23,18 @@ namespace spi { struct PersistenceProvider; } class ModifiedBucketChecker : public StorageLink, public framework::Runnable, - public Types, - private config::IFetcherCallback< - vespa::config::content::core::StorServerConfig> + public Types { public: + using StorServerConfig = vespa::config::content::core::StorServerConfig; using UP = std::unique_ptr<ModifiedBucketChecker>; ModifiedBucketChecker(ServiceLayerComponentRegister& compReg, spi::PersistenceProvider& provide, - const config::ConfigUri& configUri); + const StorServerConfig& bootstrap_config); ~ModifiedBucketChecker() override; - void configure(std::unique_ptr<vespa::config::content::core::StorServerConfig>) override; + void on_configure(const vespa::config::content::core::StorServerConfig&); void run(framework::ThreadHandle& thread) override; bool tick(); @@ -88,7 +87,6 @@ private: spi::PersistenceProvider & _provider; ServiceLayerComponent::UP _component; std::unique_ptr<framework::Thread> _thread; - std::unique_ptr<config::ConfigFetcher> _configFetcher; std::mutex _monitor; std::condition_variable _cond; std::mutex _stateLock; diff --git a/storage/src/vespa/storage/storageserver/bouncer.cpp b/storage/src/vespa/storage/storageserver/bouncer.cpp index 404058325b9..bfc38e0c8ba 100644 --- a/storage/src/vespa/storage/storageserver/bouncer.cpp +++ b/storage/src/vespa/storage/storageserver/bouncer.cpp @@ -21,34 +21,19 @@ LOG_SETUP(".bouncer"); namespace storage { -Bouncer::Bouncer(StorageComponentRegister& compReg, const config::ConfigUri & configUri) - : StorageLink("Bouncer"), - _config(new vespa::config::content::core::StorBouncerConfig()), +Bouncer::Bouncer(StorageComponentRegister& compReg, const StorBouncerConfig& bootstrap_config) + : StorageLink("Bouncer", MsgDownOnFlush::Disallowed, MsgUpOnClosed::Allowed), + _config(std::make_unique<vespa::config::content::core::StorBouncerConfig>(bootstrap_config)), _component(compReg, "bouncer"), _lock(), _baselineNodeState("s:i"), _derivedNodeStates(), _clusterState(&lib::State::UP), - _configFetcher(std::make_unique<config::ConfigFetcher>(configUri.getContext())), - _metrics(std::make_unique<BouncerMetrics>()) + _metrics(std::make_unique<BouncerMetrics>()), + _closed(false) { _component.getStateUpdater().addStateListener(*this); _component.registerMetric(*_metrics); - // Register for config. Normally not critical, so catching config - // exception allowing program to continue if missing/faulty config. - try{ - if (!configUri.empty()) { - _configFetcher->subscribe<vespa::config::content::core::StorBouncerConfig>(configUri.getConfigId(), this); - _configFetcher->start(); - } else { - LOG(info, "No config id specified. Using defaults rather than " - "config"); - } - } catch (config::InvalidConfigException& e) { - LOG(info, "Bouncer failed to load config '%s'. This " - "is not critical since it has sensible defaults: %s", - configUri.getConfigId().c_str(), e.what()); - } } Bouncer::~Bouncer() @@ -62,23 +47,25 @@ Bouncer::print(std::ostream& out, bool verbose, const std::string& indent) const { (void) verbose; (void) indent; + std::lock_guard guard(_lock); out << "Bouncer(" << _baselineNodeState << ")"; } void Bouncer::onClose() { - _configFetcher->close(); _component.getStateUpdater().removeStateListener(*this); + std::lock_guard guard(_lock); + _closed = true; } void -Bouncer::configure(std::unique_ptr<vespa::config::content::core::StorBouncerConfig> config) +Bouncer::on_configure(const vespa::config::content::core::StorBouncerConfig& config) { - log_config_received(*config); - validateConfig(*config); + validateConfig(config); + auto new_config = std::make_unique<StorBouncerConfig>(config); std::lock_guard lock(_lock); - _config = std::move(config); + _config = std::move(new_config); } const BouncerMetrics& Bouncer::metrics() const noexcept { @@ -86,8 +73,7 @@ const BouncerMetrics& Bouncer::metrics() const noexcept { } void -Bouncer::validateConfig( - const vespa::config::content::core::StorBouncerConfig& newConfig) const +Bouncer::validateConfig(const vespa::config::content::core::StorBouncerConfig& newConfig) const { if (newConfig.feedRejectionPriorityThreshold != -1) { if (newConfig.feedRejectionPriorityThreshold @@ -112,12 +98,11 @@ void Bouncer::append_node_identity(std::ostream& target_stream) const { } void -Bouncer::abortCommandForUnavailableNode(api::StorageMessage& msg, - const lib::State& state) +Bouncer::abortCommandForUnavailableNode(api::StorageMessage& msg, const lib::State& state) { // If we're not up or retired, fail due to this nodes state. std::shared_ptr<api::StorageReply> reply( - static_cast<api::StorageCommand&>(msg).makeReply().release()); + static_cast<api::StorageCommand&>(msg).makeReply()); std::ostringstream ost; ost << "We don't allow command of type " << msg.getType() << " when node is in state " << state.toString(true); @@ -128,8 +113,7 @@ Bouncer::abortCommandForUnavailableNode(api::StorageMessage& msg, } void -Bouncer::rejectCommandWithTooHighClockSkew(api::StorageMessage& msg, - int maxClockSkewInSeconds) +Bouncer::rejectCommandWithTooHighClockSkew(api::StorageMessage& msg, int maxClockSkewInSeconds) { auto& as_cmd = dynamic_cast<api::StorageCommand&>(msg); std::ostringstream ost; @@ -140,7 +124,7 @@ Bouncer::rejectCommandWithTooHighClockSkew(api::StorageMessage& msg, as_cmd.getSourceIndex(), ost.str().c_str()); _metrics->clock_skew_aborts.inc(); - std::shared_ptr<api::StorageReply> reply(as_cmd.makeReply().release()); + std::shared_ptr<api::StorageReply> reply(as_cmd.makeReply()); reply->setResult(api::ReturnCode(api::ReturnCode::REJECTED, ost.str())); sendUp(reply); } @@ -148,8 +132,7 @@ Bouncer::rejectCommandWithTooHighClockSkew(api::StorageMessage& msg, void Bouncer::abortCommandDueToClusterDown(api::StorageMessage& msg, const lib::State& cluster_state) { - std::shared_ptr<api::StorageReply> reply( - static_cast<api::StorageCommand&>(msg).makeReply().release()); + std::shared_ptr<api::StorageReply> reply(static_cast<api::StorageCommand&>(msg).makeReply()); std::ostringstream ost; ost << "We don't allow external load while cluster is in state " << cluster_state.toString(true); @@ -172,35 +155,35 @@ uint64_t Bouncer::extractMutationTimestampIfAny(const api::StorageMessage& msg) { switch (msg.getType().getId()) { - case api::MessageType::PUT_ID: - return static_cast<const api::PutCommand&>(msg).getTimestamp(); - case api::MessageType::REMOVE_ID: - return static_cast<const api::RemoveCommand&>(msg).getTimestamp(); - case api::MessageType::UPDATE_ID: - return static_cast<const api::UpdateCommand&>(msg).getTimestamp(); - default: - return 0; + case api::MessageType::PUT_ID: + return static_cast<const api::PutCommand&>(msg).getTimestamp(); + case api::MessageType::REMOVE_ID: + return static_cast<const api::RemoveCommand&>(msg).getTimestamp(); + case api::MessageType::UPDATE_ID: + return static_cast<const api::UpdateCommand&>(msg).getTimestamp(); + default: + return 0; } } bool -Bouncer::isExternalLoad(const api::MessageType& type) const noexcept +Bouncer::isExternalLoad(const api::MessageType& type) noexcept { switch (type.getId()) { - case api::MessageType::PUT_ID: - case api::MessageType::REMOVE_ID: - case api::MessageType::UPDATE_ID: - case api::MessageType::GET_ID: - case api::MessageType::VISITOR_CREATE_ID: - case api::MessageType::STATBUCKET_ID: - return true; - default: - return false; + case api::MessageType::PUT_ID: + case api::MessageType::REMOVE_ID: + case api::MessageType::UPDATE_ID: + case api::MessageType::GET_ID: + case api::MessageType::VISITOR_CREATE_ID: + case api::MessageType::STATBUCKET_ID: + return true; + default: + return false; } } bool -Bouncer::isExternalWriteOperation(const api::MessageType& type) const noexcept { +Bouncer::isExternalWriteOperation(const api::MessageType& type) noexcept { switch (type.getId()) { case api::MessageType::PUT_ID: case api::MessageType::REMOVE_ID: @@ -216,8 +199,7 @@ Bouncer::rejectDueToInsufficientPriority( api::StorageMessage& msg, api::StorageMessage::Priority feedPriorityLowerBound) { - std::shared_ptr<api::StorageReply> reply( - static_cast<api::StorageCommand&>(msg).makeReply().release()); + std::shared_ptr<api::StorageReply> reply(static_cast<api::StorageCommand&>(msg).makeReply()); std::ostringstream ost; ost << "Operation priority (" << int(msg.getPriority()) << ") is lower than currently configured threshold (" @@ -231,8 +213,7 @@ Bouncer::rejectDueToInsufficientPriority( void Bouncer::reject_due_to_too_few_bucket_bits(api::StorageMessage& msg) { - std::shared_ptr<api::StorageReply> reply( - dynamic_cast<api::StorageCommand&>(msg).makeReply()); + std::shared_ptr<api::StorageReply> reply(dynamic_cast<api::StorageCommand&>(msg).makeReply()); reply->setResult(api::ReturnCode(api::ReturnCode::REJECTED, vespalib::make_string("Operation bucket %s has too few bits used (%u < minimum of %u)", msg.getBucketId().toString().c_str(), @@ -241,31 +222,22 @@ Bouncer::reject_due_to_too_few_bucket_bits(api::StorageMessage& msg) { sendUp(reply); } +void +Bouncer::reject_due_to_node_shutdown(api::StorageMessage& msg) { + std::shared_ptr<api::StorageReply> reply(dynamic_cast<api::StorageCommand&>(msg).makeReply()); + reply->setResult(api::ReturnCode(api::ReturnCode::ABORTED, "Node is shutting down")); + sendUp(reply); +} + bool Bouncer::onDown(const std::shared_ptr<api::StorageMessage>& msg) { - const api::MessageType& type(msg->getType()); - // All replies can come in. - if (type.isReply()) { - return false; - } - - switch (type.getId()) { - case api::MessageType::SETNODESTATE_ID: - case api::MessageType::GETNODESTATE_ID: - case api::MessageType::SETSYSTEMSTATE_ID: - case api::MessageType::ACTIVATE_CLUSTER_STATE_VERSION_ID: - case api::MessageType::NOTIFYBUCKETCHANGE_ID: - // state commands are always ok - return false; - default: - break; - } const lib::State* state; int maxClockSkewInSeconds; bool isInAvailableState; bool abortLoadWhenClusterDown; - const lib::State *cluster_state; + bool closed; + const lib::State* cluster_state; int feedPriorityLowerBound; { std::lock_guard lock(_lock); @@ -275,7 +247,34 @@ Bouncer::onDown(const std::shared_ptr<api::StorageMessage>& msg) cluster_state = _clusterState; isInAvailableState = state->oneOf(_config->stopAllLoadWhenNodestateNotIn.c_str()); feedPriorityLowerBound = _config->feedRejectionPriorityThreshold; + closed = _closed; + } + const api::MessageType& type = msg->getType(); + // If the node is shutting down, we want to prevent _any_ messages from reaching + // components further down the call chain. This means this case must be handled + // _before_ any logic that explicitly allows through certain message types. + if (closed) [[unlikely]] { + if (!type.isReply()) { + reject_due_to_node_shutdown(*msg); + } // else: swallow all replies + return true; } + // All replies can come in. + if (type.isReply()) { + return false; + } + switch (type.getId()) { + case api::MessageType::SETNODESTATE_ID: + case api::MessageType::GETNODESTATE_ID: + case api::MessageType::SETSYSTEMSTATE_ID: + case api::MessageType::ACTIVATE_CLUSTER_STATE_VERSION_ID: + case api::MessageType::NOTIFYBUCKETCHANGE_ID: + // state commands are always ok + return false; + default: + break; + } + // Special case for point lookup Gets while node is in maintenance mode // to allow reads to complete during two-phase cluster state transitions if ((*state == lib::State::MAINTENANCE) && (type.getId() == api::MessageType::GET_ID) && clusterIsUp(*cluster_state)) { @@ -345,9 +344,9 @@ void Bouncer::handleNewState() noexcept { std::lock_guard lock(_lock); - const auto reportedNodeState = *_component.getStateUpdater().getReportedNodeState(); + const auto reportedNodeState = *_component.getStateUpdater().getReportedNodeState(); const auto clusterStateBundle = _component.getStateUpdater().getClusterStateBundle(); - const auto &clusterState = *clusterStateBundle->getBaselineClusterState(); + const auto& clusterState = *clusterStateBundle->getBaselineClusterState(); _clusterState = &clusterState.getClusterState(); const lib::Node node(_component.getNodeType(), _component.getIndex()); _baselineNodeState = deriveNodeState(reportedNodeState, clusterState.getNodeState(node)); diff --git a/storage/src/vespa/storage/storageserver/bouncer.h b/storage/src/vespa/storage/storageserver/bouncer.h index 78f07f10316..26282625269 100644 --- a/storage/src/vespa/storage/storageserver/bouncer.h +++ b/storage/src/vespa/storage/storageserver/bouncer.h @@ -1,12 +1,7 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. /** - * @class storage::Bouncer - * @ingroup storageserver - * - * @brief Denies messages from entering if state is not good. - * - * If we are not in up state, but process is still running, only a few - * messages should be allowed through. This link stops all messages not allowed. + * Component which rejects messages that can not be accepted by the node in + * its current state. */ #pragma once @@ -29,27 +24,28 @@ namespace storage { struct BouncerMetrics; class Bouncer : public StorageLink, - private StateListener, - private config::IFetcherCallback<vespa::config::content::core::StorBouncerConfig> + private StateListener { - std::unique_ptr<vespa::config::content::core::StorBouncerConfig> _config; - StorageComponent _component; - std::mutex _lock; - lib::NodeState _baselineNodeState; + using StorBouncerConfig = vespa::config::content::core::StorBouncerConfig; using BucketSpaceNodeStateMapping = std::unordered_map<document::BucketSpace, lib::NodeState, document::BucketSpace::hash>; - BucketSpaceNodeStateMapping _derivedNodeStates; - const lib::State* _clusterState; - std::unique_ptr<config::ConfigFetcher> _configFetcher; - std::unique_ptr<BouncerMetrics> _metrics; + + std::unique_ptr<StorBouncerConfig> _config; + StorageComponent _component; + mutable std::mutex _lock; + lib::NodeState _baselineNodeState; + BucketSpaceNodeStateMapping _derivedNodeStates; + const lib::State* _clusterState; + std::unique_ptr<BouncerMetrics> _metrics; + bool _closed; public: - Bouncer(StorageComponentRegister& compReg, const config::ConfigUri & configUri); + Bouncer(StorageComponentRegister& compReg, const StorBouncerConfig& bootstrap_config); ~Bouncer() override; void print(std::ostream& out, bool verbose, const std::string& indent) const override; - void configure(std::unique_ptr<vespa::config::content::core::StorBouncerConfig> config) override; + void on_configure(const StorBouncerConfig& config); const BouncerMetrics& metrics() const noexcept; private: @@ -60,11 +56,12 @@ private: void abortCommandDueToClusterDown(api::StorageMessage&, const lib::State&); void rejectDueToInsufficientPriority(api::StorageMessage&, api::StorageMessage::Priority); void reject_due_to_too_few_bucket_bits(api::StorageMessage&); + void reject_due_to_node_shutdown(api::StorageMessage&); static bool clusterIsUp(const lib::State& cluster_state); bool isDistributor() const; - bool isExternalLoad(const api::MessageType&) const noexcept; - bool isExternalWriteOperation(const api::MessageType&) const noexcept; - bool priorityRejectionIsEnabled(int configuredPriority) const noexcept { + static bool isExternalLoad(const api::MessageType&) noexcept; + static bool isExternalWriteOperation(const api::MessageType&) noexcept; + static bool priorityRejectionIsEnabled(int configuredPriority) noexcept { return (configuredPriority != -1); } @@ -72,7 +69,7 @@ private: * If msg is a command containing a mutating timestamp (put, remove or * update commands), return that timestamp. Otherwise, return 0. */ - uint64_t extractMutationTimestampIfAny(const api::StorageMessage& msg); + static uint64_t extractMutationTimestampIfAny(const api::StorageMessage& msg); bool onDown(const std::shared_ptr<api::StorageMessage>&) override; void handleNewState() noexcept override; const lib::NodeState &getDerivedNodeState(document::BucketSpace bucketSpace) const; diff --git a/storage/src/vespa/storage/storageserver/changedbucketownershiphandler.cpp b/storage/src/vespa/storage/storageserver/changedbucketownershiphandler.cpp index 34040bb12c0..25829f3d391 100644 --- a/storage/src/vespa/storage/storageserver/changedbucketownershiphandler.cpp +++ b/storage/src/vespa/storage/storageserver/changedbucketownershiphandler.cpp @@ -22,12 +22,11 @@ LOG_SETUP(".bucketownershiphandler"); namespace storage { ChangedBucketOwnershipHandler::ChangedBucketOwnershipHandler( - const config::ConfigUri& configUri, + const PersistenceConfig& bootstrap_config, ServiceLayerComponentRegister& compReg) : StorageLink("Changed bucket ownership handler"), _component(compReg, "changedbucketownershiphandler"), _metrics(), - _configFetcher(std::make_unique<config::ConfigFetcher>(configUri.getContext())), _state_sync_executor(1), // single thread for sequential task execution _stateLock(), _currentState(), // Not set yet, so ownership will not be valid @@ -37,25 +36,23 @@ ChangedBucketOwnershipHandler::ChangedBucketOwnershipHandler( _abortMutatingIdealStateOps(false), _abortMutatingExternalLoadOps(false) { - _configFetcher->subscribe<vespa::config::content::PersistenceConfig>(configUri.getConfigId(), this); - _configFetcher->start(); + on_configure(bootstrap_config); _component.registerMetric(_metrics); } ChangedBucketOwnershipHandler::~ChangedBucketOwnershipHandler() = default; void -ChangedBucketOwnershipHandler::configure( - std::unique_ptr<vespa::config::content::PersistenceConfig> config) +ChangedBucketOwnershipHandler::on_configure(const vespa::config::content::PersistenceConfig& config) { _abortQueuedAndPendingOnStateChange.store( - config->abortOperationsWithChangedBucketOwnership, + config.abortOperationsWithChangedBucketOwnership, std::memory_order_relaxed); _abortMutatingIdealStateOps.store( - config->abortOutdatedMutatingIdealStateOps, + config.abortOutdatedMutatingIdealStateOps, std::memory_order_relaxed); _abortMutatingExternalLoadOps.store( - config->abortOutdatedMutatingExternalLoadOps, + config.abortOutdatedMutatingExternalLoadOps, std::memory_order_relaxed); } diff --git a/storage/src/vespa/storage/storageserver/changedbucketownershiphandler.h b/storage/src/vespa/storage/storageserver/changedbucketownershiphandler.h index 3e20eb507f6..801534385f7 100644 --- a/storage/src/vespa/storage/storageserver/changedbucketownershiphandler.h +++ b/storage/src/vespa/storage/storageserver/changedbucketownershiphandler.h @@ -56,10 +56,7 @@ namespace lib { * - RemoveCommand * - RevertCommand */ -class ChangedBucketOwnershipHandler - : public StorageLink, - private config::IFetcherCallback<vespa::config::content::PersistenceConfig> -{ +class ChangedBucketOwnershipHandler : public StorageLink { public: class Metrics : public metrics::MetricSet { public: @@ -115,12 +112,11 @@ public: private: class ClusterStateSyncAndApplyTask; - using ConfigFetcherUP = std::unique_ptr<config::ConfigFetcher>; + using PersistenceConfig = vespa::config::content::PersistenceConfig; using ClusterStateBundleCSP = std::shared_ptr<const lib::ClusterStateBundle>; ServiceLayerComponent _component; Metrics _metrics; - ConfigFetcherUP _configFetcher; vespalib::ThreadStackExecutor _state_sync_executor; mutable std::mutex _stateLock; ClusterStateBundleCSP _currentState; @@ -185,7 +181,7 @@ private: bool enabledExternalLoadAborting() const; public: - ChangedBucketOwnershipHandler(const config::ConfigUri& configUri, + ChangedBucketOwnershipHandler(const PersistenceConfig& bootstrap_config, ServiceLayerComponentRegister& compReg); ~ChangedBucketOwnershipHandler() override; @@ -194,7 +190,7 @@ public: bool onInternalReply(const std::shared_ptr<api::InternalReply>& reply) override; void onClose() override; - void configure(std::unique_ptr<vespa::config::content::PersistenceConfig>) override; + void on_configure(const PersistenceConfig&); /** * We want to ensure distribution config changes are thread safe wrt. our diff --git a/storage/src/vespa/storage/storageserver/communicationmanager.cpp b/storage/src/vespa/storage/storageserver/communicationmanager.cpp index 610d9c8d707..bbd4e87cb40 100644 --- a/storage/src/vespa/storage/storageserver/communicationmanager.cpp +++ b/storage/src/vespa/storage/storageserver/communicationmanager.cpp @@ -216,18 +216,21 @@ convert_to_rpc_compression_config(const vespa::config::content::core::StorCommun } -CommunicationManager::CommunicationManager(StorageComponentRegister& compReg, const config::ConfigUri & configUri) - : StorageLink("Communication manager"), +CommunicationManager::CommunicationManager(StorageComponentRegister& compReg, + const config::ConfigUri& configUri, + const CommunicationManagerConfig& bootstrap_config) + : StorageLink("Communication manager", MsgDownOnFlush::Allowed, MsgUpOnClosed::Disallowed), _component(compReg, "communicationmanager"), _metrics(), _shared_rpc_resources(), // Created upon initial configuration _storage_api_rpc_service(), // (ditto) _cc_rpc_service(), // (ditto) _eventQueue(), + _bootstrap_config(std::make_unique<CommunicationManagerConfig>(bootstrap_config)), _mbus(), _configUri(configUri), _closed(false), - _docApiConverter(configUri, std::make_shared<PlaceHolderBucketResolver>()), + _docApiConverter(std::make_shared<PlaceHolderBucketResolver>()), _thread() { _component.registerMetricUpdateHook(*this, 5s); @@ -237,9 +240,13 @@ CommunicationManager::CommunicationManager(StorageComponentRegister& compReg, co void CommunicationManager::onOpen() { - _configFetcher = std::make_unique<config::ConfigFetcher>(_configUri.getContext()); - _configFetcher->subscribe<vespa::config::content::core::StorCommunicationmanagerConfig>(_configUri.getConfigId(), this); - _configFetcher->start(); + // We have to hold on to the bootstrap config until we reach the open-phase, as the + // actual RPC/mbus endpoints are started at the first config edge. + // Note: this is called as part of synchronous node initialization, which explicitly + // prevents any concurrent reconfiguration prior to opening all storage chain components, + // i.e. there's no risk of on_configure() being called _prior_ to us getting here. + on_configure(*_bootstrap_config); + _bootstrap_config.reset(); _thread = _component.startThread(*this, 60s); if (_shared_rpc_resources) { @@ -275,28 +282,25 @@ CommunicationManager::~CommunicationManager() void CommunicationManager::onClose() { - // Avoid getting config during shutdown - _configFetcher.reset(); - - _closed = true; - - if (_mbus) { - if (_messageBusSession) { - _messageBusSession->close(); - } - } - - // TODO remove? this no longer has any particularly useful semantics + _closed.store(true, std::memory_order_seq_cst); if (_cc_rpc_service) { - _cc_rpc_service->close(); + _cc_rpc_service->close(); // Auto-abort all incoming CC RPC requests from now on } - // TODO do this after we drain queues? + // Sync all RPC threads to ensure that any subsequent RPCs must observe the closed-flags we just set if (_shared_rpc_resources) { - _shared_rpc_resources->shutdown(); + _shared_rpc_resources->sync_all_threads(); + } + + if (_mbus && _messageBusSession) { + // Closing the mbus session unregisters the destination session and syncs the worker + // thread(s), so once this call returns we should not observe further incoming requests + // through this pipeline. Previous messages may already be in flight internally; these + // will be handled by flushing-phases. + _messageBusSession->close(); } - // Stopping pumper thread should stop all incoming messages from being - // processed. + // Stopping internal message dispatch thread should stop all incoming _async_ messages + // from being processed. _Synchronously_ dispatched RPCs are still passing through. if (_thread) { _thread->interrupt(); _eventQueue.signal(); @@ -305,13 +309,12 @@ CommunicationManager::onClose() } // Emptying remaining queued messages - // FIXME but RPC/mbus is already shut down at this point...! Make sure we handle this std::shared_ptr<api::StorageMessage> msg; api::ReturnCode code(api::ReturnCode::ABORTED, "Node shutting down"); while (_eventQueue.size() > 0) { assert(_eventQueue.getNext(msg, 0ms)); if (!msg->getType().isReply()) { - std::shared_ptr<api::StorageReply> reply(static_cast<api::StorageCommand&>(*msg).makeReply()); + std::shared_ptr<api::StorageReply> reply(dynamic_cast<api::StorageCommand&>(*msg).makeReply()); reply->setResult(code); sendReply(reply); } @@ -319,6 +322,29 @@ CommunicationManager::onClose() } void +CommunicationManager::onFlush(bool downwards) +{ + if (downwards) { + // Sync RPC threads once more (with feeling!) to ensure that any closing done by other components + // during the storage chain onClose() is visible to these. + if (_shared_rpc_resources) { + _shared_rpc_resources->sync_all_threads(); + } + // By this point, no inbound RPCs (requests and responses) should be allowed any further down + // than the Bouncer component, where they will be, well, bounced. + } else { + // All components further down the storage chain should now be completely closed + // and flushed, and all message-dispatching threads should have been shut down. + // It's possible that the RPC threads are still butting heads up against the Bouncer + // component, so we conclude the shutdown ceremony by taking down the RPC subsystem. + // This transitively waits for all RPC threads to complete. + if (_shared_rpc_resources) { + _shared_rpc_resources->shutdown(); + } + } +} + +void CommunicationManager::configureMessageBusLimits(const CommunicationManagerConfig& cfg) { const bool isDist(_component.getNodeType() == lib::NodeType::DISTRIBUTOR); @@ -330,20 +356,20 @@ CommunicationManager::configureMessageBusLimits(const CommunicationManagerConfig } void -CommunicationManager::configure(std::unique_ptr<CommunicationManagerConfig> config) +CommunicationManager::on_configure(const CommunicationManagerConfig& config) { // Only allow dynamic (live) reconfiguration of message bus limits. if (_mbus) { - configureMessageBusLimits(*config); - if (_mbus->getRPCNetwork().getPort() != config->mbusport) { + configureMessageBusLimits(config); + if (_mbus->getRPCNetwork().getPort() != config.mbusport) { auto m = make_string("mbus port changed from %d to %d. Will conduct a quick, but controlled restart.", - _mbus->getRPCNetwork().getPort(), config->mbusport); + _mbus->getRPCNetwork().getPort(), config.mbusport); LOG(warning, "%s", m.c_str()); _component.requestShutdown(m); } - if (_shared_rpc_resources->listen_port() != config->rpcport) { + if (_shared_rpc_resources->listen_port() != config.rpcport) { auto m = make_string("rpc port changed from %d to %d. Will conduct a quick, but controlled restart.", - _shared_rpc_resources->listen_port(), config->rpcport); + _shared_rpc_resources->listen_port(), config.rpcport); LOG(warning, "%s", m.c_str()); _component.requestShutdown(m); } @@ -353,25 +379,25 @@ CommunicationManager::configure(std::unique_ptr<CommunicationManagerConfig> conf if (!_configUri.empty()) { LOG(debug, "setting up slobrok config from id: '%s", _configUri.getConfigId().c_str()); mbus::RPCNetworkParams params(_configUri); - params.setConnectionExpireSecs(config->mbus.rpctargetcache.ttl); - params.setNumNetworkThreads(std::max(1, config->mbus.numNetworkThreads)); - params.setNumRpcTargets(std::max(1, config->mbus.numRpcTargets)); - params.events_before_wakeup(std::max(1, config->mbus.eventsBeforeWakeup)); - params.setTcpNoDelay(config->mbus.tcpNoDelay); + params.setConnectionExpireSecs(config.mbus.rpctargetcache.ttl); + params.setNumNetworkThreads(std::max(1, config.mbus.numNetworkThreads)); + params.setNumRpcTargets(std::max(1, config.mbus.numRpcTargets)); + params.events_before_wakeup(std::max(1, config.mbus.eventsBeforeWakeup)); + params.setTcpNoDelay(config.mbus.tcpNoDelay); params.required_capabilities(vespalib::net::tls::CapabilitySet::of({ vespalib::net::tls::Capability::content_document_api() })); params.setIdentity(mbus::Identity(_component.getIdentity())); - if (config->mbusport != -1) { - params.setListenPort(config->mbusport); + if (config.mbusport != -1) { + params.setListenPort(config.mbusport); } using CompressionConfig = vespalib::compression::CompressionConfig; CompressionConfig::Type compressionType = CompressionConfig::toType( - CommunicationManagerConfig::Mbus::Compress::getTypeName(config->mbus.compress.type).c_str()); - params.setCompressionConfig(CompressionConfig(compressionType, config->mbus.compress.level, - 90, config->mbus.compress.limit)); + CommunicationManagerConfig::Mbus::Compress::getTypeName(config.mbus.compress.type).c_str()); + params.setCompressionConfig(CompressionConfig(compressionType, config.mbus.compress.level, + 90, config.mbus.compress.limit)); // Configure messagebus here as we for legacy reasons have // config here. @@ -381,16 +407,16 @@ CommunicationManager::configure(std::unique_ptr<CommunicationManagerConfig> conf params, _configUri); - configureMessageBusLimits(*config); + configureMessageBusLimits(config); } _message_codec_provider = std::make_unique<rpc::MessageCodecProvider>(_component.getTypeRepo()->documentTypeRepo); - _shared_rpc_resources = std::make_unique<rpc::SharedRpcResources>(_configUri, config->rpcport, - config->rpc.numNetworkThreads, config->rpc.eventsBeforeWakeup); + _shared_rpc_resources = std::make_unique<rpc::SharedRpcResources>(_configUri, config.rpcport, + config.rpc.numNetworkThreads, config.rpc.eventsBeforeWakeup); _cc_rpc_service = std::make_unique<rpc::ClusterControllerApiRpcService>(*this, *_shared_rpc_resources); rpc::StorageApiRpcService::Params rpc_params; - rpc_params.compression_config = convert_to_rpc_compression_config(*config); - rpc_params.num_rpc_targets_per_node = config->rpc.numTargetsPerNode; + rpc_params.compression_config = convert_to_rpc_compression_config(config); + rpc_params.num_rpc_targets_per_node = config.rpc.numTargetsPerNode; _storage_api_rpc_service = std::make_unique<rpc::StorageApiRpcService>( *this, *_shared_rpc_resources, *_message_codec_provider, rpc_params); @@ -438,11 +464,15 @@ CommunicationManager::process(const std::shared_ptr<api::StorageMessage>& msg) } } +// Called directly by RPC threads void CommunicationManager::dispatch_sync(std::shared_ptr<api::StorageMessage> msg) { LOG(spam, "Direct dispatch of storage message %s, priority %d", msg->toString().c_str(), msg->getPriority()); + // If process is shutting down, msg will be synchronously aborted by the Bouncer component process(msg); } +// Called directly by RPC threads (for incoming CC requests) and by any other request-dispatching +// threads (i.e. calling sendUp) when address resolution fails and an internal error response is generated. void CommunicationManager::dispatch_async(std::shared_ptr<api::StorageMessage> msg) { LOG(spam, "Enqueued dispatch of storage message %s, priority %d", msg->toString().c_str(), msg->getPriority()); _eventQueue.enqueue(std::move(msg)); diff --git a/storage/src/vespa/storage/storageserver/communicationmanager.h b/storage/src/vespa/storage/storageserver/communicationmanager.h index da45124ed2d..7a910336b13 100644 --- a/storage/src/vespa/storage/storageserver/communicationmanager.h +++ b/storage/src/vespa/storage/storageserver/communicationmanager.h @@ -1,11 +1,6 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. /** - * @class CommunicationManager - * @ingroup storageserver - * - * @brief Class used for sending messages over the network. - * - * @version $Id$ + * Class used for sending messages over the network. */ #pragma once @@ -65,7 +60,6 @@ public: class CommunicationManager final : public StorageLink, public framework::Runnable, - private config::IFetcherCallback<vespa::config::content::core::StorCommunicationmanagerConfig>, public mbus::IMessageHandler, public mbus::IReplyHandler, private framework::MetricUpdateHook, @@ -80,8 +74,6 @@ private: std::unique_ptr<rpc::ClusterControllerApiRpcService> _cc_rpc_service; std::unique_ptr<rpc::MessageCodecProvider> _message_codec_provider; Queue _eventQueue; - // XXX: Should perhaps use a configsubscriber and poll from StorageComponent ? - std::unique_ptr<config::ConfigFetcher> _configFetcher; using EarlierProtocol = std::pair<vespalib::steady_time , mbus::IProtocol::SP>; using EarlierProtocols = std::vector<EarlierProtocol>; std::mutex _earlierGenerationsLock; @@ -89,6 +81,7 @@ private: void onOpen() override; void onClose() override; + void onFlush(bool downwards) override; void process(const std::shared_ptr<api::StorageMessage>& msg); @@ -96,7 +89,6 @@ private: using BucketspacesConfig = vespa::config::content::core::BucketspacesConfig; void configureMessageBusLimits(const CommunicationManagerConfig& cfg); - void configure(std::unique_ptr<CommunicationManagerConfig> config) override; void receiveStorageReply(const std::shared_ptr<api::StorageReply>&); void fail_with_unresolvable_bucket_space(std::unique_ptr<documentapi::DocumentMessage> msg, const vespalib::string& error_message); @@ -105,6 +97,7 @@ private: static const uint64_t FORWARDED_MESSAGE = 0; + std::unique_ptr<CommunicationManagerConfig> _bootstrap_config; std::unique_ptr<mbus::RPCMessageBus> _mbus; std::unique_ptr<mbus::DestinationSession> _messageBusSession; std::unique_ptr<mbus::SourceSession> _sourceSession; @@ -126,9 +119,12 @@ public: CommunicationManager(const CommunicationManager&) = delete; CommunicationManager& operator=(const CommunicationManager&) = delete; CommunicationManager(StorageComponentRegister& compReg, - const config::ConfigUri & configUri); + const config::ConfigUri& configUri, + const CommunicationManagerConfig& bootstrap_config); ~CommunicationManager() override; + void on_configure(const CommunicationManagerConfig& config); + // MessageDispatcher overrides void dispatch_sync(std::shared_ptr<api::StorageMessage> msg) override; void dispatch_async(std::shared_ptr<api::StorageMessage> msg) override; diff --git a/storage/src/vespa/storage/storageserver/distributornode.cpp b/storage/src/vespa/storage/storageserver/distributornode.cpp index 7d2a69a2200..10ee8023621 100644 --- a/storage/src/vespa/storage/storageserver/distributornode.cpp +++ b/storage/src/vespa/storage/storageserver/distributornode.cpp @@ -18,12 +18,13 @@ namespace storage { DistributorNode::DistributorNode( const config::ConfigUri& configUri, DistributorNodeContext& context, + BootstrapConfigs bootstrap_configs, ApplicationGenerationFetcher& generationFetcher, uint32_t num_distributor_stripes, StorageLink::UP communicationManager, std::unique_ptr<IStorageChainBuilder> storage_chain_builder) - : StorageNode(configUri, context, generationFetcher, - std::make_unique<HostInfo>(), + : StorageNode(configUri, context, std::move(bootstrap_configs), + generationFetcher, std::make_unique<HostInfo>(), !communicationManager ? NORMAL : SINGLE_THREADED_TEST_MODE), _threadPool(framework::TickingThreadPool::createDefault("distributor", 100ms, 1, 5s)), _stripe_pool(std::make_unique<distributor::DistributorStripePool>()), @@ -32,7 +33,8 @@ DistributorNode::DistributorNode( _timestamp_second_counter(0), _intra_second_pseudo_usec_counter(0), _num_distributor_stripes(num_distributor_stripes), - _retrievedCommunicationManager(std::move(communicationManager)) + _retrievedCommunicationManager(std::move(communicationManager)), // may be nullptr + _bouncer(nullptr) { if (storage_chain_builder) { set_storage_chain_builder(std::move(storage_chain_builder)); @@ -82,19 +84,19 @@ void DistributorNode::createChain(IStorageChainBuilder &builder) { DistributorComponentRegister& dcr(_context.getComponentRegister()); - // TODO: All components in this chain should use a common thread instead of - // each having its own configfetcher. StorageLink::UP chain; if (_retrievedCommunicationManager) { builder.add(std::move(_retrievedCommunicationManager)); } else { - auto communication_manager = std::make_unique<CommunicationManager>(dcr, _configUri); + auto communication_manager = std::make_unique<CommunicationManager>(dcr, _configUri, communication_manager_config()); _communicationManager = communication_manager.get(); builder.add(std::move(communication_manager)); } std::unique_ptr<StateManager> stateManager(releaseStateManager()); - builder.add(std::make_unique<Bouncer>(dcr, _configUri)); + auto bouncer = std::make_unique<Bouncer>(dcr, bouncer_config()); + _bouncer = bouncer.get(); + builder.add(std::move(bouncer)); // Distributor instance registers a host info reporter with the state // manager, which is safe since the lifetime of said state manager // extends to the end of the process. @@ -140,4 +142,9 @@ DistributorNode::pause() return {}; } +void DistributorNode::on_bouncer_config_changed() { + assert(_bouncer); + _bouncer->on_configure(bouncer_config()); +} + } // storage diff --git a/storage/src/vespa/storage/storageserver/distributornode.h b/storage/src/vespa/storage/storageserver/distributornode.h index ac3cad30036..7870af95a0f 100644 --- a/storage/src/vespa/storage/storageserver/distributornode.h +++ b/storage/src/vespa/storage/storageserver/distributornode.h @@ -1,11 +1,4 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -/** - * \class storage::DistributorNode - * \ingroup storageserver - * - * \brief Class for setting up a distributor node. - */ - #pragma once #include "distributornodecontext.h" @@ -19,6 +12,7 @@ namespace storage { namespace distributor { class DistributorStripePool; } +class Bouncer; class IStorageChainBuilder; class DistributorNode @@ -34,6 +28,7 @@ class DistributorNode uint32_t _intra_second_pseudo_usec_counter; uint32_t _num_distributor_stripes; std::unique_ptr<StorageLink> _retrievedCommunicationManager; + Bouncer* _bouncer; // If the current wall clock is more than the below number of seconds into the // past when compared to the highest recorded wall clock second time stamp, abort @@ -47,6 +42,7 @@ public: DistributorNode(const config::ConfigUri & configUri, DistributorNodeContext&, + BootstrapConfigs bootstrap_configs, ApplicationGenerationFetcher& generationFetcher, uint32_t num_distributor_stripes, std::unique_ptr<StorageLink> communicationManager, @@ -65,6 +61,7 @@ private: void initializeNodeSpecific() override; void createChain(IStorageChainBuilder &builder) override; api::Timestamp generate_unique_timestamp() override; + void on_bouncer_config_changed() override; /** * Shut down necessary distributor-specific components before shutting diff --git a/storage/src/vespa/storage/storageserver/documentapiconverter.cpp b/storage/src/vespa/storage/storageserver/documentapiconverter.cpp index 04b3d8b6ce7..ca46e87285b 100644 --- a/storage/src/vespa/storage/storageserver/documentapiconverter.cpp +++ b/storage/src/vespa/storage/storageserver/documentapiconverter.cpp @@ -23,9 +23,8 @@ using document::BucketSpace; namespace storage { -DocumentApiConverter::DocumentApiConverter(const config::ConfigUri &configUri, - std::shared_ptr<const BucketResolver> bucketResolver) - : _priConverter(std::make_unique<PriorityConverter>(configUri)), +DocumentApiConverter::DocumentApiConverter(std::shared_ptr<const BucketResolver> bucketResolver) + : _priConverter(std::make_unique<PriorityConverter>()), _bucketResolver(std::move(bucketResolver)) {} diff --git a/storage/src/vespa/storage/storageserver/documentapiconverter.h b/storage/src/vespa/storage/storageserver/documentapiconverter.h index 5990d6f9017..96b119ff44e 100644 --- a/storage/src/vespa/storage/storageserver/documentapiconverter.h +++ b/storage/src/vespa/storage/storageserver/documentapiconverter.h @@ -22,18 +22,17 @@ class PriorityConverter; class DocumentApiConverter { public: - DocumentApiConverter(const config::ConfigUri &configUri, - std::shared_ptr<const BucketResolver> bucketResolver); + explicit DocumentApiConverter(std::shared_ptr<const BucketResolver> bucketResolver); ~DocumentApiConverter(); - std::unique_ptr<api::StorageCommand> toStorageAPI(documentapi::DocumentMessage& msg); - std::unique_ptr<api::StorageReply> toStorageAPI(documentapi::DocumentReply& reply, api::StorageCommand& originalCommand); + [[nodiscard]] std::unique_ptr<api::StorageCommand> toStorageAPI(documentapi::DocumentMessage& msg); + [[nodiscard]] std::unique_ptr<api::StorageReply> toStorageAPI(documentapi::DocumentReply& reply, api::StorageCommand& originalCommand); void transferReplyState(storage::api::StorageReply& from, mbus::Reply& to); - std::unique_ptr<mbus::Message> toDocumentAPI(api::StorageCommand& cmd); + [[nodiscard]] std::unique_ptr<mbus::Message> toDocumentAPI(api::StorageCommand& cmd); const PriorityConverter& getPriorityConverter() const { return *_priConverter; } // BucketResolver getter and setter are both thread safe. - std::shared_ptr<const BucketResolver> bucketResolver() const; + [[nodiscard]] std::shared_ptr<const BucketResolver> bucketResolver() const; void setBucketResolver(std::shared_ptr<const BucketResolver> resolver); private: mutable std::mutex _mutex; diff --git a/storage/src/vespa/storage/storageserver/mergethrottler.cpp b/storage/src/vespa/storage/storageserver/mergethrottler.cpp index 6e2f2a77d20..4cc2a7a89ab 100644 --- a/storage/src/vespa/storage/storageserver/mergethrottler.cpp +++ b/storage/src/vespa/storage/storageserver/mergethrottler.cpp @@ -179,7 +179,7 @@ MergeThrottler::MergeNodeSequence::chain_contains_this_node() const noexcept } MergeThrottler::MergeThrottler( - const config::ConfigUri & configUri, + const StorServerConfig& bootstrap_config, StorageComponentRegister& compReg) : StorageLink("Merge Throttler"), framework::HtmlStatusReporter("merges", "Merge Throttler"), @@ -190,7 +190,6 @@ MergeThrottler::MergeThrottler( _queueSequence(0), _messageLock(), _stateLock(), - _configFetcher(std::make_unique<config::ConfigFetcher>(configUri.getContext())), _metrics(std::make_unique<Metrics>()), _component(compReg, "mergethrottler"), _thread(), @@ -203,34 +202,33 @@ MergeThrottler::MergeThrottler( { _throttlePolicy->setMinWindowSize(20); _throttlePolicy->setMaxWindowSize(20); - _configFetcher->subscribe<StorServerConfig>(configUri.getConfigId(), this); - _configFetcher->start(); + on_configure(bootstrap_config); _component.registerStatusPage(*this); _component.registerMetric(*_metrics); } void -MergeThrottler::configure(std::unique_ptr<vespa::config::content::core::StorServerConfig> newConfig) +MergeThrottler::on_configure(const StorServerConfig& new_config) { std::lock_guard lock(_stateLock); - _use_dynamic_throttling = (newConfig->mergeThrottlingPolicy.type + _use_dynamic_throttling = (new_config.mergeThrottlingPolicy.type == StorServerConfig::MergeThrottlingPolicy::Type::DYNAMIC); - if (newConfig->maxMergesPerNode < 1) { + if (new_config.maxMergesPerNode < 1) { throw config::InvalidConfigException("Cannot have a max merge count of less than 1"); } - if (newConfig->maxMergeQueueSize < 0) { + if (new_config.maxMergeQueueSize < 0) { throw config::InvalidConfigException("Max merge queue size cannot be less than 0"); } - if (newConfig->resourceExhaustionMergeBackPressureDurationSecs < 0.0) { + if (new_config.resourceExhaustionMergeBackPressureDurationSecs < 0.0) { throw config::InvalidConfigException("Merge back-pressure duration cannot be less than 0"); } if (_use_dynamic_throttling) { - auto min_win_sz = std::max(newConfig->mergeThrottlingPolicy.minWindowSize, 1); - auto max_win_sz = std::max(newConfig->mergeThrottlingPolicy.maxWindowSize, 1); + auto min_win_sz = std::max(new_config.mergeThrottlingPolicy.minWindowSize, 1); + auto max_win_sz = std::max(new_config.mergeThrottlingPolicy.maxWindowSize, 1); if (min_win_sz > max_win_sz) { min_win_sz = max_win_sz; } - auto win_sz_increment = std::max(1.0, newConfig->mergeThrottlingPolicy.windowSizeIncrement); + auto win_sz_increment = std::max(1.0, new_config.mergeThrottlingPolicy.windowSizeIncrement); _throttlePolicy->setMinWindowSize(min_win_sz); _throttlePolicy->setMaxWindowSize(max_win_sz); _throttlePolicy->setWindowSizeIncrement(win_sz_increment); @@ -238,15 +236,14 @@ MergeThrottler::configure(std::unique_ptr<vespa::config::content::core::StorServ min_win_sz, max_win_sz, win_sz_increment); } else { // Use legacy config values when static throttling is enabled. - _throttlePolicy->setMinWindowSize(newConfig->maxMergesPerNode); - _throttlePolicy->setMaxWindowSize(newConfig->maxMergesPerNode); + _throttlePolicy->setMinWindowSize(new_config.maxMergesPerNode); + _throttlePolicy->setMaxWindowSize(new_config.maxMergesPerNode); } - LOG(debug, "Setting new max queue size to %d", - newConfig->maxMergeQueueSize); - _maxQueueSize = newConfig->maxMergeQueueSize; + LOG(debug, "Setting new max queue size to %d", new_config.maxMergeQueueSize); + _maxQueueSize = new_config.maxMergeQueueSize; _backpressure_duration = std::chrono::duration_cast<std::chrono::steady_clock::duration>( - std::chrono::duration<double>(newConfig->resourceExhaustionMergeBackPressureDurationSecs)); - _disable_queue_limits_for_chained_merges = newConfig->disableQueueLimitsForChainedMerges; + std::chrono::duration<double>(new_config.resourceExhaustionMergeBackPressureDurationSecs)); + _disable_queue_limits_for_chained_merges = new_config.disableQueueLimitsForChainedMerges; } MergeThrottler::~MergeThrottler() @@ -275,8 +272,6 @@ MergeThrottler::onOpen() void MergeThrottler::onClose() { - // Avoid getting config on shutdown - _configFetcher->close(); { std::lock_guard guard(_messageLock); // Note: used to prevent taking locks in different order if onFlush diff --git a/storage/src/vespa/storage/storageserver/mergethrottler.h b/storage/src/vespa/storage/storageserver/mergethrottler.h index 840f1df1177..5362c2f6df8 100644 --- a/storage/src/vespa/storage/storageserver/mergethrottler.h +++ b/storage/src/vespa/storage/storageserver/mergethrottler.h @@ -35,10 +35,11 @@ class AbortBucketOperationsCommand; class MergeThrottler : public framework::Runnable, public StorageLink, - public framework::HtmlStatusReporter, - private config::IFetcherCallback<vespa::config::content::core::StorServerConfig> + public framework::HtmlStatusReporter { public: + using StorServerConfig = vespa::config::content::core::StorServerConfig; + class MergeFailureMetrics : public metrics::MetricSet { public: metrics::SumMetric<metrics::LongCountMetric> sum; @@ -172,7 +173,6 @@ private: mutable std::mutex _messageLock; std::condition_variable _messageCond; mutable std::mutex _stateLock; - std::unique_ptr<config::ConfigFetcher> _configFetcher; // Messages pending to be processed by the worker thread std::vector<api::StorageMessage::SP> _messagesDown; std::vector<api::StorageMessage::SP> _messagesUp; @@ -190,7 +190,7 @@ public: * windowSizeIncrement used for allowing unit tests to start out with more * than 1 as their window size. */ - MergeThrottler(const config::ConfigUri & configUri, StorageComponentRegister&); + MergeThrottler(const StorServerConfig& bootstrap_config, StorageComponentRegister&); ~MergeThrottler() override; /** Implements document::Runnable::run */ @@ -204,6 +204,8 @@ public: bool onSetSystemState(const std::shared_ptr<api::SetSystemStateCommand>& stateCmd) override; + void on_configure(const StorServerConfig& new_config); + /* * When invoked, merges to the node will be BUSY-bounced by the throttler * for a configurable period of time instead of being processed. @@ -282,11 +284,6 @@ private: [[nodiscard]] bool isChainCompleted() const noexcept; }; - /** - * Callback method for config system (IFetcherCallback) - */ - void configure(std::unique_ptr<vespa::config::content::core::StorServerConfig> newConfig) override; - // NOTE: unless explicitly specified, all the below functions require // _sync lock to be held upon call (usually implicitly via MessageGuard) diff --git a/storage/src/vespa/storage/storageserver/priorityconverter.cpp b/storage/src/vespa/storage/storageserver/priorityconverter.cpp index 13ab572c561..fe7570ff53a 100644 --- a/storage/src/vespa/storage/storageserver/priorityconverter.cpp +++ b/storage/src/vespa/storage/storageserver/priorityconverter.cpp @@ -1,85 +1,91 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "priorityconverter.h" -#include <vespa/config/subscription/configuri.h> -#include <vespa/config/helper/configfetcher.hpp> - +#include <map> namespace storage { -PriorityConverter::PriorityConverter(const config::ConfigUri & configUri) - : _configFetcher(std::make_unique<config::ConfigFetcher>(configUri.getContext())) +PriorityConverter::PriorityConverter() + : _mapping(), + _reverse_mapping() { - _configFetcher->subscribe<vespa::config::content::core::StorPrioritymappingConfig>(configUri.getConfigId(), this); - _configFetcher->start(); + init_static_priority_mappings(); } PriorityConverter::~PriorityConverter() = default; -uint8_t -PriorityConverter::toStoragePriority(documentapi::Priority::Value documentApiPriority) const +void +PriorityConverter::init_static_priority_mappings() { - const uint32_t index(static_cast<uint32_t>(documentApiPriority)); - if (index >= PRI_ENUM_SIZE) { - return 255; - } + // Defaults from `stor-prioritymapping` config + constexpr uint8_t highest = 50; + constexpr uint8_t very_high = 60; + constexpr uint8_t high_1 = 70; + constexpr uint8_t high_2 = 80; + constexpr uint8_t high_3 = 90; + constexpr uint8_t normal_1 = 100; + constexpr uint8_t normal_2 = 110; + constexpr uint8_t normal_3 = 120; + constexpr uint8_t normal_4 = 130; + constexpr uint8_t normal_5 = 140; + constexpr uint8_t normal_6 = 150; + constexpr uint8_t low_1 = 160; + constexpr uint8_t low_2 = 170; + constexpr uint8_t low_3 = 180; + constexpr uint8_t very_low = 190; + constexpr uint8_t lowest = 200; - return _mapping[index]; -} + _mapping[documentapi::Priority::PRI_HIGHEST] = highest; + _mapping[documentapi::Priority::PRI_VERY_HIGH] = very_high; + _mapping[documentapi::Priority::PRI_HIGH_1] = high_1; + _mapping[documentapi::Priority::PRI_HIGH_2] = high_2; + _mapping[documentapi::Priority::PRI_HIGH_3] = high_3; + _mapping[documentapi::Priority::PRI_NORMAL_1] = normal_1; + _mapping[documentapi::Priority::PRI_NORMAL_2] = normal_2; + _mapping[documentapi::Priority::PRI_NORMAL_3] = normal_3; + _mapping[documentapi::Priority::PRI_NORMAL_4] = normal_4; + _mapping[documentapi::Priority::PRI_NORMAL_5] = normal_5; + _mapping[documentapi::Priority::PRI_NORMAL_6] = normal_6; + _mapping[documentapi::Priority::PRI_LOW_1] = low_1; + _mapping[documentapi::Priority::PRI_LOW_2] = low_2; + _mapping[documentapi::Priority::PRI_LOW_3] = low_3; + _mapping[documentapi::Priority::PRI_VERY_LOW] = very_low; + _mapping[documentapi::Priority::PRI_LOWEST] = lowest; -documentapi::Priority::Value -PriorityConverter::toDocumentPriority(uint8_t storagePriority) const -{ - std::lock_guard guard(_mutex); - std::map<uint8_t, documentapi::Priority::Value>::const_iterator iter = - _reverseMapping.lower_bound(storagePriority); + std::map<uint8_t, documentapi::Priority::Value> reverse_map_helper; + reverse_map_helper[highest] = documentapi::Priority::PRI_HIGHEST; + reverse_map_helper[very_high] = documentapi::Priority::PRI_VERY_HIGH; + reverse_map_helper[high_1] = documentapi::Priority::PRI_HIGH_1; + reverse_map_helper[high_2] = documentapi::Priority::PRI_HIGH_2; + reverse_map_helper[high_3] = documentapi::Priority::PRI_HIGH_3; + reverse_map_helper[normal_1] = documentapi::Priority::PRI_NORMAL_1; + reverse_map_helper[normal_2] = documentapi::Priority::PRI_NORMAL_2; + reverse_map_helper[normal_3] = documentapi::Priority::PRI_NORMAL_3; + reverse_map_helper[normal_4] = documentapi::Priority::PRI_NORMAL_4; + reverse_map_helper[normal_5] = documentapi::Priority::PRI_NORMAL_5; + reverse_map_helper[normal_6] = documentapi::Priority::PRI_NORMAL_6; + reverse_map_helper[low_1] = documentapi::Priority::PRI_LOW_1; + reverse_map_helper[low_2] = documentapi::Priority::PRI_LOW_2; + reverse_map_helper[low_3] = documentapi::Priority::PRI_LOW_3; + reverse_map_helper[very_low] = documentapi::Priority::PRI_VERY_LOW; + reverse_map_helper[lowest] = documentapi::Priority::PRI_LOWEST; - if (iter != _reverseMapping.end()) { - return iter->second; + // Precompute a 1-1 LUT to avoid having to lower-bound lookup values in a fixed map + _reverse_mapping.resize(256); + for (size_t i = 0; i < 256; ++i) { + auto iter = reverse_map_helper.lower_bound(static_cast<uint8_t>(i)); + _reverse_mapping[i] = (iter != reverse_map_helper.cend()) ? iter->second : documentapi::Priority::PRI_LOWEST; } - - return documentapi::Priority::PRI_LOWEST; } -void -PriorityConverter::configure(std::unique_ptr<vespa::config::content::core::StorPrioritymappingConfig> config) +uint8_t +PriorityConverter::toStoragePriority(documentapi::Priority::Value documentApiPriority) const noexcept { - // Data race free; _mapping is an array of std::atomic. - _mapping[documentapi::Priority::PRI_HIGHEST] = config->highest; - _mapping[documentapi::Priority::PRI_VERY_HIGH] = config->veryHigh; - _mapping[documentapi::Priority::PRI_HIGH_1] = config->high1; - _mapping[documentapi::Priority::PRI_HIGH_2] = config->high2; - _mapping[documentapi::Priority::PRI_HIGH_3] = config->high3; - _mapping[documentapi::Priority::PRI_NORMAL_1] = config->normal1; - _mapping[documentapi::Priority::PRI_NORMAL_2] = config->normal2; - _mapping[documentapi::Priority::PRI_NORMAL_3] = config->normal3; - _mapping[documentapi::Priority::PRI_NORMAL_4] = config->normal4; - _mapping[documentapi::Priority::PRI_NORMAL_5] = config->normal5; - _mapping[documentapi::Priority::PRI_NORMAL_6] = config->normal6; - _mapping[documentapi::Priority::PRI_LOW_1] = config->low1; - _mapping[documentapi::Priority::PRI_LOW_2] = config->low2; - _mapping[documentapi::Priority::PRI_LOW_3] = config->low3; - _mapping[documentapi::Priority::PRI_VERY_LOW] = config->veryLow; - _mapping[documentapi::Priority::PRI_LOWEST] = config->lowest; - - std::lock_guard guard(_mutex); - _reverseMapping.clear(); - _reverseMapping[config->highest] = documentapi::Priority::PRI_HIGHEST; - _reverseMapping[config->veryHigh] = documentapi::Priority::PRI_VERY_HIGH; - _reverseMapping[config->high1] = documentapi::Priority::PRI_HIGH_1; - _reverseMapping[config->high2] = documentapi::Priority::PRI_HIGH_2; - _reverseMapping[config->high3] = documentapi::Priority::PRI_HIGH_3; - _reverseMapping[config->normal1] = documentapi::Priority::PRI_NORMAL_1; - _reverseMapping[config->normal2] = documentapi::Priority::PRI_NORMAL_2; - _reverseMapping[config->normal3] = documentapi::Priority::PRI_NORMAL_3; - _reverseMapping[config->normal4] = documentapi::Priority::PRI_NORMAL_4; - _reverseMapping[config->normal5] = documentapi::Priority::PRI_NORMAL_5; - _reverseMapping[config->normal6] = documentapi::Priority::PRI_NORMAL_6; - _reverseMapping[config->low1] = documentapi::Priority::PRI_LOW_1; - _reverseMapping[config->low2] = documentapi::Priority::PRI_LOW_2; - _reverseMapping[config->low3] = documentapi::Priority::PRI_LOW_3; - _reverseMapping[config->veryLow] = documentapi::Priority::PRI_VERY_LOW; - _reverseMapping[config->lowest] = documentapi::Priority::PRI_LOWEST; + const auto index = static_cast<uint32_t>(documentApiPriority); + if (index >= PRI_ENUM_SIZE) { + return 255; + } + return _mapping[index]; } } // storage diff --git a/storage/src/vespa/storage/storageserver/priorityconverter.h b/storage/src/vespa/storage/storageserver/priorityconverter.h index 47326e54243..48c7424433b 100644 --- a/storage/src/vespa/storage/storageserver/priorityconverter.h +++ b/storage/src/vespa/storage/storageserver/priorityconverter.h @@ -2,50 +2,34 @@ #pragma once -#include <vespa/storage/config/config-stor-prioritymapping.h> -#include <vespa/config/helper/ifetchercallback.h> #include <vespa/documentapi/messagebus/priority.h> -#include <atomic> #include <array> -#include <mutex> - -namespace config { - class ConfigUri; - class ConfigFetcher; -} +#include <vector> namespace storage { -class PriorityConverter - : public config::IFetcherCallback< - vespa::config::content::core::StorPrioritymappingConfig> -{ +class PriorityConverter { public: - using Config = vespa::config::content::core::StorPrioritymappingConfig; - - explicit PriorityConverter(const config::ConfigUri& configUri); - ~PriorityConverter() override; + PriorityConverter(); + ~PriorityConverter(); /** Converts the given priority into a storage api priority number. */ - uint8_t toStoragePriority(documentapi::Priority::Value) const; + [[nodiscard]] uint8_t toStoragePriority(documentapi::Priority::Value) const noexcept; /** Converts the given priority into a document api priority number. */ - documentapi::Priority::Value toDocumentPriority(uint8_t) const; - - void configure(std::unique_ptr<Config> config) override; + [[nodiscard]] documentapi::Priority::Value toDocumentPriority(uint8_t storage_priority) const noexcept { + return _reverse_mapping[storage_priority]; + } private: - static_assert(documentapi::Priority::PRI_ENUM_SIZE == 16, - "Unexpected size of priority enumeration"); - static_assert(documentapi::Priority::PRI_LOWEST == 15, - "Priority enum value out of bounds"); - static constexpr size_t PRI_ENUM_SIZE = documentapi::Priority::PRI_ENUM_SIZE; + void init_static_priority_mappings(); - std::array<std::atomic<uint8_t>, PRI_ENUM_SIZE> _mapping; - std::map<uint8_t, documentapi::Priority::Value> _reverseMapping; - mutable std::mutex _mutex; + static_assert(documentapi::Priority::PRI_ENUM_SIZE == 16, "Unexpected size of priority enumeration"); + static_assert(documentapi::Priority::PRI_LOWEST == 15, "Priority enum value out of bounds"); + static constexpr size_t PRI_ENUM_SIZE = documentapi::Priority::PRI_ENUM_SIZE; - std::unique_ptr<config::ConfigFetcher> _configFetcher; + std::array<uint8_t, PRI_ENUM_SIZE> _mapping; + std::vector<documentapi::Priority::Value> _reverse_mapping; }; } // storage diff --git a/storage/src/vespa/storage/storageserver/rpc/shared_rpc_resources.cpp b/storage/src/vespa/storage/storageserver/rpc/shared_rpc_resources.cpp index 172084662e2..eb933f5eb2c 100644 --- a/storage/src/vespa/storage/storageserver/rpc/shared_rpc_resources.cpp +++ b/storage/src/vespa/storage/storageserver/rpc/shared_rpc_resources.cpp @@ -105,6 +105,10 @@ void SharedRpcResources::wait_until_slobrok_is_ready() { } } +void SharedRpcResources::sync_all_threads() { + _transport->sync(); +} + void SharedRpcResources::shutdown() { assert(!_shutdown); if (listen_port() > 0) { diff --git a/storage/src/vespa/storage/storageserver/rpc/shared_rpc_resources.h b/storage/src/vespa/storage/storageserver/rpc/shared_rpc_resources.h index 1da89dd8869..d8f7eefad53 100644 --- a/storage/src/vespa/storage/storageserver/rpc/shared_rpc_resources.h +++ b/storage/src/vespa/storage/storageserver/rpc/shared_rpc_resources.h @@ -42,6 +42,8 @@ public: // To be called after all RPC handlers have been registered. void start_server_and_register_slobrok(vespalib::stringref my_handle); + void sync_all_threads(); + void shutdown(); [[nodiscard]] int listen_port() const noexcept; // Only valid if server has been started diff --git a/storage/src/vespa/storage/storageserver/servicelayernode.cpp b/storage/src/vespa/storage/storageserver/servicelayernode.cpp index ba4d8a96120..0cce2c27e95 100644 --- a/storage/src/vespa/storage/storageserver/servicelayernode.cpp +++ b/storage/src/vespa/storage/storageserver/servicelayernode.cpp @@ -24,16 +24,33 @@ LOG_SETUP(".node.servicelayer"); namespace storage { -ServiceLayerNode::ServiceLayerNode(const config::ConfigUri & configUri, ServiceLayerNodeContext& context, +ServiceLayerNode::ServiceLayerBootstrapConfigs::ServiceLayerBootstrapConfigs() = default; +ServiceLayerNode::ServiceLayerBootstrapConfigs::~ServiceLayerBootstrapConfigs() = default; +ServiceLayerNode::ServiceLayerBootstrapConfigs::ServiceLayerBootstrapConfigs(ServiceLayerBootstrapConfigs&&) noexcept = default; +ServiceLayerNode::ServiceLayerBootstrapConfigs& +ServiceLayerNode::ServiceLayerBootstrapConfigs::operator=(ServiceLayerBootstrapConfigs&&) noexcept = default; + +ServiceLayerNode::ServiceLayerNode(const config::ConfigUri & configUri, + ServiceLayerNodeContext& context, + ServiceLayerBootstrapConfigs bootstrap_configs, ApplicationGenerationFetcher& generationFetcher, spi::PersistenceProvider& persistenceProvider, const VisitorFactory::Map& externalVisitors) - : StorageNode(configUri, context, generationFetcher, std::make_unique<HostInfo>()), + : StorageNode(configUri, context, std::move(bootstrap_configs.storage_bootstrap_configs), + generationFetcher, std::make_unique<HostInfo>()), _context(context), _persistenceProvider(persistenceProvider), _externalVisitors(externalVisitors), + _persistence_bootstrap_config(std::move(bootstrap_configs.persistence_cfg)), + _visitor_bootstrap_config(std::move(bootstrap_configs.visitor_cfg)), + _filestor_bootstrap_config(std::move(bootstrap_configs.filestor_cfg)), + _bouncer(nullptr), _bucket_manager(nullptr), + _changed_bucket_ownership_handler(nullptr), _fileStorManager(nullptr), + _merge_throttler(nullptr), + _visitor_manager(nullptr), + _modified_bucket_checker(nullptr), _init_has_been_called(false) { } @@ -85,20 +102,6 @@ ServiceLayerNode::~ServiceLayerNode() } void -ServiceLayerNode::subscribeToConfigs() -{ - StorageNode::subscribeToConfigs(); - _configFetcher.reset(new config::ConfigFetcher(_configUri.getContext())); -} - -void -ServiceLayerNode::removeConfigSubscriptions() -{ - StorageNode::removeConfigSubscriptions(); - _configFetcher.reset(); -} - -void ServiceLayerNode::initializeNodeSpecific() { // Give node state to mount point initialization, such that we can @@ -106,7 +109,7 @@ ServiceLayerNode::initializeNodeSpecific() NodeStateUpdater::Lock::SP lock(_component->getStateUpdater().grabStateChangeLock()); lib::NodeState ns(*_component->getStateUpdater().getReportedNodeState()); - ns.setCapacity(_serverConfig->nodeCapacity); + ns.setCapacity(server_config().nodeCapacity); LOG(debug, "Adjusting reported node state to include capacity: %s", ns.toString().c_str()); _component->getStateUpdater().setReportedNodeState(ns); } @@ -117,10 +120,10 @@ ServiceLayerNode::initializeNodeSpecific() void ServiceLayerNode::handleLiveConfigUpdate(const InitialGuard & initGuard) { - if (_newServerConfig) { + if (_server_config.staging) { bool updated = false; - vespa::config::content::core::StorServerConfigBuilder oldC(*_serverConfig); - StorServerConfig& newC(*_newServerConfig); + vespa::config::content::core::StorServerConfigBuilder oldC(*_server_config.active); + StorServerConfig& newC(*_server_config.staging); { updated = false; NodeStateUpdater::Lock::SP lock(_component->getStateUpdater().grabStateChangeLock()); @@ -132,7 +135,8 @@ ServiceLayerNode::handleLiveConfigUpdate(const InitialGuard & initGuard) ns.setCapacity(newC.nodeCapacity); } if (updated) { - _serverConfig.reset(new vespa::config::content::core::StorServerConfig(oldC)); + // FIXME this always gets overwritten by StorageNode::handleLiveConfigUpdate...! Intentional? + _server_config.active = std::make_unique<vespa::config::content::core::StorServerConfig>(oldC); _component->getStateUpdater().setReportedNodeState(ns); } } @@ -162,22 +166,31 @@ ServiceLayerNode::createChain(IStorageChainBuilder &builder) { ServiceLayerComponentRegister& compReg(_context.getComponentRegister()); - auto communication_manager = std::make_unique<CommunicationManager>(compReg, _configUri); + auto communication_manager = std::make_unique<CommunicationManager>(compReg, _configUri, communication_manager_config()); _communicationManager = communication_manager.get(); builder.add(std::move(communication_manager)); - builder.add(std::make_unique<Bouncer>(compReg, _configUri)); - auto merge_throttler_up = std::make_unique<MergeThrottler>(_configUri, compReg); - auto merge_throttler = merge_throttler_up.get(); + auto bouncer = std::make_unique<Bouncer>(compReg, bouncer_config()); + _bouncer = bouncer.get(); + builder.add(std::move(bouncer)); + auto merge_throttler_up = std::make_unique<MergeThrottler>(server_config(), compReg); + _merge_throttler = merge_throttler_up.get(); builder.add(std::move(merge_throttler_up)); - builder.add(std::make_unique<ChangedBucketOwnershipHandler>(_configUri, compReg)); - auto bucket_manager = std::make_unique<BucketManager>(_configUri, _context.getComponentRegister()); + auto bucket_ownership_handler = std::make_unique<ChangedBucketOwnershipHandler>(*_persistence_bootstrap_config, compReg); + _changed_bucket_ownership_handler = bucket_ownership_handler.get(); + builder.add(std::move(bucket_ownership_handler)); + auto bucket_manager = std::make_unique<BucketManager>(server_config(), _context.getComponentRegister()); _bucket_manager = bucket_manager.get(); builder.add(std::move(bucket_manager)); - builder.add(std::make_unique<VisitorManager>(_configUri, _context.getComponentRegister(), - static_cast<VisitorMessageSessionFactory &>(*this), _externalVisitors)); - builder.add(std::make_unique<ModifiedBucketChecker>(_context.getComponentRegister(), _persistenceProvider, _configUri)); + auto visitor_manager = std::make_unique<VisitorManager>(*_visitor_bootstrap_config, _context.getComponentRegister(), + static_cast<VisitorMessageSessionFactory &>(*this), _externalVisitors); + _visitor_manager = visitor_manager.get(); + builder.add(std::move(visitor_manager)); + auto bucket_checker = std::make_unique<ModifiedBucketChecker>(_context.getComponentRegister(), _persistenceProvider, server_config()); + _modified_bucket_checker = bucket_checker.get(); + builder.add(std::move(bucket_checker)); auto state_manager = releaseStateManager(); - auto filstor_manager = std::make_unique<FileStorManager>(_configUri, _persistenceProvider, _context.getComponentRegister(), + auto filstor_manager = std::make_unique<FileStorManager>(*_filestor_bootstrap_config, _persistenceProvider, + _context.getComponentRegister(), getDoneInitializeHandler(), state_manager->getHostInfo()); _fileStorManager = filstor_manager.get(); builder.add(std::move(filstor_manager)); @@ -186,8 +199,43 @@ ServiceLayerNode::createChain(IStorageChainBuilder &builder) // Lifetimes of all referenced components shall outlive the last call going // through the SPI, as queues are flushed and worker threads joined when // the storage link chain is closed prior to destruction. - auto error_listener = std::make_shared<ServiceLayerErrorListener>(*_component, *merge_throttler); + auto error_listener = std::make_shared<ServiceLayerErrorListener>(*_component, *_merge_throttler); _fileStorManager->error_wrapper().register_error_listener(std::move(error_listener)); + + // Purge config no longer needed + _persistence_bootstrap_config.reset(); + _visitor_bootstrap_config.reset(); + _filestor_bootstrap_config.reset(); +} + +void +ServiceLayerNode::on_configure(const StorServerConfig& config) +{ + assert(_merge_throttler); + _merge_throttler->on_configure(config); + assert(_modified_bucket_checker); + _modified_bucket_checker->on_configure(config); +} + +void +ServiceLayerNode::on_configure(const PersistenceConfig& config) +{ + assert(_changed_bucket_ownership_handler); + _changed_bucket_ownership_handler->on_configure(config); +} + +void +ServiceLayerNode::on_configure(const StorVisitorConfig& config) +{ + assert(_visitor_manager); + _visitor_manager->on_configure(config); +} + +void +ServiceLayerNode::on_configure(const StorFilestorConfig& config) +{ + assert(_fileStorManager); + _fileStorManager->on_configure(config); } ResumeGuard @@ -214,4 +262,9 @@ void ServiceLayerNode::perform_post_chain_creation_init_steps() { _fileStorManager->complete_internal_initialization(); } +void ServiceLayerNode::on_bouncer_config_changed() { + assert(_bouncer); + _bouncer->on_configure(bouncer_config()); +} + } // storage diff --git a/storage/src/vespa/storage/storageserver/servicelayernode.h b/storage/src/vespa/storage/storageserver/servicelayernode.h index 91a799c1295..ae39bb0805e 100644 --- a/storage/src/vespa/storage/storageserver/servicelayernode.h +++ b/storage/src/vespa/storage/storageserver/servicelayernode.h @@ -1,10 +1,4 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -/** - * \class storage::ServiceLayerNode - * \ingroup storageserver - * - * \brief Class for setting up a service layer node. - */ #pragma once @@ -12,16 +6,24 @@ #include "servicelayernodecontext.h" #include "storagenode.h" #include "vespa/vespalib/util/jsonstream.h" -#include <vespa/storage/visiting/visitormessagesessionfactory.h> -#include <vespa/storage/common/visitorfactory.h> +#include <vespa/config-persistence.h> +#include <vespa/config-stor-filestor.h> #include <vespa/storage/common/nodestateupdater.h> +#include <vespa/storage/common/visitorfactory.h> +#include <vespa/storage/visiting/config-stor-visitor.h> +#include <vespa/storage/visiting/visitormessagesessionfactory.h> namespace storage { namespace spi { struct PersistenceProvider; } +class Bouncer; class BucketManager; +class ChangedBucketOwnershipHandler; class FileStorManager; +class MergeThrottler; +class ModifiedBucketChecker; +class VisitorManager; class ServiceLayerNode : public StorageNode, @@ -29,21 +31,44 @@ class ServiceLayerNode private NodeStateReporter { - ServiceLayerNodeContext & _context; - spi::PersistenceProvider & _persistenceProvider; - VisitorFactory::Map _externalVisitors; - - // FIXME: Should probably use the fetcher in StorageNode - std::unique_ptr<config::ConfigFetcher> _configFetcher; - BucketManager * _bucket_manager; - FileStorManager * _fileStorManager; - bool _init_has_been_called; +public: + using PersistenceConfig = vespa::config::content::PersistenceConfig; + using StorVisitorConfig = vespa::config::content::core::StorVisitorConfig; + using StorFilestorConfig = vespa::config::content::StorFilestorConfig; +private: + ServiceLayerNodeContext& _context; + spi::PersistenceProvider& _persistenceProvider; + VisitorFactory::Map _externalVisitors; + std::unique_ptr<PersistenceConfig> _persistence_bootstrap_config; + std::unique_ptr<StorVisitorConfig> _visitor_bootstrap_config; + std::unique_ptr<StorFilestorConfig> _filestor_bootstrap_config; + Bouncer* _bouncer; + BucketManager* _bucket_manager; + ChangedBucketOwnershipHandler* _changed_bucket_ownership_handler; + FileStorManager* _fileStorManager; + MergeThrottler* _merge_throttler; + VisitorManager* _visitor_manager; + ModifiedBucketChecker* _modified_bucket_checker; + bool _init_has_been_called; public: using UP = std::unique_ptr<ServiceLayerNode>; + struct ServiceLayerBootstrapConfigs { + BootstrapConfigs storage_bootstrap_configs; + std::unique_ptr<PersistenceConfig> persistence_cfg; + std::unique_ptr<StorVisitorConfig> visitor_cfg; + std::unique_ptr<StorFilestorConfig> filestor_cfg; + + ServiceLayerBootstrapConfigs(); + ~ServiceLayerBootstrapConfigs(); + ServiceLayerBootstrapConfigs(ServiceLayerBootstrapConfigs&&) noexcept; + ServiceLayerBootstrapConfigs& operator=(ServiceLayerBootstrapConfigs&&) noexcept; + }; + ServiceLayerNode(const config::ConfigUri & configUri, ServiceLayerNodeContext& context, + ServiceLayerBootstrapConfigs bootstrap_configs, ApplicationGenerationFetcher& generationFetcher, spi::PersistenceProvider& persistenceProvider, const VisitorFactory::Map& externalVisitors); @@ -53,20 +78,24 @@ public: */ void init(); + void on_configure(const StorServerConfig& config); + void on_configure(const PersistenceConfig& config); + void on_configure(const StorVisitorConfig& config); + void on_configure(const StorFilestorConfig& config); + const lib::NodeType& getNodeType() const override { return lib::NodeType::STORAGE; } ResumeGuard pause() override; private: void report(vespalib::JsonStream &writer) const override; - void subscribeToConfigs() override; void initializeNodeSpecific() override; void perform_post_chain_creation_init_steps() override; void handleLiveConfigUpdate(const InitialGuard & initGuard) override; VisitorMessageSession::UP createSession(Visitor&, VisitorThread&) override; documentapi::Priority::Value toDocumentPriority(uint8_t storagePriority) const override; void createChain(IStorageChainBuilder &builder) override; - void removeConfigSubscriptions() override; + void on_bouncer_config_changed() override; }; } // storage diff --git a/storage/src/vespa/storage/storageserver/storagenode.cpp b/storage/src/vespa/storage/storageserver/storagenode.cpp index 3231deef268..f7a426a0527 100644 --- a/storage/src/vespa/storage/storageserver/storagenode.cpp +++ b/storage/src/vespa/storage/storageserver/storagenode.cpp @@ -66,14 +66,19 @@ removePidFile(const vespalib::string& pidfile) } // End of anonymous namespace +StorageNode::BootstrapConfigs::BootstrapConfigs() = default; +StorageNode::BootstrapConfigs::~BootstrapConfigs() = default; +StorageNode::BootstrapConfigs::BootstrapConfigs(BootstrapConfigs&&) noexcept = default; +StorageNode::BootstrapConfigs& StorageNode::BootstrapConfigs::operator=(BootstrapConfigs&&) noexcept = default; + StorageNode::StorageNode( const config::ConfigUri & configUri, StorageNodeContext& context, + BootstrapConfigs bootstrap_configs, ApplicationGenerationFetcher& generationFetcher, std::unique_ptr<HostInfo> hostInfo, RunMode mode) : _singleThreadedDebugMode(mode == SINGLE_THREADED_TEST_MODE), - _configFetcher(), _hostInfo(std::move(hostInfo)), _context(context), _generationFetcher(generationFetcher), @@ -90,16 +95,11 @@ StorageNode::StorageNode( _chain(), _configLock(), _initial_config_mutex(), - _serverConfig(), - _clusterConfig(), - _distributionConfig(), - _doctypesConfig(), - _bucketSpacesConfig(), - _newServerConfig(), - _newClusterConfig(), - _newDistributionConfig(), - _newDoctypesConfig(), - _newBucketSpacesConfig(), + _bouncer_config(std::move(bootstrap_configs.bouncer_cfg)), + _bucket_spaces_config(std::move(bootstrap_configs.bucket_spaces_cfg)), + _comm_mgr_config(std::move(bootstrap_configs.comm_mgr_cfg)), + _distribution_config(std::move(bootstrap_configs.distribution_cfg)), + _server_config(std::move(bootstrap_configs.server_cfg)), _component(), _node_identity(), _configUri(configUri), @@ -109,45 +109,24 @@ StorageNode::StorageNode( } void -StorageNode::subscribeToConfigs() -{ - _configFetcher = std::make_unique<config::ConfigFetcher>(_configUri.getContext()); - _configFetcher->subscribe<StorDistributionConfig>(_configUri.getConfigId(), this); - _configFetcher->subscribe<UpgradingConfig>(_configUri.getConfigId(), this); - _configFetcher->subscribe<StorServerConfig>(_configUri.getConfigId(), this); - _configFetcher->subscribe<BucketspacesConfig>(_configUri.getConfigId(), this); - - _configFetcher->start(); - - std::lock_guard configLockGuard(_configLock); - _serverConfig = std::move(_newServerConfig); - _clusterConfig = std::move(_newClusterConfig); - _distributionConfig = std::move(_newDistributionConfig); - _bucketSpacesConfig = std::move(_newBucketSpacesConfig); -} - -void StorageNode::initialize(const NodeStateReporter & nodeStateReporter) { // Avoid racing with concurrent reconfigurations before we've set up the entire // node component stack. + // TODO no longer needed... probably std::lock_guard<std::mutex> concurrent_config_guard(_initial_config_mutex); _context.getComponentRegister().registerShutdownListener(*this); - // Fetch configs needed first. These functions will just grab the config - // and store them away, while having the config lock. - subscribeToConfigs(); - // First update some basics that doesn't depend on anything else to be // available - _rootFolder = _serverConfig->rootFolder; + _rootFolder = server_config().rootFolder; - _context.getComponentRegister().setNodeInfo(_serverConfig->clusterName, getNodeType(), _serverConfig->nodeIndex); + _context.getComponentRegister().setNodeInfo(server_config().clusterName, getNodeType(), server_config().nodeIndex); _context.getComponentRegister().setBucketIdFactory(document::BucketIdFactory()); - _context.getComponentRegister().setDistribution(make_shared<lib::Distribution>(*_distributionConfig)); - _context.getComponentRegister().setBucketSpacesConfig(*_bucketSpacesConfig); - _node_identity = std::make_unique<NodeIdentity>(_serverConfig->clusterName, getNodeType(), _serverConfig->nodeIndex); + _context.getComponentRegister().setDistribution(make_shared<lib::Distribution>(distribution_config())); + _context.getComponentRegister().setBucketSpacesConfig(bucket_spaces_config()); + _node_identity = std::make_unique<NodeIdentity>(server_config().clusterName, getNodeType(), server_config().nodeIndex); _metrics = std::make_shared<StorageMetricSet>(); _component = std::make_unique<StorageComponent>(_context.getComponentRegister(), "storagenode"); @@ -184,17 +163,17 @@ StorageNode::initialize(const NodeStateReporter & nodeStateReporter) // Start deadlock detector _deadLockDetector = std::make_unique<DeadLockDetector>(_context.getComponentRegister()); - _deadLockDetector->enableWarning(_serverConfig->enableDeadLockDetectorWarnings); - _deadLockDetector->enableShutdown(_serverConfig->enableDeadLockDetector); - _deadLockDetector->setProcessSlack(vespalib::from_s(_serverConfig->deadLockDetectorTimeoutSlack)); - _deadLockDetector->setWaitSlack(vespalib::from_s(_serverConfig->deadLockDetectorTimeoutSlack)); + _deadLockDetector->enableWarning(server_config().enableDeadLockDetectorWarnings); + _deadLockDetector->enableShutdown(server_config().enableDeadLockDetector); + _deadLockDetector->setProcessSlack(vespalib::from_s(server_config().deadLockDetectorTimeoutSlack)); + _deadLockDetector->setWaitSlack(vespalib::from_s(server_config().deadLockDetectorTimeoutSlack)); createChain(*_chain_builder); _chain = std::move(*_chain_builder).build(); _chain_builder.reset(); assert(_communicationManager != nullptr); - _communicationManager->updateBucketSpacesConfig(*_bucketSpacesConfig); + _communicationManager->updateBucketSpacesConfig(bucket_spaces_config()); perform_post_chain_creation_init_steps(); @@ -256,23 +235,23 @@ StorageNode::handleLiveConfigUpdate(const InitialGuard & initGuard) // If we get here, initialize is done running. We have to handle changes // we want to handle. - if (_newServerConfig) { - StorServerConfigBuilder oldC(*_serverConfig); - StorServerConfig& newC(*_newServerConfig); + if (_server_config.staging) { + StorServerConfigBuilder oldC(*_server_config.active); + StorServerConfig& newC(*_server_config.staging); DIFFERWARN(rootFolder, "Cannot alter root folder of node live"); DIFFERWARN(clusterName, "Cannot alter cluster name of node live"); DIFFERWARN(nodeIndex, "Cannot alter node index of node live"); DIFFERWARN(isDistributor, "Cannot alter role of node live"); - _serverConfig = std::make_unique<StorServerConfig>(oldC); - _newServerConfig.reset(); - _deadLockDetector->enableWarning(_serverConfig->enableDeadLockDetectorWarnings); - _deadLockDetector->enableShutdown(_serverConfig->enableDeadLockDetector); - _deadLockDetector->setProcessSlack(vespalib::from_s(_serverConfig->deadLockDetectorTimeoutSlack)); - _deadLockDetector->setWaitSlack(vespalib::from_s(_serverConfig->deadLockDetectorTimeoutSlack)); - } - if (_newDistributionConfig) { - StorDistributionConfigBuilder oldC(*_distributionConfig); - StorDistributionConfig& newC(*_newDistributionConfig); + _server_config.active = std::make_unique<StorServerConfig>(oldC); // TODO this overwrites from ServiceLayerNode + _server_config.staging.reset(); + _deadLockDetector->enableWarning(server_config().enableDeadLockDetectorWarnings); + _deadLockDetector->enableShutdown(server_config().enableDeadLockDetector); + _deadLockDetector->setProcessSlack(vespalib::from_s(server_config().deadLockDetectorTimeoutSlack)); + _deadLockDetector->setWaitSlack(vespalib::from_s(server_config().deadLockDetectorTimeoutSlack)); + } + if (_distribution_config.staging) { + StorDistributionConfigBuilder oldC(*_distribution_config.active); + StorDistributionConfig& newC(*_distribution_config.staging); bool updated = false; if (DIFFER(redundancy)) { LOG(info, "Live config update: Altering redundancy from %u to %u.", oldC.redundancy, newC.redundancy); @@ -303,8 +282,9 @@ StorageNode::handleLiveConfigUpdate(const InitialGuard & initGuard) LOG(info, "Live config update: Group structure altered."); ASSIGN(group); } - _distributionConfig = std::make_unique<StorDistributionConfig>(oldC); - _newDistributionConfig.reset(); + // This looks weird, but the magical ASSIGN() macro mutates `oldC` in-place upon changes + _distribution_config.active = std::make_unique<StorDistributionConfig>(oldC); + _distribution_config.staging.reset(); if (updated) { _context.getComponentRegister().setDistribution(make_shared<lib::Distribution>(oldC)); for (StorageLink* link = _chain.get(); link != nullptr; link = link->getNextLink()) { @@ -312,17 +292,19 @@ StorageNode::handleLiveConfigUpdate(const InitialGuard & initGuard) } } } - if (_newClusterConfig) { - if (*_clusterConfig != *_newClusterConfig) { - LOG(warning, "Live config failure: Cannot alter cluster config of node live."); - } - _newClusterConfig.reset(); - } - if (_newBucketSpacesConfig) { - _bucketSpacesConfig = std::move(_newBucketSpacesConfig); - _context.getComponentRegister().setBucketSpacesConfig(*_bucketSpacesConfig); - _communicationManager->updateBucketSpacesConfig(*_bucketSpacesConfig); + if (_bucket_spaces_config.staging) { + _bucket_spaces_config.promote_staging_to_active(); + _context.getComponentRegister().setBucketSpacesConfig(bucket_spaces_config()); + _communicationManager->updateBucketSpacesConfig(bucket_spaces_config()); + } + if (_comm_mgr_config.staging) { + _comm_mgr_config.promote_staging_to_active(); + _communicationManager->on_configure(communication_manager_config()); + } + if (_bouncer_config.staging) { + _bouncer_config.promote_staging_to_active(); + on_bouncer_config_changed(); } } @@ -347,25 +329,16 @@ StorageNode::notifyDoneInitializing() StorageNode::~StorageNode() = default; void -StorageNode::removeConfigSubscriptions() -{ - LOG(debug, "Removing config subscribers"); - _configFetcher.reset(); -} - -void StorageNode::shutdown() { // Try to shut down in opposite order of initialize. Bear in mind that // we might be shutting down after init exception causing only parts - // of the server to have initialize + // of the server to have been initialized LOG(debug, "Shutting down storage node of type %s", getNodeType().toString().c_str()); if (!attemptedStopped()) { LOG(debug, "Storage killed before requestShutdown() was called. No " "reason has been given for why we're stopping."); } - // Remove the subscription to avoid more callbacks from config - removeConfigSubscriptions(); if (_chain) { LOG(debug, "Closing storage chain"); @@ -433,72 +406,42 @@ StorageNode::shutdown() void StorageNode::configure(std::unique_ptr<StorServerConfig> config) { - log_config_received(*config); - // When we get config, we try to grab the config lock to ensure noone - // else is doing configuration work, and then we write the new config - // to a variable where we can find it later when processing config - // updates - { - std::lock_guard configLockGuard(_configLock); - _newServerConfig = std::move(config); - } - if (_serverConfig) { - InitialGuard concurrent_config_guard(_initial_config_mutex); - handleLiveConfigUpdate(concurrent_config_guard); - } + stage_config_change(_server_config, std::move(config)); } void -StorageNode::configure(std::unique_ptr<UpgradingConfig> config) { - log_config_received(*config); - { - std::lock_guard configLockGuard(_configLock); - _newClusterConfig = std::move(config); - } - if (_clusterConfig) { - InitialGuard concurrent_config_guard(_initial_config_mutex); - handleLiveConfigUpdate(concurrent_config_guard); - } +StorageNode::configure(std::unique_ptr<StorDistributionConfig> config) { + stage_config_change(_distribution_config, std::move(config)); } void -StorageNode::configure(std::unique_ptr<StorDistributionConfig> config) { - log_config_received(*config); - { - std::lock_guard configLockGuard(_configLock); - _newDistributionConfig = std::move(config); - } - if (_distributionConfig) { - InitialGuard concurrent_config_guard(_initial_config_mutex); - handleLiveConfigUpdate(concurrent_config_guard); - } +StorageNode::configure(std::unique_ptr<BucketspacesConfig> config) { + stage_config_change(_bucket_spaces_config, std::move(config)); } + void -StorageNode::configure(std::unique_ptr<document::config::DocumenttypesConfig> config, - bool hasChanged, int64_t generation) -{ - log_config_received(*config); - (void) generation; - if (!hasChanged) - return; - { - std::lock_guard configLockGuard(_configLock); - _newDoctypesConfig = std::move(config); - } - if (_doctypesConfig) { - InitialGuard concurrent_config_guard(_initial_config_mutex); - handleLiveConfigUpdate(concurrent_config_guard); - } +StorageNode::configure(std::unique_ptr<CommunicationManagerConfig> config) { + stage_config_change(_comm_mgr_config, std::move(config)); } void -StorageNode::configure(std::unique_ptr<BucketspacesConfig> config) { - log_config_received(*config); +StorageNode::configure(std::unique_ptr<StorBouncerConfig> config) { + stage_config_change(_bouncer_config, std::move(config)); +} + +template <typename ConfigT> +void +StorageNode::stage_config_change(ConfigWrapper<ConfigT>& cfg, std::unique_ptr<ConfigT> new_cfg) { + log_config_received(*new_cfg); + // When we get config, we try to grab the config lock to ensure no one + // else is doing configuration work, and then we write the new config + // to a variable where we can find it later when processing config + // updates { - std::lock_guard configLockGuard(_configLock); - _newBucketSpacesConfig = std::move(config); + std::lock_guard config_lock_guard(_configLock); + cfg.staging = std::move(new_cfg); } - if (_bucketSpacesConfig) { + if (cfg.active) { InitialGuard concurrent_config_guard(_initial_config_mutex); handleLiveConfigUpdate(concurrent_config_guard); } @@ -564,4 +507,23 @@ StorageNode::set_storage_chain_builder(std::unique_ptr<IStorageChainBuilder> bui _chain_builder = std::move(builder); } +template <typename ConfigT> +StorageNode::ConfigWrapper<ConfigT>::ConfigWrapper() noexcept = default; + +template <typename ConfigT> +StorageNode::ConfigWrapper<ConfigT>::ConfigWrapper(std::unique_ptr<ConfigT> initial_active) noexcept + : staging(), + active(std::move(initial_active)) +{ +} + +template <typename ConfigT> +StorageNode::ConfigWrapper<ConfigT>::~ConfigWrapper() = default; + +template <typename ConfigT> +void StorageNode::ConfigWrapper<ConfigT>::promote_staging_to_active() noexcept { + assert(staging); + active = std::move(staging); +} + } // storage diff --git a/storage/src/vespa/storage/storageserver/storagenode.h b/storage/src/vespa/storage/storageserver/storagenode.h index 9538c2e1606..a96f6b52a66 100644 --- a/storage/src/vespa/storage/storageserver/storagenode.h +++ b/storage/src/vespa/storage/storageserver/storagenode.h @@ -1,9 +1,6 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. /** - * @class storage::StorageNode - * @ingroup storageserver - * - * @brief Main storage server class. + * Main storage server class. * * This class sets up the entire storage server. * @@ -12,13 +9,14 @@ #pragma once +#include <vespa/config-bucketspaces.h> #include <vespa/config-stor-distribution.h> -#include <vespa/config-upgrading.h> #include <vespa/config/helper/ifetchercallback.h> #include <vespa/config/subscription/configuri.h> #include <vespa/document/config/config-documenttypes.h> #include <vespa/storage/common/doneinitializehandler.h> -#include <vespa/config-bucketspaces.h> +#include <vespa/storage/config/config-stor-bouncer.h> +#include <vespa/storage/config/config-stor-communicationmanager.h> #include <vespa/storage/config/config-stor-server.h> #include <vespa/storage/storageutil/resumeguard.h> #include <vespa/storageframework/defaultimplementation/component/componentregisterimpl.h> @@ -51,33 +49,45 @@ struct StorageNodeContext; namespace lib { class NodeType; } -class StorageNode : private config::IFetcherCallback<vespa::config::content::core::StorServerConfig>, - private config::IFetcherCallback<vespa::config::content::StorDistributionConfig>, - private config::IFetcherCallback<vespa::config::content::UpgradingConfig>, - private config::IFetcherCallback<vespa::config::content::core::BucketspacesConfig>, - private framework::MetricUpdateHook, +class StorageNode : private framework::MetricUpdateHook, private DoneInitializeHandler, private framework::defaultimplementation::ShutdownListener { public: + using BucketspacesConfig = vespa::config::content::core::BucketspacesConfig; + using CommunicationManagerConfig = vespa::config::content::core::StorCommunicationmanagerConfig; + using StorBouncerConfig = vespa::config::content::core::StorBouncerConfig; + using StorDistributionConfig = vespa::config::content::StorDistributionConfig; + using StorServerConfig = vespa::config::content::core::StorServerConfig; + enum RunMode { NORMAL, SINGLE_THREADED_TEST_MODE }; StorageNode(const StorageNode &) = delete; StorageNode & operator = (const StorageNode &) = delete; - /** - * @param excludeStorageChain With this option set, no chain will be set - * up. This can be useful in unit testing if you need a storage server - * instance, but you want to have full control over the components yourself. - */ - StorageNode(const config::ConfigUri & configUri, + + struct BootstrapConfigs { + std::unique_ptr<StorBouncerConfig> bouncer_cfg; + std::unique_ptr<BucketspacesConfig> bucket_spaces_cfg; + std::unique_ptr<CommunicationManagerConfig> comm_mgr_cfg; + std::unique_ptr<StorDistributionConfig> distribution_cfg; + std::unique_ptr<StorServerConfig> server_cfg; + + BootstrapConfigs(); + ~BootstrapConfigs(); + BootstrapConfigs(BootstrapConfigs&&) noexcept; + BootstrapConfigs& operator=(BootstrapConfigs&&) noexcept; + }; + + StorageNode(const config::ConfigUri& configUri, StorageNodeContext& context, + BootstrapConfigs bootstrap_configs, ApplicationGenerationFetcher& generationFetcher, std::unique_ptr<HostInfo> hostInfo, RunMode = NORMAL); ~StorageNode() override; virtual const lib::NodeType& getNodeType() const = 0; - bool attemptedStopped() const; + [[nodiscard]] bool attemptedStopped() const; void notifyDoneInitializing() override; void waitUntilInitialized(vespalib::duration timeout = 15s); void updateMetrics(const MetricLockGuard & guard) override; @@ -93,18 +103,17 @@ public: void requestShutdown(vespalib::stringref reason) override; DoneInitializeHandler& getDoneInitializeHandler() { return *this; } + void configure(std::unique_ptr<StorServerConfig> config); + void configure(std::unique_ptr<StorDistributionConfig> config); + void configure(std::unique_ptr<BucketspacesConfig>); + void configure(std::unique_ptr<CommunicationManagerConfig> config); + void configure(std::unique_ptr<StorBouncerConfig> config); + // For testing StorageLink* getChain() { return _chain.get(); } virtual void initializeStatusWebServer(); -protected: - using StorServerConfig = vespa::config::content::core::StorServerConfig; - using UpgradingConfig = vespa::config::content::UpgradingConfig; - using StorDistributionConfig = vespa::config::content::StorDistributionConfig; - using BucketspacesConfig = vespa::config::content::core::BucketspacesConfig; private: bool _singleThreadedDebugMode; - // Subscriptions to config - std::unique_ptr<config::ConfigFetcher> _configFetcher; std::unique_ptr<HostInfo> _hostInfo; @@ -130,32 +139,49 @@ private: // The storage chain can depend on anything. std::unique_ptr<StorageLink> _chain; - /** Implementation of config callbacks. */ - void configure(std::unique_ptr<StorServerConfig> config) override; - void configure(std::unique_ptr<UpgradingConfig> config) override; - void configure(std::unique_ptr<StorDistributionConfig> config) override; - virtual void configure(std::unique_ptr<document::config::DocumenttypesConfig> config, - bool hasChanged, int64_t generation); - void configure(std::unique_ptr<BucketspacesConfig>) override; + template <typename ConfigT> + struct ConfigWrapper { + std::unique_ptr<ConfigT> staging; + std::unique_ptr<ConfigT> active; + + ConfigWrapper() noexcept; + explicit ConfigWrapper(std::unique_ptr<ConfigT> initial_active) noexcept; + ~ConfigWrapper(); + + void promote_staging_to_active() noexcept; + }; + + template <typename ConfigT> + void stage_config_change(ConfigWrapper<ConfigT>& my_cfg, std::unique_ptr<ConfigT> new_cfg); protected: // Lock taken while doing configuration of the server. std::mutex _configLock; - std::mutex _initial_config_mutex; + std::mutex _initial_config_mutex; // TODO can probably be removed using InitialGuard = std::lock_guard<std::mutex>; - // Current running config. Kept, such that we can see what has been - // changed in live config updates. - std::unique_ptr<StorServerConfig> _serverConfig; - std::unique_ptr<UpgradingConfig> _clusterConfig; - std::unique_ptr<StorDistributionConfig> _distributionConfig; - std::unique_ptr<document::config::DocumenttypesConfig> _doctypesConfig; - std::unique_ptr<BucketspacesConfig> _bucketSpacesConfig; - // New configs gotten that has yet to have been handled - std::unique_ptr<StorServerConfig> _newServerConfig; - std::unique_ptr<UpgradingConfig> _newClusterConfig; - std::unique_ptr<StorDistributionConfig> _newDistributionConfig; - std::unique_ptr<document::config::DocumenttypesConfig> _newDoctypesConfig; - std::unique_ptr<BucketspacesConfig> _newBucketSpacesConfig; + + ConfigWrapper<StorBouncerConfig> _bouncer_config; + ConfigWrapper<BucketspacesConfig> _bucket_spaces_config; + ConfigWrapper<CommunicationManagerConfig> _comm_mgr_config; + ConfigWrapper<StorDistributionConfig> _distribution_config; + ConfigWrapper<StorServerConfig> _server_config; + + [[nodiscard]] const StorBouncerConfig& bouncer_config() const noexcept { + return *_bouncer_config.active; + } + [[nodiscard]] const BucketspacesConfig& bucket_spaces_config() const noexcept { + return *_bucket_spaces_config.active; + } + [[nodiscard]] const CommunicationManagerConfig& communication_manager_config() const noexcept { + return *_comm_mgr_config.active; + } + [[nodiscard]] const StorDistributionConfig& distribution_config() const noexcept { + return *_distribution_config.active; + } + [[nodiscard]] const StorServerConfig& server_config() const noexcept { + return *_server_config.active; + } + std::unique_ptr<StorageComponent> _component; std::unique_ptr<NodeIdentity> _node_identity; config::ConfigUri _configUri; @@ -174,13 +200,13 @@ protected: std::unique_ptr<StateManager> releaseStateManager(); void initialize(const NodeStateReporter & reporter); - virtual void subscribeToConfigs(); virtual void initializeNodeSpecific() = 0; virtual void perform_post_chain_creation_init_steps() = 0; virtual void createChain(IStorageChainBuilder &builder) = 0; virtual void handleLiveConfigUpdate(const InitialGuard & initGuard); void shutdown(); - virtual void removeConfigSubscriptions(); + + virtual void on_bouncer_config_changed() { /* no-op by default */ } public: void set_storage_chain_builder(std::unique_ptr<IStorageChainBuilder> builder); }; diff --git a/storage/src/vespa/storage/visiting/visitormanager.cpp b/storage/src/vespa/storage/visiting/visitormanager.cpp index 1c26923b15f..dc1635bc4b1 100644 --- a/storage/src/vespa/storage/visiting/visitormanager.cpp +++ b/storage/src/vespa/storage/visiting/visitormanager.cpp @@ -21,7 +21,7 @@ LOG_SETUP(".visitor.manager"); namespace storage { -VisitorManager::VisitorManager(const config::ConfigUri & configUri, +VisitorManager::VisitorManager(const StorVisitorConfig& bootstrap_config, StorageComponentRegister& componentRegister, VisitorMessageSessionFactory& messageSF, VisitorFactory::Map externalFactories, @@ -35,7 +35,6 @@ VisitorManager::VisitorManager(const config::ConfigUri & configUri, _visitorLock(), _visitorCond(), _visitorCounter(0), - _configFetcher(std::make_unique<config::ConfigFetcher>(configUri.getContext())), _metrics(std::make_shared<VisitorMetrics>()), _maxFixedConcurrentVisitors(1), _maxVariableConcurrentVisitors(0), @@ -51,8 +50,7 @@ VisitorManager::VisitorManager(const config::ConfigUri & configUri, _enforceQueueUse(false), _visitorFactories(std::move(externalFactories)) { - _configFetcher->subscribe<vespa::config::content::core::StorVisitorConfig>(configUri.getConfigId(), this); - _configFetcher->start(); + on_configure(bootstrap_config); _component.registerMetric(*_metrics); if (!defer_manager_thread_start) { create_and_start_manager_thread(); @@ -94,8 +92,6 @@ VisitorManager::updateMetrics(const MetricLockGuard &) void VisitorManager::onClose() { - // Avoid getting config during shutdown - _configFetcher->close(); { std::lock_guard sync(_visitorLock); for (auto& enqueued : _visitorQueue) { @@ -118,25 +114,25 @@ VisitorManager::print(std::ostream& out, bool verbose, const std::string& indent } void -VisitorManager::configure(std::unique_ptr<vespa::config::content::core::StorVisitorConfig> config) +VisitorManager::on_configure(const vespa::config::content::core::StorVisitorConfig& config) { std::lock_guard sync(_visitorLock); - if (config->defaultdocblocksize % 512 != 0) { + if (config.defaultdocblocksize % 512 != 0) { throw config::InvalidConfigException( - "The default docblock size needs to be a multiplum of the " + "The default docblock size needs to be a multiple of the " "disk block size. (512b)"); } // Do some sanity checking of input. Cannot haphazardly mix and match // old and new max concurrency config values - if (config->maxconcurrentvisitors == 0 - && config->maxconcurrentvisitorsFixed == 0) + if (config.maxconcurrentvisitors == 0 + && config.maxconcurrentvisitorsFixed == 0) { throw config::InvalidConfigException( "Maximum concurrent visitor count cannot be 0."); } - else if (config->maxconcurrentvisitorsFixed == 0 - && config->maxconcurrentvisitorsVariable != 0) + else if (config.maxconcurrentvisitorsFixed == 0 + && config.maxconcurrentvisitorsVariable != 0) { throw config::InvalidConfigException( "Cannot specify 'variable' parameter for max concurrent " @@ -147,21 +143,21 @@ VisitorManager::configure(std::unique_ptr<vespa::config::content::core::StorVisi uint32_t maxConcurrentVisitorsVariable; // Concurrency parameter fixed takes precedence over legacy maxconcurrent - if (config->maxconcurrentvisitorsFixed > 0) { - maxConcurrentVisitorsFixed = config->maxconcurrentvisitorsFixed; - maxConcurrentVisitorsVariable = config->maxconcurrentvisitorsVariable; + if (config.maxconcurrentvisitorsFixed > 0) { + maxConcurrentVisitorsFixed = config.maxconcurrentvisitorsFixed; + maxConcurrentVisitorsVariable = config.maxconcurrentvisitorsVariable; } else { - maxConcurrentVisitorsFixed = config->maxconcurrentvisitors; + maxConcurrentVisitorsFixed = config.maxconcurrentvisitors; maxConcurrentVisitorsVariable = 0; } bool liveUpdate = !_visitorThread.empty(); if (liveUpdate) { - if (_visitorThread.size() != static_cast<uint32_t>(config->visitorthreads)) { + if (_visitorThread.size() != static_cast<uint32_t>(config.visitorthreads)) { LOG(warning, "Ignoring config change requesting %u visitor " "threads, still running %u. Restart storage to apply " "change.", - config->visitorthreads, + config.visitorthreads, (uint32_t) _visitorThread.size()); } @@ -174,18 +170,18 @@ VisitorManager::configure(std::unique_ptr<vespa::config::content::core::StorVisi maxConcurrentVisitorsFixed, maxConcurrentVisitorsVariable); } - if (_maxVisitorQueueSize != static_cast<uint32_t>(config->maxvisitorqueuesize)) { + if (_maxVisitorQueueSize != static_cast<uint32_t>(config.maxvisitorqueuesize)) { LOG(info, "Altered max visitor queue size setting from %u to %u.", - _maxVisitorQueueSize, config->maxvisitorqueuesize); + _maxVisitorQueueSize, config.maxvisitorqueuesize); } } else { - if (config->visitorthreads == 0) { + if (config.visitorthreads == 0) { throw config::InvalidConfigException( "No visitor threads configured. If you don't want visitors " "to run, don't use visitormanager.", VESPA_STRLOC); } - _metrics->initThreads(config->visitorthreads); - for (int32_t i=0; i<config->visitorthreads; ++i) { + _metrics->initThreads(config.visitorthreads); + for (int32_t i=0; i<config.visitorthreads; ++i) { _visitorThread.emplace_back( // Naked new due to a lot of private inheritance in VisitorThread and VisitorManager std::shared_ptr<VisitorThread>(new VisitorThread(i, _componentRegister, _messageSessionFactory, @@ -195,9 +191,9 @@ VisitorManager::configure(std::unique_ptr<vespa::config::content::core::StorVisi } _maxFixedConcurrentVisitors = maxConcurrentVisitorsFixed; _maxVariableConcurrentVisitors = maxConcurrentVisitorsVariable; - _maxVisitorQueueSize = config->maxvisitorqueuesize; + _maxVisitorQueueSize = config.maxvisitorqueuesize; - auto cmd = std::make_shared<PropagateVisitorConfig>(*config); + auto cmd = std::make_shared<PropagateVisitorConfig>(config); for (auto& thread : _visitorThread) { thread.first->processMessage(0, cmd); } diff --git a/storage/src/vespa/storage/visiting/visitormanager.h b/storage/src/vespa/storage/visiting/visitormanager.h index 9fe906d4465..fefa2c218ab 100644 --- a/storage/src/vespa/storage/visiting/visitormanager.h +++ b/storage/src/vespa/storage/visiting/visitormanager.h @@ -44,10 +44,11 @@ class VisitorManager : public framework::Runnable, public StorageLink, public framework::HtmlStatusReporter, private VisitorMessageHandler, - private config::IFetcherCallback<vespa::config::content::core::StorVisitorConfig>, private framework::MetricUpdateHook { private: + using StorVisitorConfig = vespa::config::content::core::StorVisitorConfig; + StorageComponentRegister& _componentRegister; VisitorMessageSessionFactory& _messageSessionFactory; std::vector<std::pair<std::shared_ptr<VisitorThread>, @@ -64,7 +65,6 @@ private: mutable std::mutex _visitorLock; std::condition_variable _visitorCond; uint64_t _visitorCounter; - std::unique_ptr<config::ConfigFetcher> _configFetcher; std::shared_ptr<VisitorMetrics> _metrics; uint32_t _maxFixedConcurrentVisitors; uint32_t _maxVariableConcurrentVisitors; @@ -82,7 +82,7 @@ private: bool _enforceQueueUse; VisitorFactory::Map _visitorFactories; public: - VisitorManager(const config::ConfigUri & configUri, + VisitorManager(const StorVisitorConfig& bootstrap_config, StorageComponentRegister&, VisitorMessageSessionFactory&, VisitorFactory::Map external = VisitorFactory::Map(), @@ -94,6 +94,8 @@ public: uint32_t getActiveVisitorCount() const; void setTimeBetweenTicks(uint32_t time); + void on_configure(const vespa::config::content::core::StorVisitorConfig&); + void setMaxConcurrentVisitors(uint32_t count) { // Used in unit testing _maxFixedConcurrentVisitors = count; _maxVariableConcurrentVisitors = 0; @@ -122,7 +124,6 @@ public: private: using MonitorGuard = std::unique_lock<std::mutex>; - void configure(std::unique_ptr<vespa::config::content::core::StorVisitorConfig>) override; void run(framework::ThreadHandle&) override; /** diff --git a/storage/src/vespa/storageapi/messageapi/storagemessage.cpp b/storage/src/vespa/storageapi/messageapi/storagemessage.cpp index 5a836ec1021..f62c6410882 100644 --- a/storage/src/vespa/storageapi/messageapi/storagemessage.cpp +++ b/storage/src/vespa/storageapi/messageapi/storagemessage.cpp @@ -13,7 +13,7 @@ namespace storage::api { namespace { -std::atomic<uint64_t> _G_lastMsgId(1000); +std::atomic<uint64_t> _g_lastMsgId(1000); } @@ -231,7 +231,7 @@ TransportContext::~TransportContext() = default; StorageMessage::Id StorageMessage::generateMsgId() noexcept { - return _G_lastMsgId.fetch_add(1, std::memory_order_relaxed); + return _g_lastMsgId.fetch_add(1, std::memory_order_relaxed); } StorageMessage::StorageMessage(const MessageType& type, Id internal_id, Id originator_id) noexcept diff --git a/storageserver/src/tests/storageservertest.cpp b/storageserver/src/tests/storageservertest.cpp index 2633449f28a..b18b241acaa 100644 --- a/storageserver/src/tests/storageservertest.cpp +++ b/storageserver/src/tests/storageservertest.cpp @@ -43,7 +43,7 @@ struct Node { struct Distributor : public Node { DistributorProcess _process; - Distributor(vdstestlib::DirConfig& config); + explicit Distributor(vdstestlib::DirConfig& config); ~Distributor() override; StorageNode& getNode() override { return _process.getNode(); } @@ -54,7 +54,7 @@ struct Storage : public Node { DummyServiceLayerProcess _process; StorageComponent::UP _component; - Storage(vdstestlib::DirConfig& config); + explicit Storage(vdstestlib::DirConfig& config); ~Storage() override; StorageNode& getNode() override { return _process.getNode(); } @@ -75,8 +75,7 @@ Storage::Storage(vdstestlib::DirConfig& config) { _process.setupConfig(60000ms); _process.createNode(); - _component = std::make_unique<StorageComponent>( - getContext().getComponentRegister(), "test"); + _component = std::make_unique<StorageComponent>(getContext().getComponentRegister(), "test"); } Storage::~Storage() = default; @@ -93,7 +92,6 @@ StorageServerTest::SetUp() storConfig = std::make_unique<vdstestlib::DirConfig>(getStandardConfig(true)); addSlobrokConfig(*distConfig, *slobrok); addSlobrokConfig(*storConfig, *slobrok); - storConfig->getConfig("stor-filestor").set("fail_disk_after_error_count", "1"); systemResult = system("mkdir -p vdsroot/disks/d0"); systemResult = system("mkdir -p vdsroot.distributor"); } @@ -101,6 +99,7 @@ StorageServerTest::SetUp() void StorageServerTest::TearDown() { + // TODO wipe temp dirs storConfig.reset(nullptr); distConfig.reset(nullptr); slobrok.reset(nullptr); diff --git a/storageserver/src/vespa/storageserver/app/distributorprocess.cpp b/storageserver/src/vespa/storageserver/app/distributorprocess.cpp index aad813a49fc..b56a4e1884b 100644 --- a/storageserver/src/vespa/storageserver/app/distributorprocess.cpp +++ b/storageserver/src/vespa/storageserver/app/distributorprocess.cpp @@ -61,9 +61,6 @@ DistributorProcess::setupConfig(vespalib::duration subscribeTimeout) using vespa::config::content::core::StorDistributormanagerConfig; using vespa::config::content::core::StorVisitordispatcherConfig; - auto distr_cfg = config::ConfigGetter<StorDistributormanagerConfig>::getConfig( - _configUri.getConfigId(), _configUri.getContext(), subscribeTimeout); - _num_distributor_stripes = adjusted_num_distributor_stripes(distr_cfg->numDistributorStripes); _distributorConfigHandler = _configSubscriber.subscribe<StorDistributormanagerConfig>(_configUri.getConfigId(), subscribeTimeout); _visitDispatcherConfigHandler = _configSubscriber.subscribe<StorVisitordispatcherConfig>(_configUri.getConfigId(), subscribeTimeout); Process::setupConfig(subscribeTimeout); @@ -99,8 +96,19 @@ DistributorProcess::configUpdated() void DistributorProcess::createNode() { - _node = std::make_unique<DistributorNode>(_configUri, _context, *this, _num_distributor_stripes, StorageLink::UP(), std::move(_storage_chain_builder)); - _node->handleConfigChange(*_distributorConfigHandler->getConfig()); + auto distributor_config = _distributorConfigHandler->getConfig(); + _num_distributor_stripes = adjusted_num_distributor_stripes(distributor_config->numDistributorStripes); + // TODO dedupe, consolidate + StorageNode::BootstrapConfigs bc; + bc.bucket_spaces_cfg = _bucket_spaces_cfg_handle->getConfig(); + bc.bouncer_cfg = _bouncer_cfg_handle->getConfig(); + bc.comm_mgr_cfg = _comm_mgr_cfg_handle->getConfig(); + bc.distribution_cfg = _distribution_cfg_handle->getConfig(); + bc.server_cfg = _server_cfg_handle->getConfig(); + + _node = std::make_unique<DistributorNode>(_configUri, _context, std::move(bc), *this, _num_distributor_stripes, + StorageLink::UP(), std::move(_storage_chain_builder)); + _node->handleConfigChange(*distributor_config); _node->handleConfigChange(*_visitDispatcherConfigHandler->getConfig()); } diff --git a/storageserver/src/vespa/storageserver/app/process.cpp b/storageserver/src/vespa/storageserver/app/process.cpp index 4b1586032e4..87b20f9ec2e 100644 --- a/storageserver/src/vespa/storageserver/app/process.cpp +++ b/storageserver/src/vespa/storageserver/app/process.cpp @@ -1,11 +1,12 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "process.h" + +#include <vespa/config/subscription/configsubscriber.hpp> #include <vespa/document/repo/document_type_repo_factory.h> #include <vespa/storage/storageserver/storagenode.h> #include <vespa/storage/storageserver/storagenodecontext.h> #include <vespa/vespalib/util/exceptions.h> -#include <vespa/config/subscription/configsubscriber.hpp> #include <vespa/log/log.h> LOG_SETUP(".process"); @@ -24,11 +25,17 @@ Process::~Process() = default; void Process::setupConfig(vespalib::duration subscribeTimeout) { - _documentHandler = _configSubscriber.subscribe<document::config::DocumenttypesConfig>(_configUri.getConfigId(), subscribeTimeout); + _document_cfg_handle = _configSubscriber.subscribe<DocumentTypesConfig>(_configUri.getConfigId(), subscribeTimeout); + _bucket_spaces_cfg_handle = _configSubscriber.subscribe<BucketspacesConfig>(_configUri.getConfigId(), subscribeTimeout); + _comm_mgr_cfg_handle = _configSubscriber.subscribe<CommunicationManagerConfig>(_configUri.getConfigId(), subscribeTimeout); + _bouncer_cfg_handle = _configSubscriber.subscribe<StorBouncerConfig>(_configUri.getConfigId(), subscribeTimeout); + _distribution_cfg_handle = _configSubscriber.subscribe<StorDistributionConfig>(_configUri.getConfigId(), subscribeTimeout); + _server_cfg_handle = _configSubscriber.subscribe<StorServerConfig>(_configUri.getConfigId(), subscribeTimeout); + if (!_configSubscriber.nextConfig()) { - throw vespalib::TimeoutException("Could not subscribe to document config within timeout"); + throw vespalib::TimeoutException("Could not subscribe to configs within timeout"); } - _repos.push_back(DocumentTypeRepoFactory::make(*_documentHandler->getConfig())); + _repos.push_back(DocumentTypeRepoFactory::make(*_document_cfg_handle->getConfig())); getContext().getComponentRegister().setDocumentTypeRepo(_repos.back()); } @@ -36,26 +43,46 @@ bool Process::configUpdated() { _configSubscriber.nextGenerationNow(); - if (_documentHandler->isChanged()) { + if (_document_cfg_handle->isChanged()) { LOG(info, "Document config detected changed"); return true; } - return false; + bool changed = (_bucket_spaces_cfg_handle->isChanged() + || _comm_mgr_cfg_handle->isChanged() + || _bouncer_cfg_handle->isChanged() + || _distribution_cfg_handle->isChanged() + || _server_cfg_handle->isChanged()); + return changed; } void Process::updateConfig() { - if (_documentHandler->isChanged()) { - _repos.push_back(DocumentTypeRepoFactory::make(*_documentHandler->getConfig())); + if (_document_cfg_handle->isChanged()) { + _repos.push_back(DocumentTypeRepoFactory::make(*_document_cfg_handle->getConfig())); getNode().setNewDocumentRepo(_repos.back()); } + if (_bucket_spaces_cfg_handle->isChanged()) { + getNode().configure(_bucket_spaces_cfg_handle->getConfig()); + } + if (_comm_mgr_cfg_handle->isChanged()) { + getNode().configure(_comm_mgr_cfg_handle->getConfig()); + } + if (_bouncer_cfg_handle->isChanged()) { + getNode().configure(_bouncer_cfg_handle->getConfig()); + } + if (_distribution_cfg_handle->isChanged()) { + getNode().configure(_distribution_cfg_handle->getConfig()); + } + if (_server_cfg_handle->isChanged()) { + getNode().configure(_server_cfg_handle->getConfig()); + } } void Process::shutdown() { - removeConfigSubscriptions(); + removeConfigSubscriptions(); // TODO remove? unused } int64_t diff --git a/storageserver/src/vespa/storageserver/app/process.h b/storageserver/src/vespa/storageserver/app/process.h index e70427c6e04..72b399ac870 100644 --- a/storageserver/src/vespa/storageserver/app/process.h +++ b/storageserver/src/vespa/storageserver/app/process.h @@ -8,16 +8,21 @@ * contains the process as a library such that it can be tested and used in * other pieces of code. * - * Specializations of this class will exist to add the funcionality needed for + * Specializations of this class will exist to add the functionality needed for * the various process types. */ #pragma once +#include <vespa/config-bucketspaces.h> +#include <vespa/config-stor-distribution.h> +#include <vespa/config/subscription/configsubscriber.h> +#include <vespa/config/subscription/configuri.h> #include <vespa/document/config/config-documenttypes.h> +#include <vespa/storage/config/config-stor-bouncer.h> +#include <vespa/storage/config/config-stor-communicationmanager.h> +#include <vespa/storage/config/config-stor-server.h> #include <vespa/storage/storageserver/applicationgenerationfetcher.h> -#include <vespa/config/subscription/configuri.h> -#include <vespa/config/subscription/configsubscriber.h> namespace document { class DocumentTypeRepo; } @@ -28,19 +33,32 @@ struct StorageNodeContext; class Process : public ApplicationGenerationFetcher { protected: + using DocumentTypesConfig = document::config::DocumenttypesConfig; + using BucketspacesConfig = vespa::config::content::core::BucketspacesConfig; + using CommunicationManagerConfig = vespa::config::content::core::StorCommunicationmanagerConfig; + using StorBouncerConfig = vespa::config::content::core::StorBouncerConfig; + using StorDistributionConfig = vespa::config::content::StorDistributionConfig; + using StorServerConfig = vespa::config::content::core::StorServerConfig; + using DocumentTypeRepoSP = std::shared_ptr<const document::DocumentTypeRepo>; config::ConfigUri _configUri; DocumentTypeRepoSP getTypeRepo() { return _repos.back(); } config::ConfigSubscriber _configSubscriber; + std::unique_ptr<config::ConfigHandle<DocumentTypesConfig>> _document_cfg_handle; + std::unique_ptr<config::ConfigHandle<BucketspacesConfig>> _bucket_spaces_cfg_handle; + std::unique_ptr<config::ConfigHandle<CommunicationManagerConfig>> _comm_mgr_cfg_handle; + std::unique_ptr<config::ConfigHandle<StorBouncerConfig>> _bouncer_cfg_handle; + std::unique_ptr<config::ConfigHandle<StorDistributionConfig>> _distribution_cfg_handle; + std::unique_ptr<config::ConfigHandle<StorServerConfig>> _server_cfg_handle; + private: - config::ConfigHandle<document::config::DocumenttypesConfig>::UP _documentHandler; std::vector<DocumentTypeRepoSP> _repos; public: using UP = std::unique_ptr<Process>; - Process(const config::ConfigUri & configUri); + explicit Process(const config::ConfigUri & configUri); ~Process() override; virtual void setupConfig(vespalib::duration subscribeTimeout); diff --git a/storageserver/src/vespa/storageserver/app/servicelayerprocess.cpp b/storageserver/src/vespa/storageserver/app/servicelayerprocess.cpp index 369cca4b166..bb284bfc108 100644 --- a/storageserver/src/vespa/storageserver/app/servicelayerprocess.cpp +++ b/storageserver/src/vespa/storageserver/app/servicelayerprocess.cpp @@ -34,6 +34,9 @@ bucket_db_options_from_config(const config::ConfigUri& config_uri) { ServiceLayerProcess::ServiceLayerProcess(const config::ConfigUri& configUri) : Process(configUri), _externalVisitors(), + _persistence_cfg_handle(), + _visitor_cfg_handle(), + _filestor_cfg_handle(), _node(), _storage_chain_builder(), _context(std::make_unique<framework::defaultimplementation::RealClock>(), @@ -51,11 +54,59 @@ ServiceLayerProcess::shutdown() } void +ServiceLayerProcess::setupConfig(vespalib::duration subscribe_timeout) +{ + _persistence_cfg_handle = _configSubscriber.subscribe<PersistenceConfig>(_configUri.getConfigId(), subscribe_timeout); + _visitor_cfg_handle = _configSubscriber.subscribe<StorVisitorConfig>(_configUri.getConfigId(), subscribe_timeout); + _filestor_cfg_handle = _configSubscriber.subscribe<StorFilestorConfig>(_configUri.getConfigId(), subscribe_timeout); + // We reuse the StorServerConfig subscription from the parent Process + Process::setupConfig(subscribe_timeout); +} + +void +ServiceLayerProcess::updateConfig() +{ + Process::updateConfig(); + if (_server_cfg_handle->isChanged()) { + _node->on_configure(*_server_cfg_handle->getConfig()); + } + if (_persistence_cfg_handle->isChanged()) { + _node->on_configure(*_persistence_cfg_handle->getConfig()); + } + if (_visitor_cfg_handle->isChanged()) { + _node->on_configure(*_visitor_cfg_handle->getConfig()); + } + if (_filestor_cfg_handle->isChanged()) { + _node->on_configure(*_filestor_cfg_handle->getConfig()); + } +} + +bool +ServiceLayerProcess::configUpdated() +{ + return Process::configUpdated(); +} + +void ServiceLayerProcess::createNode() { add_external_visitors(); setupProvider(); - _node = std::make_unique<ServiceLayerNode>(_configUri, _context, *this, getProvider(), _externalVisitors); + + StorageNode::BootstrapConfigs bc; + bc.bucket_spaces_cfg = _bucket_spaces_cfg_handle->getConfig(); + bc.bouncer_cfg = _bouncer_cfg_handle->getConfig(); + bc.comm_mgr_cfg = _comm_mgr_cfg_handle->getConfig(); + bc.distribution_cfg = _distribution_cfg_handle->getConfig(); + bc.server_cfg = _server_cfg_handle->getConfig(); + + ServiceLayerNode::ServiceLayerBootstrapConfigs sbc; + sbc.storage_bootstrap_configs = std::move(bc); + sbc.persistence_cfg = _persistence_cfg_handle->getConfig(); + sbc.visitor_cfg = _visitor_cfg_handle->getConfig(); + sbc.filestor_cfg = _filestor_cfg_handle->getConfig(); + + _node = std::make_unique<ServiceLayerNode>(_configUri, _context, std::move(sbc), *this, getProvider(), _externalVisitors); if (_storage_chain_builder) { _node->set_storage_chain_builder(std::move(_storage_chain_builder)); } diff --git a/storageserver/src/vespa/storageserver/app/servicelayerprocess.h b/storageserver/src/vespa/storageserver/app/servicelayerprocess.h index f95b952a68d..dcc56f373c4 100644 --- a/storageserver/src/vespa/storageserver/app/servicelayerprocess.h +++ b/storageserver/src/vespa/storageserver/app/servicelayerprocess.h @@ -1,24 +1,12 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -/** - * \class storage::ServiceLayerProcess - * - * \brief A process running a service layer. - */ -/** - * \class storage::MemFileServiceLayerProcess - * - * \brief A process running a service layer with memfile persistence provider. - */ -/** - * \class storage::RpcServiceLayerProcess - * - * \brief A process running a service layer with RPC persistence provider. - */ #pragma once #include "process.h" -#include <vespa/storage/storageserver/servicelayernodecontext.h> +#include <vespa/config-persistence.h> +#include <vespa/config-stor-filestor.h> #include <vespa/storage/common/visitorfactory.h> +#include <vespa/storage/storageserver/servicelayernodecontext.h> +#include <vespa/storage/visiting/config-stor-visitor.h> namespace config { class ConfigUri; } @@ -33,6 +21,14 @@ class ServiceLayerProcess : public Process { protected: VisitorFactory::Map _externalVisitors; private: + using PersistenceConfig = vespa::config::content::PersistenceConfig; + using StorVisitorConfig = vespa::config::content::core::StorVisitorConfig; + using StorFilestorConfig = vespa::config::content::StorFilestorConfig; + + std::unique_ptr<config::ConfigHandle<PersistenceConfig>> _persistence_cfg_handle; + std::unique_ptr<config::ConfigHandle<StorVisitorConfig>> _visitor_cfg_handle; + std::unique_ptr<config::ConfigHandle<StorFilestorConfig>> _filestor_cfg_handle; + std::unique_ptr<ServiceLayerNode> _node; std::unique_ptr<IStorageChainBuilder> _storage_chain_builder; @@ -45,6 +41,10 @@ public: void shutdown() override; + void setupConfig(vespalib::duration subscribe_timeout) override; + bool configUpdated() override; + void updateConfig() override; + virtual void setupProvider() = 0; virtual spi::PersistenceProvider& getProvider() = 0; diff --git a/streamingvisitors/src/vespa/vsm/vsm/docsumfilter.cpp b/streamingvisitors/src/vespa/vsm/vsm/docsumfilter.cpp index 23985fbce13..b94de154a35 100644 --- a/streamingvisitors/src/vespa/vsm/vsm/docsumfilter.cpp +++ b/streamingvisitors/src/vespa/vsm/vsm/docsumfilter.cpp @@ -132,6 +132,7 @@ public: } ~SnippetModifierJuniperConverter() override = default; void convert(const document::StringFieldValue &input, vespalib::slime::Inserter& inserter) override; + bool render_weighted_set_as_array() const override; }; void @@ -147,6 +148,12 @@ SnippetModifierJuniperConverter::convert(const document::StringFieldValue &input } } +bool +SnippetModifierJuniperConverter::render_weighted_set_as_array() const +{ + return false; +} + /** * Class providing access to a document retrieved from an IDocsumStore * (vsm::DocsumFilter). VSM specific transforms might be applied when @@ -172,7 +179,7 @@ public: DocsumStoreVsmDocument(DocsumFilter& docsum_filter, const Document& vsm_document); ~DocsumStoreVsmDocument() override; DocsumStoreFieldValue get_field_value(const vespalib::string& field_name) const override; - void insert_summary_field(const vespalib::string& field_name, vespalib::slime::Inserter& inserter) const override; + void insert_summary_field(const vespalib::string& field_name, vespalib::slime::Inserter& inserter, IStringFieldConverter* converter) const override; void insert_juniper_field(const vespalib::string& field_name, vespalib::slime::Inserter& inserter, IJuniperConverter& converter) const override; void insert_document_id(vespalib::slime::Inserter& inserter) const override; }; @@ -212,13 +219,13 @@ DocsumStoreVsmDocument::get_field_value(const vespalib::string& field_name) cons } void -DocsumStoreVsmDocument::insert_summary_field(const vespalib::string& field_name, vespalib::slime::Inserter& inserter) const +DocsumStoreVsmDocument::insert_summary_field(const vespalib::string& field_name, vespalib::slime::Inserter& inserter, IStringFieldConverter* converter) const { if (_document != nullptr) { auto entry_idx = _result_class.getIndexFromName(field_name.c_str()); if (entry_idx >= 0) { assert((uint32_t) entry_idx < _result_class.getNumEntries()); - _docsum_filter.insert_summary_field(entry_idx, _vsm_document, inserter); + _docsum_filter.insert_summary_field(entry_idx, _vsm_document, inserter, converter); return; } try { @@ -226,7 +233,7 @@ DocsumStoreVsmDocument::insert_summary_field(const vespalib::string& field_name, auto value(field.getDataType().createFieldValue()); if (value) { if (_document->getValue(field, *value)) { - SlimeFiller::insert_summary_field(*value, inserter); + SlimeFiller::insert_summary_field(*value, inserter, converter); } } } catch (document::FieldNotFoundException&) { @@ -393,14 +400,14 @@ DocsumFilter::get_summary_field(uint32_t entry_idx, const Document& doc) } void -DocsumFilter::insert_summary_field(uint32_t entry_idx, const Document& doc, vespalib::slime::Inserter& inserter) +DocsumFilter::insert_summary_field(uint32_t entry_idx, const Document& doc, vespalib::slime::Inserter& inserter, IStringFieldConverter* converter) { const auto& field_spec = _fields[entry_idx]; auto single_source_field_id = get_single_source_field_id(field_spec); if (single_source_field_id.has_value()) { auto field_value = doc.getField(single_source_field_id.value()); if (field_value != nullptr) { - SlimeFiller::insert_summary_field_with_field_filter(*field_value, inserter, field_spec.get_filter()); + SlimeFiller::insert_summary_field_with_field_filter(*field_value, inserter, converter, field_spec.get_filter()); } return; } diff --git a/streamingvisitors/src/vespa/vsm/vsm/docsumfilter.h b/streamingvisitors/src/vespa/vsm/vsm/docsumfilter.h index db35fa2ff27..2232d723781 100644 --- a/streamingvisitors/src/vespa/vsm/vsm/docsumfilter.h +++ b/streamingvisitors/src/vespa/vsm/vsm/docsumfilter.h @@ -13,6 +13,8 @@ using search::docsummary::IDocsumStore; +namespace search::docsummary { class IStringFieldConverter; } + namespace vsm { /** @@ -66,7 +68,7 @@ public: std::unique_ptr<const search::docsummary::IDocsumStoreDocument> get_document(uint32_t id) override; search::docsummary::DocsumStoreFieldValue get_summary_field(uint32_t entry_idx, const Document& doc); - void insert_summary_field(uint32_t entry_idx, const Document& doc, vespalib::slime::Inserter& inserter); + void insert_summary_field(uint32_t entry_idx, const Document& doc, vespalib::slime::Inserter& inserter, search::docsummary::IStringFieldConverter* converter); bool has_flatten_juniper_command(uint32_t entry_idx) const; FieldModifier* get_field_modifier(uint32_t entry_idx); }; diff --git a/vespa-dependencies-enforcer/allowed-maven-dependencies.txt b/vespa-dependencies-enforcer/allowed-maven-dependencies.txt index 619b032d24f..1fc7fabc161 100644 --- a/vespa-dependencies-enforcer/allowed-maven-dependencies.txt +++ b/vespa-dependencies-enforcer/allowed-maven-dependencies.txt @@ -20,6 +20,7 @@ com.fasterxml.jackson.core:jackson-databind:${jackson-databind.vespa.version} com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:${jackson2.vespa.version} com.fasterxml.jackson.datatype:jackson-datatype-jdk8:${jackson2.vespa.version} com.fasterxml.jackson.datatype:jackson-datatype-jsr310:${jackson2.vespa.version} +com.github.cliftonlabs:json-simple:${findbugs.vespa.version} com.github.luben:zstd-jni:${luben.zstd.vespa.version} com.github.spotbugs:spotbugs-annotations:3.1.9 com.google.code.findbugs:jsr305:${findbugs.vespa.version} @@ -32,7 +33,7 @@ com.google.jimfs:jimfs:${jimfs.vespa.version} com.google.protobuf:protobuf-java:${protobuf.vespa.version} com.ibm.icu:icu4j:${icu4j.vespa.version} com.microsoft.onnxruntime:onnxruntime:${onnxruntime.vespa.version} -com.sun.activation:javax.activation:${properties-maven-plugin.vespa.version} +com.sun.activation:javax.activation:1.2.0 com.sun.istack:istack-commons-runtime:4.1.2 com.sun.xml.bind:jaxb-core:${jaxb-core.vespa.version} com.sun.xml.bind:jaxb-impl:${jaxb-impl.vespa.version} @@ -43,8 +44,10 @@ com.yahoo.athenz:athenz-zms-core:${athenz.vespa.version} com.yahoo.athenz:athenz-zpe-java-client:${athenz.vespa.version} com.yahoo.athenz:athenz-zts-core:${athenz.vespa.version} com.yahoo.rdl:rdl-java:1.5.4 -commons-cli:commons-cli:1.5.0 +commons-beanutils:commons-beanutils:${commons-beanutils.vespa.version} +commons-cli:commons-cli:${commons-cli.vespa.version} commons-codec:commons-codec:${commons-codec.vespa.version} +commons-collections:commons-collections:${commons-collections.vespa.version} commons-fileupload:commons-fileupload:1.5 commons-io:commons-io:${commons-io.vespa.version} commons-logging:commons-logging:${commons-logging.vespa.version} @@ -70,7 +73,7 @@ io.prometheus:simpleclient_tracer_common:${prometheus.client.vespa.version} io.prometheus:simpleclient_tracer_otel:${prometheus.client.vespa.version} io.prometheus:simpleclient_tracer_otel_agent:${prometheus.client.vespa.version} jakarta.inject:jakarta.inject-api:${jakarta.inject.vespa.version} -javax.activation:javax.activation-api:${properties-maven-plugin.vespa.version} +javax.activation:javax.activation-api:1.2.0 javax.annotation:javax.annotation-api:${commons-logging.vespa.version} javax.inject:javax.inject:${javax.inject.vespa.version} javax.servlet:javax.servlet-api:${javax.servlet-api.vespa.version} @@ -87,6 +90,7 @@ org.antlr:antlr4-runtime:${antlr4.vespa.version} org.apache.aries.spifly:org.apache.aries.spifly.dynamic.bundle:${spifly.vespa.version} org.apache.commons:commons-compress:${commons-compress.vespa.version} org.apache.commons:commons-csv:${commons-csv.vespa.version} +org.apache.commons:commons-digester3:${commons-digester.vespa.version} org.apache.commons:commons-exec:${commons-exec.vespa.version} org.apache.commons:commons-lang3:${commons-lang3.vespa.version} org.apache.commons:commons-math3:${commons.math3.vespa.version} @@ -119,7 +123,8 @@ org.apache.maven:maven-project:2.2.1 org.apache.maven:maven-repository-metadata:${maven-core.vespa.version} org.apache.maven:maven-settings:${maven-core.vespa.version} org.apache.opennlp:opennlp-tools:${opennlp.vespa.version} -org.apache.velocity:velocity-engine-core:2.3 +org.apache.velocity.tools:velocity-tools-generic:${velocity.tools.vespa.version} +org.apache.velocity:velocity-engine-core:${velocity.vespa.version} org.apache.yetus:audience-annotations:0.12.0 org.apache.zookeeper:zookeeper-jute:${zookeeper.client.vespa.version} org.apache.zookeeper:zookeeper-jute:3.8.1 diff --git a/vespa-enforcer-extensions/src/main/java/com/yahoo/vespa/maven/plugin/enforcer/AllowedDependencies.java b/vespa-enforcer-extensions/src/main/java/com/yahoo/vespa/maven/plugin/enforcer/AllowedDependencies.java index 9c943bb2341..77f097e3b88 100644 --- a/vespa-enforcer-extensions/src/main/java/com/yahoo/vespa/maven/plugin/enforcer/AllowedDependencies.java +++ b/vespa-enforcer-extensions/src/main/java/com/yahoo/vespa/maven/plugin/enforcer/AllowedDependencies.java @@ -76,7 +76,11 @@ public class AllowedDependencies extends AbstractEnforcerRule implements Enforce var spec = loadDependencySpec(specFile); var resolved = resolve(spec, dependencies); if (System.getProperties().containsKey(WRITE_SPEC_PROP)) { - writeDependencySpec(specFile, resolved, System.getProperties().containsKey(GUESS_VERSION)); + // Guess property for version by default, can be disabled with <prop>=false + var guessProperty = Optional.ofNullable(System.getProperty(GUESS_VERSION)) + .map(p -> p.isEmpty() || Boolean.parseBoolean(p)) + .orElse(true); + writeDependencySpec(specFile, resolved, guessProperty); getLog().info("Updated spec file '%s'".formatted(specFile.toString())); } else { warnOnDuplicateVersions(resolved); diff --git a/vespa-feed-client/src/main/sh/vespa-version-generator.sh b/vespa-feed-client/src/main/sh/vespa-version-generator.sh index ee46a8debed..cd4684eb24d 100755 --- a/vespa-feed-client/src/main/sh/vespa-version-generator.sh +++ b/vespa-feed-client/src/main/sh/vespa-version-generator.sh @@ -18,7 +18,7 @@ versionNumber=$(cat $source | grep V_TAG_COMPONENT | awk '{print $2}' ) cat > $destination <<- END package ai.vespa.feed.client.impl; -class Vespa { - static final String VERSION = "$versionNumber"; +public class Vespa { + public static final String VERSION = "$versionNumber"; } END diff --git a/vespajlib/abi-spec.json b/vespajlib/abi-spec.json index 1c19c2ba5d6..3e588e24d47 100644 --- a/vespajlib/abi-spec.json +++ b/vespajlib/abi-spec.json @@ -4045,7 +4045,8 @@ "abstract" ], "methods" : [ - "public abstract java.util.List complete(ai.vespa.llm.completion.Prompt)" + "public abstract java.util.List complete(ai.vespa.llm.completion.Prompt)", + "public abstract java.util.concurrent.CompletableFuture completeAsync(ai.vespa.llm.completion.Prompt, java.util.function.Consumer)" ], "fields" : [ ] }, @@ -4059,6 +4060,7 @@ "public void <init>(java.lang.String)", "public ai.vespa.llm.client.openai.OpenAiClient$Builder model(java.lang.String)", "public ai.vespa.llm.client.openai.OpenAiClient$Builder temperature(double)", + "public ai.vespa.llm.client.openai.OpenAiClient$Builder maxTokens(long)", "public ai.vespa.llm.client.openai.OpenAiClient build()" ], "fields" : [ ] @@ -4072,7 +4074,8 @@ "public" ], "methods" : [ - "public java.util.List complete(ai.vespa.llm.completion.Prompt)" + "public java.util.List complete(ai.vespa.llm.completion.Prompt)", + "public java.util.concurrent.CompletableFuture completeAsync(ai.vespa.llm.completion.Prompt, java.util.function.Consumer)" ], "fields" : [ ] }, @@ -4090,7 +4093,8 @@ ], "fields" : [ "public static final enum ai.vespa.llm.completion.Completion$FinishReason length", - "public static final enum ai.vespa.llm.completion.Completion$FinishReason stop" + "public static final enum ai.vespa.llm.completion.Completion$FinishReason stop", + "public static final enum ai.vespa.llm.completion.Completion$FinishReason none" ] }, "ai.vespa.llm.completion.Completion" : { @@ -4167,7 +4171,8 @@ ], "methods" : [ "public void <init>(ai.vespa.llm.test.MockLanguageModel$Builder)", - "public java.util.List complete(ai.vespa.llm.completion.Prompt)" + "public java.util.List complete(ai.vespa.llm.completion.Prompt)", + "public java.util.concurrent.CompletableFuture completeAsync(ai.vespa.llm.completion.Prompt, java.util.function.Consumer)" ], "fields" : [ ] } diff --git a/vespajlib/src/main/java/ai/vespa/llm/LanguageModel.java b/vespajlib/src/main/java/ai/vespa/llm/LanguageModel.java index bd9004a659b..f4b8938934b 100644 --- a/vespajlib/src/main/java/ai/vespa/llm/LanguageModel.java +++ b/vespajlib/src/main/java/ai/vespa/llm/LanguageModel.java @@ -6,6 +6,8 @@ import ai.vespa.llm.completion.Prompt; import com.yahoo.api.annotations.Beta; import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.function.Consumer; /** * Interface to language models. @@ -17,4 +19,6 @@ public interface LanguageModel { List<Completion> complete(Prompt prompt); + CompletableFuture<Completion.FinishReason> completeAsync(Prompt prompt, Consumer<Completion> action); + } diff --git a/vespajlib/src/main/java/ai/vespa/llm/client/openai/OpenAiClient.java b/vespajlib/src/main/java/ai/vespa/llm/client/openai/OpenAiClient.java index efa8927988c..d7334b40963 100644 --- a/vespajlib/src/main/java/ai/vespa/llm/client/openai/OpenAiClient.java +++ b/vespajlib/src/main/java/ai/vespa/llm/client/openai/OpenAiClient.java @@ -18,25 +18,34 @@ import java.net.http.HttpRequest; import java.net.http.HttpResponse; import java.util.ArrayList; import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.function.Consumer; +import java.util.stream.Collectors; +import java.util.stream.Stream; /** * A client to the OpenAI language model API. Refer to https://platform.openai.com/docs/api-reference/. - * Currently only completions are implemented. + * Currently, only completions are implemented. * * @author bratseth */ @Beta public class OpenAiClient implements LanguageModel { + private static final String DATA_FIELD = "data: "; + private final String token; private final String model; private final double temperature; + private final long maxTokens; + private final HttpClient httpClient; private OpenAiClient(Builder builder) { this.token = builder.token; this.model = builder.model; this.temperature = builder.temperature; + this.maxTokens = builder.maxTokens; this.httpClient = HttpClient.newBuilder().build(); } @@ -54,13 +63,63 @@ public class OpenAiClient implements LanguageModel { } } + @Override + public CompletableFuture<Completion.FinishReason> completeAsync(Prompt prompt, Consumer<Completion> consumer) { + try { + var request = toRequest(prompt, true); + var futureResponse = httpClient.sendAsync(request, HttpResponse.BodyHandlers.ofLines()); + var completionFuture = new CompletableFuture<Completion.FinishReason>(); + + futureResponse.thenAcceptAsync(response -> { + try { + int responseCode = response.statusCode(); + if (responseCode != 200) { + throw new IllegalArgumentException("Received code " + responseCode + ": " + + response.body().collect(Collectors.joining())); + } + + Stream<String> lines = response.body(); + lines.forEach(line -> { + if (line.startsWith(DATA_FIELD)) { + var root = SlimeUtils.jsonToSlime(line.substring(DATA_FIELD.length())).get(); + var completion = toCompletions(root, "delta").get(0); + consumer.accept(completion); + if (!completion.finishReason().equals(Completion.FinishReason.none)) { + completionFuture.complete(completion.finishReason()); + } + } + }); + } catch (Exception e) { + completionFuture.completeExceptionally(e); + } + }); + return completionFuture; + + } catch (Exception e) { + throw new RuntimeException(e); + } + } + private HttpRequest toRequest(Prompt prompt) throws IOException, URISyntaxException { + return toRequest(prompt, false); + } + + private HttpRequest toRequest(Prompt prompt, boolean stream) throws IOException, URISyntaxException { var slime = new Slime(); var root = slime.setObject(); root.setString("model", model); root.setDouble("temperature", temperature); - root.setString("prompt", prompt.asString()); - return HttpRequest.newBuilder(new URI("https://api.openai.com/v1/completions")) + root.setBool("stream", stream); + root.setLong("n", 1); + if (maxTokens > 0) { + root.setLong("max_tokens", maxTokens); + } + var messagesArray = root.setArray("messages"); + var messagesObject = messagesArray.addObject(); + messagesObject.setString("role", "user"); + messagesObject.setString("content", prompt.asString()); + + return HttpRequest.newBuilder(new URI("https://api.openai.com/v1/chat/completions")) .header("Content-Type", "application/json") .header("Authorization", "Bearer " + token) .POST(HttpRequest.BodyPublishers.ofByteArray(SlimeUtils.toJsonBytes(slime))) @@ -68,21 +127,27 @@ public class OpenAiClient implements LanguageModel { } private List<Completion> toCompletions(Inspector response) { + return toCompletions(response, "message"); + } + + private List<Completion> toCompletions(Inspector response, String field) { List<Completion> completions = new ArrayList<>(); response.field("choices") - .traverse((ArrayTraverser) (__, choice) -> completions.add(toCompletion(choice))); + .traverse((ArrayTraverser) (__, choice) -> completions.add(toCompletion(choice, field))); return completions; } - private Completion toCompletion(Inspector choice) { - return new Completion(choice.field("text").asString(), - toFinishReason(choice.field("finish_reason").asString())); + private Completion toCompletion(Inspector choice, String field) { + var content = choice.field(field).field("content").asString(); + var finishReason = toFinishReason(choice.field("finish_reason").asString()); + return new Completion(content, finishReason); } private Completion.FinishReason toFinishReason(String finishReasonString) { return switch(finishReasonString) { case "length" -> Completion.FinishReason.length; case "stop" -> Completion.FinishReason.stop; + case "", "null" -> Completion.FinishReason.none; default -> throw new IllegalStateException("Unknown OpenAi completion finish reason '" + finishReasonString + "'"); }; } @@ -90,8 +155,9 @@ public class OpenAiClient implements LanguageModel { public static class Builder { private final String token; - private String model = "text-davinci-003"; - private double temperature = 0; + private String model = "gpt-3.5-turbo"; + private double temperature = 0.0; + private long maxTokens = 0; public Builder(String token) { this.token = token; @@ -109,6 +175,12 @@ public class OpenAiClient implements LanguageModel { return this; } + /** Maximum number of tokens to generate */ + public Builder maxTokens(long maxTokens) { + this.maxTokens = maxTokens; + return this; + } + public OpenAiClient build() { return new OpenAiClient(this); } diff --git a/vespajlib/src/main/java/ai/vespa/llm/completion/Completion.java b/vespajlib/src/main/java/ai/vespa/llm/completion/Completion.java index f5731852d93..ea784013812 100644 --- a/vespajlib/src/main/java/ai/vespa/llm/completion/Completion.java +++ b/vespajlib/src/main/java/ai/vespa/llm/completion/Completion.java @@ -19,8 +19,10 @@ public record Completion(String text, FinishReason finishReason) { length, /** The completion is the predicted ending of the prompt. */ - stop + stop, + /** The completion is not finished yet, more tokens are incoming. */ + none } public Completion(String text, FinishReason finishReason) { diff --git a/vespajlib/src/main/java/ai/vespa/llm/test/MockLanguageModel.java b/vespajlib/src/main/java/ai/vespa/llm/test/MockLanguageModel.java index 16e9c4e1848..db1b42fbbac 100644 --- a/vespajlib/src/main/java/ai/vespa/llm/test/MockLanguageModel.java +++ b/vespajlib/src/main/java/ai/vespa/llm/test/MockLanguageModel.java @@ -7,6 +7,8 @@ import ai.vespa.llm.completion.Prompt; import com.yahoo.api.annotations.Beta; import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.function.Consumer; import java.util.function.Function; /** @@ -26,6 +28,11 @@ public class MockLanguageModel implements LanguageModel { return completer.apply(prompt); } + @Override + public CompletableFuture<Completion.FinishReason> completeAsync(Prompt prompt, Consumer<Completion> action) { + throw new RuntimeException("Not implemented"); + } + public static class Builder { private Function<Prompt, List<Completion>> completer = prompt -> List.of(Completion.from("")); diff --git a/vespajlib/src/main/java/com/yahoo/collections/Optionals.java b/vespajlib/src/main/java/com/yahoo/collections/Optionals.java new file mode 100644 index 00000000000..27a1e837900 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/collections/Optionals.java @@ -0,0 +1,54 @@ +package com.yahoo.collections; + +import com.google.common.collect.Comparators; + +import java.util.Optional; +import java.util.function.BinaryOperator; + +/** + * @author jonmv + */ +public class Optionals { + + private Optionals() { } + + /** Returns the first non-empty optional, or empty if all are empty. */ + @SafeVarargs + public static <T> Optional<T> firstNonEmpty(Optional<T>... optionals) { + for (Optional<T> optional : optionals) + if (optional.isPresent()) + return optional; + return Optional.empty(); + } + + /** Returns the non-empty optional with the lowest value, or empty if all are empty. */ + @SafeVarargs + public static <T extends Comparable<T>> Optional<T> min(Optional<T>... optionals) { + Optional<T> best = Optional.empty(); + for (Optional<T> optional : optionals) + if (best.isEmpty() || optional.isPresent() && optional.get().compareTo(best.get()) < 0) + best = optional; + return best; + } + + /** Returns the non-empty optional with the highest value, or empty if all are empty. */ + @SafeVarargs + public static <T extends Comparable<T>> Optional<T> max(Optional<T>... optionals) { + Optional<T> best = Optional.empty(); + for (Optional<T> optional : optionals) + if (best.isEmpty() || optional.isPresent() && optional.get().compareTo(best.get()) > 0) + best = optional; + return best; + } + + /** Returns whether either optional is empty, or both are present and equal. */ + public static <T> boolean equalIfBothPresent(Optional<T> first, Optional<T> second) { + return first.isEmpty() || second.isEmpty() || first.equals(second); + } + + /** Returns whether the optional is empty, or present and equal to the given value. */ + public static <T> boolean emptyOrEqual(Optional<T> optional, T value) { + return optional.isEmpty() || optional.equals(Optional.of(value)); + } + +} diff --git a/vespajlib/src/main/java/com/yahoo/text/Text.java b/vespajlib/src/main/java/com/yahoo/text/Text.java index 7c835965a1a..fe931ef34a3 100644 --- a/vespajlib/src/main/java/com/yahoo/text/Text.java +++ b/vespajlib/src/main/java/com/yahoo/text/Text.java @@ -170,15 +170,15 @@ public final class Text { } /** - * Returns a string which is never larger than the given number of characters. + * Returns a string which is never larger than the given number of code points. * If the string is longer than the given length it will be truncated. * If length is 4 or less the string will be truncated to length. * If length is longer than 4, it will be truncated at length-4 with " ..." added at the end. */ public static String truncate(String s, int length) { - if (s.length() <= length) return s; - if (length <= 4) return s.substring(0, length); - return s.substring(0, length - 4) + " ..."; + if (s.codePointCount(0, s.length()) <= length) return s; + if (length <= 4) return substringByCodepoints(s, 0, length); + return substringByCodepoints(s, 0, length - 4) + " ..."; } public static String substringByCodepoints(String s, int fromCP, int toCP) { @@ -208,4 +208,5 @@ public final class Text { public static String format(String format, Object... args) { return String.format(Locale.US, format, args); } + } diff --git a/vespajlib/src/test/java/ai/vespa/llm/client/openai/OpenAiClientCompletionTest.java b/vespajlib/src/test/java/ai/vespa/llm/client/openai/OpenAiClientCompletionTest.java index 444f082b1c0..45ef7e270aa 100644 --- a/vespajlib/src/test/java/ai/vespa/llm/client/openai/OpenAiClientCompletionTest.java +++ b/vespajlib/src/test/java/ai/vespa/llm/client/openai/OpenAiClientCompletionTest.java @@ -11,12 +11,14 @@ import org.junit.jupiter.api.Test; */ public class OpenAiClientCompletionTest { + private static final String apiKey = "your-api-key-here"; + @Test @Disabled public void testClient() { - var client = new OpenAiClient.Builder("your token here").build(); + var client = new OpenAiClient.Builder(apiKey).maxTokens(10).build(); String input = "You are an unhelpful assistant who never answers questions straightforwardly. " + - "Be as long-winded as possible. Are humans smarter than cats?"; + "Be as long-winded as possible. Are humans smarter than cats?\n\n"; StringPrompt prompt = StringPrompt.from(input); System.out.print(prompt); for (int i = 0; i < 10; i++) { @@ -27,4 +29,19 @@ public class OpenAiClientCompletionTest { } } + @Test + @Disabled + public void testAsyncClient() { + var client = new OpenAiClient.Builder(apiKey).build(); + String input = "You are an unhelpful assistant who never answers questions straightforwardly. " + + "Be as long-winded as possible. Are humans smarter than cats?\n\n"; + StringPrompt prompt = StringPrompt.from(input); + System.out.print(prompt); + var future = client.completeAsync(prompt, completion -> { + System.out.print(completion.text()); + }); + System.out.println("Waiting for completion..."); + System.out.println("\nFinished streaming because of " + future.join()); + } + } diff --git a/vespajlib/src/test/java/com/yahoo/text/TextTestCase.java b/vespajlib/src/test/java/com/yahoo/text/TextTestCase.java index f192f678c13..9bb4668b7cb 100644 --- a/vespajlib/src/test/java/com/yahoo/text/TextTestCase.java +++ b/vespajlib/src/test/java/com/yahoo/text/TextTestCase.java @@ -104,6 +104,9 @@ public class TextTestCase { assertEquals("", Text.truncate("ab", 0)); assertEquals("ab c", Text.truncate("ab cde", 4)); assertEquals("a ...", Text.truncate("ab cde", 5)); + assertEquals("abc ...", Text.truncate("abc\uD83D\uDE48\uD83D\uDE49\uD83D\uDE4Adef", 7)); + assertEquals("abc\uD83D\uDE48 ...", Text.truncate("abc\uD83D\uDE48\uD83D\uDE49\uD83D\uDE4Adef", 8)); + assertEquals("abc\uD83D\uDE48\uD83D\uDE49\uD83D\uDE4Adef", Text.truncate("abc\uD83D\uDE48\uD83D\uDE49\uD83D\uDE4Adef", 9)); } @Test @@ -152,6 +155,6 @@ public class TextTestCase { sum = benchmarkIsValid(strings, 100000000); diff = System.nanoTime() - start; System.out.println("Validation num isValid = " + sum + ". Took " + diff + "ns"); - } + } diff --git a/vespalib/src/vespa/fastos/linux_file.cpp b/vespalib/src/vespa/fastos/linux_file.cpp index b6094a050d9..0f32aa953a8 100644 --- a/vespalib/src/vespa/fastos/linux_file.cpp +++ b/vespalib/src/vespa/fastos/linux_file.cpp @@ -202,7 +202,7 @@ FastOS_Linux_File::Write2(const void *buffer, size_t length) if (writtenNow > 0) { written += writtenNow; } else { - return (written > 0) ? written : writtenNow;; + return (written > 0) ? written : writtenNow; } } return written; @@ -239,8 +239,8 @@ FastOS_Linux_File::internalWrite2(const void *buffer, size_t length) } if (writeRes > 0) { _filePointer += writeRes; - if (_filePointer > _cachedSize) { - _cachedSize = _filePointer; + if (_filePointer > _cachedSize.load(std::memory_order_relaxed)) { + _cachedSize.store(_filePointer, std::memory_order_relaxed); } } } else { @@ -277,7 +277,7 @@ FastOS_Linux_File::SetSize(int64_t newSize) bool rc = FastOS_UNIX_File::SetSize(newSize); if (rc) { - _cachedSize = newSize; + _cachedSize.store(newSize, std::memory_order_relaxed); } return rc; } @@ -334,19 +334,21 @@ FastOS_Linux_File::DirectIOPadding (int64_t offset, size_t length, size_t &padBe if (padAfter == _directIOFileAlign) { padAfter = 0; } - if (int64_t(offset+length+padAfter) > _cachedSize) { + int64_t fileSize = _cachedSize.load(std::memory_order_relaxed); + if (int64_t(offset+length+padAfter) > fileSize) { // _cachedSize is not really trustworthy, so if we suspect it is not correct, we correct it. // The main reason is that it will not reflect the file being extended by another filedescriptor. - _cachedSize = getSize(); + fileSize = getSize(); + _cachedSize.store(fileSize, std::memory_order_relaxed); } if ((padAfter != 0) && - (static_cast<int64_t>(offset + length + padAfter) > _cachedSize) && - (static_cast<int64_t>(offset + length) <= _cachedSize)) + (static_cast<int64_t>(offset + length + padAfter) > fileSize) && + (static_cast<int64_t>(offset + length) <= fileSize)) { - padAfter = _cachedSize - (offset + length); + padAfter = fileSize - (offset + length); } - if (static_cast<uint64_t>(offset + length + padAfter) <= static_cast<uint64_t>(_cachedSize)) { + if (static_cast<uint64_t>(offset + length + padAfter) <= static_cast<uint64_t>(fileSize)) { return true; } } diff --git a/vespalib/src/vespa/fastos/linux_file.h b/vespalib/src/vespa/fastos/linux_file.h index 1295ce38316..af6e6af51af 100644 --- a/vespalib/src/vespa/fastos/linux_file.h +++ b/vespalib/src/vespa/fastos/linux_file.h @@ -10,21 +10,23 @@ #pragma once #include "unix_file.h" +#include <atomic> /** * This is the Linux implementation of @ref FastOS_File. Most * methods are inherited from @ref FastOS_UNIX_File. */ -class FastOS_Linux_File : public FastOS_UNIX_File +class FastOS_Linux_File final : public FastOS_UNIX_File { public: using FastOS_UNIX_File::ReadBuf; protected: - int64_t _cachedSize; + std::atomic<int64_t> _cachedSize; int64_t _filePointer; // Only maintained/used in directio mode public: - FastOS_Linux_File (const char *filename = nullptr); + FastOS_Linux_File() : FastOS_Linux_File(nullptr) {} + explicit FastOS_Linux_File(const char *filename); ~FastOS_Linux_File () override; bool GetDirectIORestrictions(size_t &memoryAlignment, size_t &transferGranularity, size_t &transferMaximum) override; bool DirectIOPadding(int64_t offset, size_t length, size_t &padBefore, size_t &padAfter) override; diff --git a/vespalib/src/vespa/vespalib/fuzzy/sparse_state.h b/vespalib/src/vespa/vespalib/fuzzy/sparse_state.h index 0f58853170e..e023f4d3de2 100644 --- a/vespalib/src/vespa/vespalib/fuzzy/sparse_state.h +++ b/vespalib/src/vespa/vespalib/fuzzy/sparse_state.h @@ -2,13 +2,12 @@ #pragma once #include <vespa/config.h> +#include <vespa/vespalib/stllike/hash_fun.h> #include <algorithm> #include <array> #include <cassert> -#include <cstdint> #include <ostream> #include <span> -#include <xxh3.h> // TODO factor out? namespace vespalib::fuzzy { @@ -80,15 +79,8 @@ public: size_t operator()(const FixedSparseState& s) const noexcept { static_assert(std::is_same_v<uint32_t, std::decay_t<decltype(s.indices[0])>>); static_assert(std::is_same_v<uint8_t, std::decay_t<decltype(s.costs[0])>>); - // FIXME GCC 12.2 worse-than-useless(tm) warning false positives :I -#pragma GCC diagnostic push -#ifdef VESPA_USE_SANITIZER -# pragma GCC diagnostic ignored "-Wstringop-overread" // https://gcc.gnu.org/bugzilla/show_bug.cgi?id=98465 etc. -#endif -#pragma GCC diagnostic ignored "-Warray-bounds" - return (XXH3_64bits(s.indices.data(), s.sz * sizeof(uint32_t)) ^ - XXH3_64bits(s.costs.data(), s.sz)); -#pragma GCC diagnostic pop + return (xxhash::xxh3_64(s.indices.data(), s.sz * sizeof(uint32_t)) ^ + xxhash::xxh3_64(s.costs.data(), s.sz)); } }; }; diff --git a/vespalib/src/vespa/vespalib/stllike/hash_fun.cpp b/vespalib/src/vespa/vespalib/stllike/hash_fun.cpp index e4911d2e0c6..6802e5f91a6 100644 --- a/vespalib/src/vespa/vespalib/stllike/hash_fun.cpp +++ b/vespalib/src/vespa/vespalib/stllike/hash_fun.cpp @@ -6,23 +6,21 @@ namespace vespalib { size_t -hashValue(const char *str) noexcept -{ - return hashValue(str, strlen(str)); +hashValue(const char *str) noexcept { + return xxhash::xxh3_64(str, strlen(str)); } -/** - * @brief Calculate hash value. - * - * The hash function XXH3_64bits from xxhash library. - * @param buf input buffer - * @param sz input buffer size - * @return hash value of input - **/ -size_t -hashValue(const void * buf, size_t sz) noexcept -{ +namespace xxhash { + +uint64_t +xxh3_64(uint64_t value) noexcept { + return XXH3_64bits(&value, sizeof(value)); +} + +uint64_t +xxh3_64(const void * buf, size_t sz) noexcept { return XXH3_64bits(buf, sz); } } +} diff --git a/vespalib/src/vespa/vespalib/stllike/hash_fun.h b/vespalib/src/vespa/vespalib/stllike/hash_fun.h index f8a8a06b921..8fecc41b4c1 100644 --- a/vespalib/src/vespa/vespalib/stllike/hash_fun.h +++ b/vespalib/src/vespa/vespalib/stllike/hash_fun.h @@ -2,7 +2,6 @@ #pragma once #include <vespa/vespalib/stllike/string.h> -#include <cstdint> namespace vespalib { @@ -64,9 +63,18 @@ template<typename T> struct hash<const T *> { size_t operator() (const T * arg) const noexcept { return size_t(arg); } }; +namespace xxhash { + +uint64_t xxh3_64(uint64_t value) noexcept; +uint64_t xxh3_64(const void *str, size_t sz) noexcept; + +} + // reuse old string hash function size_t hashValue(const char *str) noexcept; -size_t hashValue(const void *str, size_t sz) noexcept; +inline size_t hashValue(const void *buf, size_t sz) noexcept { + return xxhash::xxh3_64(buf, sz); +} struct hash_strings { size_t operator() (const vespalib::string & arg) const noexcept { return hashValue(arg.data(), arg.size()); } diff --git a/vespalib/src/vespa/vespalib/util/shared_string_repo.cpp b/vespalib/src/vespa/vespalib/util/shared_string_repo.cpp index a5942e1af9b..09f2bbd828d 100644 --- a/vespalib/src/vespa/vespalib/util/shared_string_repo.cpp +++ b/vespalib/src/vespa/vespalib/util/shared_string_repo.cpp @@ -1,7 +1,7 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "shared_string_repo.h" -#include <xxhash.h> +#include <vespa/vespalib/stllike/hash_fun.h> #include <charconv> #include <cassert> @@ -232,7 +232,7 @@ string_id SharedStringRepo::resolve(vespalib::stringref str) { uint32_t direct_id = try_make_direct_id(str); if (direct_id >= ID_BIAS) { - uint64_t full_hash = XXH3_64bits(str.data(), str.size()); + uint64_t full_hash = xxhash::xxh3_64(str.data(), str.size()); uint32_t part = full_hash & PART_MASK; uint32_t local_hash = full_hash >> PART_BITS; uint32_t local_idx = _partitions[part].resolve(AltKey{str, local_hash}); diff --git a/vespamalloc/src/tests/stacktrace/stacktrace.cpp b/vespamalloc/src/tests/stacktrace/stacktrace.cpp index d7f7d45656d..2592222fce3 100644 --- a/vespamalloc/src/tests/stacktrace/stacktrace.cpp +++ b/vespamalloc/src/tests/stacktrace/stacktrace.cpp @@ -29,7 +29,7 @@ void verify_that_vespamalloc_datasegment_size_exists() { assert(info.ordblks == 0); assert(info.smblks == 0); assert(info.uordblks > 0); - assert(info.usmblks == 0); + assert(info.usmblks > 0); #else struct mallinfo info = mallinfo(); printf("Malloc used %dm of memory\n",info.arena); @@ -43,7 +43,7 @@ void verify_that_vespamalloc_datasegment_size_exists() { assert(info.ordblks == 0); assert(info.smblks == 0); assert(info.uordblks > 0); - assert(info.usmblks == 0); + assert(info.usmblks > 0); #endif } diff --git a/vespamalloc/src/vespamalloc/malloc/mmappool.cpp b/vespamalloc/src/vespamalloc/malloc/mmappool.cpp index e43ea2e8342..cee709ed0ed 100644 --- a/vespamalloc/src/vespamalloc/malloc/mmappool.cpp +++ b/vespamalloc/src/vespamalloc/malloc/mmappool.cpp @@ -10,6 +10,8 @@ namespace vespamalloc { MMapPool::MMapPool() : _page_size(getpagesize()), _huge_flags((getenv("VESPA_USE_HUGEPAGES") != nullptr) ? MAP_HUGETLB : 0), + _peakBytes(0ul), + _currentBytes(0ul), _count(0), _mutex(), _mappings() @@ -28,9 +30,13 @@ MMapPool::getNumMappings() const { size_t MMapPool::getMmappedBytes() const { std::lock_guard guard(_mutex); - size_t sum(0); - std::for_each(_mappings.begin(), _mappings.end(), [&sum](const auto & e){ sum += e.second._sz; }); - return sum; + return _currentBytes; +} + +size_t +MMapPool::getMmappedBytesPeak() const { + std::lock_guard guard(_mutex); + return _peakBytes; } void * @@ -76,10 +82,10 @@ MMapPool::mmap(size_t sz) { std::lock_guard guard(_mutex); auto [it, inserted] = _mappings.insert(std::make_pair(buf, MMapInfo(mmapId, sz))); ASSERT_STACKTRACE(inserted); + _currentBytes += sz; + _peakBytes = std::max(_peakBytes, _currentBytes); if (sz >= _G_bigBlockLimit) { - size_t sum(0); - std::for_each(_mappings.begin(), _mappings.end(), [&sum](const auto & e){ sum += e.second._sz; }); - fprintf(_G_logFile, "%ld mappings of accumulated size %ld\n", _mappings.size(), sum); + fprintf(_G_logFile, "%ld mappings of accumulated size %ld\n", _mappings.size(), _currentBytes); } } return buf; @@ -98,6 +104,7 @@ MMapPool::unmap(void * ptr) { } sz = found->second._sz; _mappings.erase(found); + _currentBytes -= sz; } int munmap_ok = ::munmap(ptr, sz); ASSERT_STACKTRACE(munmap_ok == 0); diff --git a/vespamalloc/src/vespamalloc/malloc/mmappool.h b/vespamalloc/src/vespamalloc/malloc/mmappool.h index c0b73c56e4e..0a3dd5d0a0a 100644 --- a/vespamalloc/src/vespamalloc/malloc/mmappool.h +++ b/vespamalloc/src/vespamalloc/malloc/mmappool.h @@ -19,6 +19,7 @@ public: size_t get_size(void *) const; size_t getNumMappings() const; size_t getMmappedBytes() const; + size_t getMmappedBytesPeak() const; void info(FILE * os, size_t level) const; private: struct MMapInfo { @@ -28,6 +29,8 @@ private: }; const size_t _page_size; const int _huge_flags; + size_t _peakBytes; + size_t _currentBytes; std::atomic<size_t> _count; std::atomic<bool> _has_hugepage_failure_just_happened; mutable std::mutex _mutex; diff --git a/vespamalloc/src/vespamalloc/malloc/overload.h b/vespamalloc/src/vespamalloc/malloc/overload.h index cb57e6b2fa8..bf9710b9b32 100644 --- a/vespamalloc/src/vespamalloc/malloc/overload.h +++ b/vespamalloc/src/vespamalloc/malloc/overload.h @@ -119,7 +119,9 @@ struct mallinfo2 mallinfo2() __THROW { info.smblks = 0; info.hblkhd = vespamalloc::_GmemP->mmapPool().getNumMappings(); info.hblks = vespamalloc::_GmemP->mmapPool().getMmappedBytes(); - info.usmblks = 0; + size_t highwaterMark = vespamalloc::_GmemP->dataSegment().dataSize() + + vespamalloc::_GmemP->mmapPool().getMmappedBytesPeak(); + info.usmblks = highwaterMark; info.fsmblks = 0; info.fordblks = vespamalloc::_GmemP->dataSegment().freeSize(); info.uordblks = info.arena + info.hblks - info.fordblks; @@ -135,7 +137,9 @@ struct mallinfo mallinfo() __THROW { info.smblks = 0; info.hblkhd = vespamalloc::_GmemP->mmapPool().getNumMappings(); info.hblks = (vespamalloc::_GmemP->mmapPool().getMmappedBytes() >> 20); - info.usmblks = 0; + size_t highwaterMark = vespamalloc::_GmemP->dataSegment().dataSize() + + vespamalloc::_GmemP->mmapPool().getMmappedBytesPeak(); + info.usmblks = (highwaterMark >> 20); info.fsmblks = 0; info.fordblks = (vespamalloc::_GmemP->dataSegment().freeSize() >> 20); info.uordblks = info.arena + info.hblks - info.fordblks; |