summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorJon Bratseth <bratseth@verizonmedia.com>2020-02-25 15:07:53 +0100
committerJon Bratseth <bratseth@verizonmedia.com>2020-02-25 15:07:53 +0100
commit035651fade6aba9bdcfdcd43d52e01f9c42382a9 (patch)
tree3ad1b48967797c372342874e860904535905060d
parent58ccc5ac198042ac8b64647b3d2d9121d571860d (diff)
parent1e8356694522c1eabbdad2e27ac1b6b77828f7ae (diff)
Merge branch 'master' into bratseth/node-metrics
-rw-r--r--athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/instanceconfirmation/InstanceValidatorTest.java3
-rw-r--r--client/src/main/java/ai/vespa/client/dsl/EndQuery.java2
-rw-r--r--client/src/test/groovy/ai/vespa/client/dsl/QTest.groovy2
-rw-r--r--config-model-api/abi-spec.json12
-rw-r--r--config-model-api/src/main/java/com/yahoo/config/model/api/SuperModel.java20
-rw-r--r--config-model-api/src/main/java/com/yahoo/config/model/api/SuperModelListener.java8
-rw-r--r--config-model/pom.xml4
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/processing/TensorFieldProcessor.java18
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/admin/LogserverContainerCluster.java7
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java21
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/CloudWatch.java49
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/MetricsConsumer.java14
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/builder/xml/CloudWatchBuilder.java36
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/builder/xml/MetricsBuilder.java8
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/application/validation/EndpointCertificateSecretsValidator.java2
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java1
-rwxr-xr-xconfig-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java1
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/container/http/AccessControl.java3
-rwxr-xr-xconfig-model/src/main/perl/vespa-deploy2
-rw-r--r--config-model/src/main/resources/schema/admin.rnc17
-rw-r--r--config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsConsumersTest.java244
-rw-r--r--config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerClusterTest.java242
-rw-r--r--config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyModelTester.java24
-rw-r--r--config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/TelegrafTest.java87
-rw-r--r--config-model/src/test/java/com/yahoo/vespa/model/application/validation/EndpointCertificateSecretsValidatorTest.java2
-rwxr-xr-xconfig-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java2
-rw-r--r--config-model/src/test/java/com/yahoo/vespa/model/container/xml/DocprocBuilderTest.java25
-rw-r--r--config-model/src/test/schema-test-files/services.xml17
-rw-r--r--config-provisioning/abi-spec.json2
-rw-r--r--config-provisioning/src/main/java/com/yahoo/config/provision/InfraDeployer.java5
-rw-r--r--config-provisioning/src/main/java/com/yahoo/config/provision/zone/RoutingMethod.java8
-rw-r--r--config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ConfigProxyRpcServer.java6
-rw-r--r--config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ConfigSourceClient.java2
-rw-r--r--config-proxy/src/main/java/com/yahoo/vespa/config/proxy/MemoryCacheConfigClient.java6
-rw-r--r--config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ProxyServer.java70
-rw-r--r--config-proxy/src/main/java/com/yahoo/vespa/config/proxy/RpcConfigSourceClient.java84
-rw-r--r--config-proxy/src/main/java/com/yahoo/vespa/config/proxy/UpstreamConfigSubscriber.java15
-rw-r--r--config-proxy/src/test/java/com/yahoo/vespa/config/proxy/ConfigProxyRpcServerTest.java2
-rw-r--r--config-proxy/src/test/java/com/yahoo/vespa/config/proxy/MockConfigSourceClient.java7
-rw-r--r--config-proxy/src/test/java/com/yahoo/vespa/config/proxy/ProxyServerTest.java11
-rw-r--r--config-proxy/src/test/java/com/yahoo/vespa/config/proxy/RpcConfigSourceClientTest.java8
-rwxr-xr-xconfig/src/main/java/com/yahoo/config/subscription/ConfigSourceSet.java6
-rw-r--r--config/src/main/java/com/yahoo/config/subscription/ConfigSubscriber.java1
-rw-r--r--config/src/main/java/com/yahoo/config/subscription/impl/JRTConfigRequester.java33
-rw-r--r--config/src/main/java/com/yahoo/config/subscription/impl/JRTConfigSubscription.java4
-rw-r--r--config/src/main/java/com/yahoo/config/subscription/impl/JRTManagedConnectionPools.java66
-rw-r--r--config/src/main/java/com/yahoo/vespa/config/JRTConnectionPool.java13
-rw-r--r--config/src/main/java/com/yahoo/vespa/config/TimingValues.java60
-rw-r--r--config/src/test/java/com/yahoo/config/subscription/BasicTest.java10
-rw-r--r--config/src/test/java/com/yahoo/config/subscription/ConfigSetSubscriptionTest.java1
-rw-r--r--config/src/test/java/com/yahoo/config/subscription/ConfigSubscriptionTest.java2
-rw-r--r--config/src/test/java/com/yahoo/config/subscription/GenericConfigSubscriberTest.java7
-rw-r--r--config/src/test/java/com/yahoo/config/subscription/impl/JRTConfigRequesterTest.java23
-rw-r--r--config/src/tests/trace/trace.cpp1
-rw-r--r--config/src/vespa/config/common/configdefinition.cpp1
-rw-r--r--config/src/vespa/config/common/configdefinition.h5
-rw-r--r--config/src/vespa/config/common/trace.cpp1
-rw-r--r--config/src/vespa/config/common/trace.h7
-rw-r--r--configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerBootstrap.java15
-rw-r--r--configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelManager.java53
-rw-r--r--configserver/src/test/java/com/yahoo/vespa/config/server/SuperModelControllerTest.java6
-rw-r--r--container-accesslogging/pom.xml9
-rw-r--r--container-accesslogging/src/main/java/com/yahoo/container/logging/LogFileHandler.java3
-rw-r--r--container-accesslogging/src/test/java/com/yahoo/container/logging/LogFileHandlerTestCase.java163
-rw-r--r--container-core/src/main/java/com/yahoo/container/jdisc/ThreadedRequestHandler.java6
-rwxr-xr-xcontainer-disc/src/main/sh/vespa-start-container-daemon.sh11
-rw-r--r--container-search/src/main/resources/configdefinitions/qr-start.def3
-rw-r--r--controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/EndpointCertificateMetadata.java28
-rw-r--r--controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/EndpointCertificateMock.java3
-rw-r--r--controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/EndpointCertificateProvider.java3
-rw-r--r--controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/PathGroup.java3
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java2
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java3
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java70
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificateManager.java (renamed from controller-server/src/main/java/com/yahoo/vespa/hosted/controller/endpointcertificates/EndpointCertificateManager.java)25
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java37
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CostReportMaintainer.java6
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/metric/CostCalculator.java (renamed from controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/cost/CostCalculator.java)20
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java4
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/EndpointCertificateMetadataSerializer.java59
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java4
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java19
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/cost/CostApiHandler.java52
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/cost/package-info.java5
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicy.java11
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/EndpointTest.java41
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificateManagerTest.java (renamed from controller-server/src/test/java/com/yahoo/vespa/hosted/controller/endpointcertificates/EndpointCertificateManagerTest.java)2
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/EndpointCertificateMetadataSerializerTest.java11
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java3
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java12
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/cost/CostApiTest.java49
-rw-r--r--default_build_settings.cmake22
-rw-r--r--dist/vespa.spec22
-rw-r--r--docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/CreateContainerCommandImpl.java9
-rw-r--r--docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/Docker.java1
-rw-r--r--eval/src/vespa/eval/eval/interpreted_function.cpp21
-rw-r--r--eval/src/vespa/eval/eval/interpreted_function.h6
-rw-r--r--eval/src/vespa/eval/eval/lazy_params.cpp15
-rw-r--r--eval/src/vespa/eval/eval/tensor_function.h13
-rw-r--r--eval/src/vespa/eval/eval/value.h11
-rw-r--r--fbench/CMakeLists.txt1
-rw-r--r--fbench/src/fbench/fbench.cpp11
-rw-r--r--fbench/src/httpclient/CMakeLists.txt1
-rw-r--r--fbench/src/httpclient/httpclient.cpp16
-rw-r--r--fbench/src/httpclient/httpclient.h4
-rw-r--r--fbench/src/test/authority/CMakeLists.txt10
-rw-r--r--fbench/src/test/authority/authority_test.cpp87
-rw-r--r--fbench/src/util/CMakeLists.txt3
-rw-r--r--fbench/src/util/authority.cpp42
-rw-r--r--fbench/src/util/authority.h30
-rw-r--r--flags/src/main/java/com/yahoo/vespa/flags/Flags.java10
-rw-r--r--fnet/src/tests/connect/connect_test.cpp2
-rw-r--r--fnet/src/tests/frt/values/values_test.cpp1
-rw-r--r--fnet/src/vespa/fnet/frt/rpcrequest.h1
-rw-r--r--fnet/src/vespa/fnet/frt/values.cpp5
-rw-r--r--fnet/src/vespa/fnet/frt/values.h5
-rw-r--r--jdisc_http_service/src/test/java/com/yahoo/jdisc/http/ssl/impl/TlsContextBasedProviderTest.java3
-rw-r--r--jrt/src/com/yahoo/jrt/TlsCryptoEngine.java3
-rw-r--r--jrt/tests/com/yahoo/jrt/CryptoUtils.java13
-rw-r--r--metrics-proxy/pom.xml4
-rw-r--r--metrics-proxy/src/main/java/ai/vespa/metricsproxy/telegraf/Telegraf.java95
-rw-r--r--metrics-proxy/src/main/java/ai/vespa/metricsproxy/telegraf/TelegrafRegistry.java33
-rw-r--r--metrics-proxy/src/main/resources/configdefinitions/telegraf.def5
-rw-r--r--metrics-proxy/src/main/resources/templates/telegraf.conf.vm44
-rw-r--r--metrics-proxy/src/test/java/ai/vespa/metricsproxy/telegraf/TelegrafTest.java42
-rw-r--r--metrics-proxy/src/test/resources/telegraf-config-with-two-cloudwatch-plugins.txt46
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImpl.java3
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java12
-rw-r--r--node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerMock.java5
-rw-r--r--node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java26
-rw-r--r--node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/InfrastructureProvisioner.java21
-rw-r--r--node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java2
-rw-r--r--node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/InfraDeployerImpl.java20
-rw-r--r--node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockDuperModel.java4
-rw-r--r--node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockInfraDeployer.java5
-rw-r--r--orchestrator/pom.xml6
-rw-r--r--orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorContext.java6
-rw-r--r--orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorImpl.java9
-rw-r--r--orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ClusterApi.java3
-rw-r--r--orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ClusterApiImpl.java61
-rw-r--r--orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/HostStateChangeDeniedException.java18
-rw-r--r--orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/HostedVespaClusterPolicy.java10
-rw-r--r--orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/HostResource.java5
-rw-r--r--orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/HostSuspensionResource.java3
-rw-r--r--orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/MutableStatusRegistry.java3
-rw-r--r--orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/ZookeeperStatusService.java5
-rw-r--r--orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorContextTest.java17
-rw-r--r--orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorImplTest.java3
-rw-r--r--orchestrator/src/test/java/com/yahoo/vespa/orchestrator/model/ClusterApiImplTest.java20
-rw-r--r--orchestrator/src/test/java/com/yahoo/vespa/orchestrator/policy/HostedVespaClusterPolicyTest.java11
-rw-r--r--orchestrator/src/test/java/com/yahoo/vespa/orchestrator/resources/HostResourceTest.java26
-rw-r--r--orchestrator/src/test/java/com/yahoo/vespa/orchestrator/status/ZookeeperStatusServiceTest.java4
-rw-r--r--parent/pom.xml4
-rw-r--r--searchcommon/src/vespa/searchcommon/attribute/config.cpp9
-rw-r--r--searchcommon/src/vespa/searchcommon/attribute/config.h18
-rw-r--r--searchcommon/src/vespa/searchcommon/attribute/hnsw_index_params.h32
-rw-r--r--searchcore/src/tests/proton/documentdb/configurer/configurer_test.cpp4
-rw-r--r--searchcore/src/tests/proton/documentdb/threading_service_config/threading_service_config_test.cpp6
-rw-r--r--searchcore/src/tests/proton/matchengine/matchengine.cpp54
-rw-r--r--searchcore/src/tests/proton/matching/matching_test.cpp2
-rw-r--r--searchcore/src/tests/proton/summaryengine/summaryengine.cpp56
-rw-r--r--searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.cpp8
-rw-r--r--searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.h2
-rw-r--r--searchcore/src/vespa/searchcore/proton/common/doctypename.cpp12
-rw-r--r--searchcore/src/vespa/searchcore/proton/common/handlermap.hpp1
-rw-r--r--searchcore/src/vespa/searchcore/proton/matchengine/imatchhandler.h46
-rw-r--r--searchcore/src/vespa/searchcore/proton/matchengine/matchengine.cpp7
-rw-r--r--searchcore/src/vespa/searchcore/proton/matchengine/matchengine.h8
-rw-r--r--searchcore/src/vespa/searchcore/proton/matching/match_thread.cpp2
-rw-r--r--searchcore/src/vespa/searchcore/proton/matching/search_session.h2
-rw-r--r--searchcore/src/vespa/searchcore/proton/server/documentdb.cpp4
-rw-r--r--searchcore/src/vespa/searchcore/proton/server/documentdb.h3
-rw-r--r--searchcore/src/vespa/searchcore/proton/server/emptysearchview.cpp19
-rw-r--r--searchcore/src/vespa/searchcore/proton/server/emptysearchview.h13
-rw-r--r--searchcore/src/vespa/searchcore/proton/server/matchview.cpp24
-rw-r--r--searchcore/src/vespa/searchcore/proton/server/matchview.h12
-rw-r--r--searchcore/src/vespa/searchcore/proton/server/proton.cpp3
-rw-r--r--searchcore/src/vespa/searchcore/proton/server/searchable_doc_subdb_configurer.cpp10
-rw-r--r--searchcore/src/vespa/searchcore/proton/server/searchable_doc_subdb_configurer.h5
-rw-r--r--searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.cpp8
-rw-r--r--searchcore/src/vespa/searchcore/proton/server/searchhandlerproxy.cpp10
-rw-r--r--searchcore/src/vespa/searchcore/proton/server/searchhandlerproxy.h6
-rw-r--r--searchcore/src/vespa/searchcore/proton/server/searchview.cpp15
-rw-r--r--searchcore/src/vespa/searchcore/proton/server/searchview.h9
-rw-r--r--searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.h1
-rw-r--r--searchcore/src/vespa/searchcore/proton/server/threading_service_config.cpp5
-rw-r--r--searchcore/src/vespa/searchcore/proton/summaryengine/isearchhandler.h5
-rw-r--r--searchcore/src/vespa/searchcore/proton/summaryengine/summaryengine.cpp4
-rw-r--r--searchlib/src/apps/tests/memoryindexstress_test.cpp2
-rw-r--r--searchlib/src/tests/attribute/attributemanager/attributemanager_test.cpp16
-rw-r--r--searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp332
-rw-r--r--searchlib/src/tests/common/sequencedtaskexecutor/.gitignore1
-rw-r--r--searchlib/src/tests/common/sequencedtaskexecutor/CMakeLists.txt7
-rw-r--r--searchlib/src/tests/common/sequencedtaskexecutor/sequencedtaskexecutor_benchmark.cpp24
-rw-r--r--searchlib/src/tests/features/prod_features.cpp170
-rw-r--r--searchlib/src/tests/features/prod_features.h19
-rw-r--r--searchlib/src/tests/fef/resolver/resolver_test.cpp21
-rw-r--r--searchlib/src/tests/queryeval/nearest_neighbor/nearest_neighbor_test.cpp67
-rw-r--r--searchlib/src/tests/rankingexpression/intrinsic_blueprint_adapter/intrinsic_blueprint_adapter_test.cpp1
-rw-r--r--searchlib/src/tests/tensor/hnsw_index/hnsw_index_test.cpp49
-rw-r--r--searchlib/src/vespa/searchlib/attribute/configconverter.cpp4
-rw-r--r--searchlib/src/vespa/searchlib/common/sequencedtaskexecutor.cpp1
-rw-r--r--searchlib/src/vespa/searchlib/common/sequencedtaskexecutor.h3
-rw-r--r--searchlib/src/vespa/searchlib/features/agefeature.cpp11
-rw-r--r--searchlib/src/vespa/searchlib/features/attributematchfeature.cpp1
-rw-r--r--searchlib/src/vespa/searchlib/features/bm25_feature.cpp2
-rw-r--r--searchlib/src/vespa/searchlib/features/closenessfeature.cpp10
-rw-r--r--searchlib/src/vespa/searchlib/features/constant_feature.cpp17
-rw-r--r--searchlib/src/vespa/searchlib/features/constant_tensor_executor.h2
-rw-r--r--searchlib/src/vespa/searchlib/features/debug_attribute_wait.cpp2
-rw-r--r--searchlib/src/vespa/searchlib/features/debug_wait.cpp1
-rw-r--r--searchlib/src/vespa/searchlib/features/distancefeature.cpp29
-rw-r--r--searchlib/src/vespa/searchlib/features/distancetopathfeature.cpp16
-rw-r--r--searchlib/src/vespa/searchlib/features/dotproductfeature.cpp5
-rw-r--r--searchlib/src/vespa/searchlib/features/element_completeness_feature.cpp3
-rw-r--r--searchlib/src/vespa/searchlib/features/element_similarity_feature.cpp1
-rw-r--r--searchlib/src/vespa/searchlib/features/euclidean_distance_feature.cpp14
-rw-r--r--searchlib/src/vespa/searchlib/features/fieldinfofeature.cpp8
-rw-r--r--searchlib/src/vespa/searchlib/features/fieldlengthfeature.cpp13
-rw-r--r--searchlib/src/vespa/searchlib/features/fieldmatchfeature.cpp10
-rw-r--r--searchlib/src/vespa/searchlib/features/fieldtermmatchfeature.cpp9
-rw-r--r--searchlib/src/vespa/searchlib/features/firstphasefeature.cpp10
-rw-r--r--searchlib/src/vespa/searchlib/features/flow_completeness_feature.cpp6
-rw-r--r--searchlib/src/vespa/searchlib/features/foreachfeature.cpp9
-rw-r--r--searchlib/src/vespa/searchlib/features/freshnessfeature.cpp11
-rw-r--r--searchlib/src/vespa/searchlib/features/internal_max_reduce_prod_join_feature.cpp3
-rw-r--r--searchlib/src/vespa/searchlib/features/item_raw_score_feature.cpp8
-rw-r--r--searchlib/src/vespa/searchlib/features/jarowinklerdistancefeature.cpp4
-rw-r--r--searchlib/src/vespa/searchlib/features/matchcountfeature.cpp11
-rw-r--r--searchlib/src/vespa/searchlib/features/matchesfeature.cpp12
-rw-r--r--searchlib/src/vespa/searchlib/features/matchfeature.cpp17
-rw-r--r--searchlib/src/vespa/searchlib/features/native_dot_product_feature.cpp7
-rw-r--r--searchlib/src/vespa/searchlib/features/nativeattributematchfeature.cpp10
-rw-r--r--searchlib/src/vespa/searchlib/features/nativefieldmatchfeature.cpp13
-rw-r--r--searchlib/src/vespa/searchlib/features/nativeproximityfeature.cpp20
-rw-r--r--searchlib/src/vespa/searchlib/features/nativerankfeature.cpp10
-rw-r--r--searchlib/src/vespa/searchlib/features/nowfeature.cpp1
-rw-r--r--searchlib/src/vespa/searchlib/features/proximityfeature.cpp10
-rw-r--r--searchlib/src/vespa/searchlib/features/querycompletenessfeature.cpp9
-rw-r--r--searchlib/src/vespa/searchlib/features/querytermcountfeature.cpp13
-rw-r--r--searchlib/src/vespa/searchlib/features/random_normal_feature.cpp1
-rw-r--r--searchlib/src/vespa/searchlib/features/random_normal_stable_feature.cpp4
-rw-r--r--searchlib/src/vespa/searchlib/features/randomfeature.cpp3
-rw-r--r--searchlib/src/vespa/searchlib/features/raw_score_feature.cpp1
-rw-r--r--searchlib/src/vespa/searchlib/features/reverseproximityfeature.cpp11
-rw-r--r--searchlib/src/vespa/searchlib/features/subqueries_feature.cpp7
-rw-r--r--searchlib/src/vespa/searchlib/features/term_field_md_feature.cpp4
-rw-r--r--searchlib/src/vespa/searchlib/features/termdistancefeature.cpp10
-rw-r--r--searchlib/src/vespa/searchlib/features/termeditdistancefeature.cpp4
-rw-r--r--searchlib/src/vespa/searchlib/features/termfeature.cpp13
-rw-r--r--searchlib/src/vespa/searchlib/features/terminfofeature.cpp25
-rw-r--r--searchlib/src/vespa/searchlib/features/text_similarity_feature.cpp5
-rw-r--r--searchlib/src/vespa/searchlib/features/valuefeature.cpp24
-rw-r--r--searchlib/src/vespa/searchlib/fef/blueprintresolver.cpp12
-rw-r--r--searchlib/src/vespa/searchlib/fef/rank_program.cpp8
-rw-r--r--searchlib/src/vespa/searchlib/fef/rank_program.h2
-rw-r--r--searchlib/src/vespa/searchlib/fef/ranksetup.cpp21
-rw-r--r--searchlib/src/vespa/searchlib/fef/ranksetup.h8
-rw-r--r--searchlib/src/vespa/searchlib/fef/test/plugin/cfgvalue.cpp13
-rw-r--r--searchlib/src/vespa/searchlib/fef/test/plugin/chain.cpp9
-rw-r--r--searchlib/src/vespa/searchlib/fef/test/plugin/double.cpp1
-rw-r--r--searchlib/src/vespa/searchlib/fef/test/plugin/query.cpp3
-rw-r--r--searchlib/src/vespa/searchlib/fef/test/plugin/staticrank.cpp15
-rw-r--r--searchlib/src/vespa/searchlib/fef/test/plugin/sum.cpp9
-rw-r--r--searchlib/src/vespa/searchlib/fef/test/plugin/unbox.cpp1
-rw-r--r--searchlib/src/vespa/searchlib/fef/test/test_features.cpp2
-rw-r--r--searchlib/src/vespa/searchlib/queryeval/CMakeLists.txt1
-rw-r--r--searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_blueprint.cpp38
-rw-r--r--searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_blueprint.h4
-rw-r--r--searchlib/src/vespa/searchlib/queryeval/nns_index_iterator.cpp68
-rw-r--r--searchlib/src/vespa/searchlib/queryeval/nns_index_iterator.h20
-rw-r--r--searchlib/src/vespa/searchlib/tensor/CMakeLists.txt4
-rw-r--r--searchlib/src/vespa/searchlib/tensor/default_nearest_neighbor_index_factory.cpp51
-rw-r--r--searchlib/src/vespa/searchlib/tensor/default_nearest_neighbor_index_factory.h19
-rw-r--r--searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute.cpp43
-rw-r--r--searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute.h48
-rw-r--r--searchlib/src/vespa/searchlib/tensor/distance_function.h3
-rw-r--r--searchlib/src/vespa/searchlib/tensor/distance_functions.h1
-rw-r--r--searchlib/src/vespa/searchlib/tensor/hnsw_index.cpp61
-rw-r--r--searchlib/src/vespa/searchlib/tensor/hnsw_index.h28
-rw-r--r--searchlib/src/vespa/searchlib/tensor/nearest_neighbor_index.cpp3
-rw-r--r--searchlib/src/vespa/searchlib/tensor/nearest_neighbor_index.h12
-rw-r--r--searchlib/src/vespa/searchlib/tensor/nearest_neighbor_index_factory.h26
-rw-r--r--searchlib/src/vespa/searchlib/tensor/random_level_generator.h3
-rw-r--r--searchsummary/src/tests/docsummary/matched_elements_filter/matched_elements_filter_test.cpp2
-rw-r--r--searchsummary/src/vespa/searchsummary/docsummary/summaryfieldconverter.cpp14
-rw-r--r--security-tools/pom.xml1
-rw-r--r--security-tools/src/main/java/com/yahoo/vespa/security/tool/securityenv/Main.java3
-rw-r--r--security-tools/src/main/java/com/yahoo/vespa/security/tool/securityenv/OutputVariable.java3
-rwxr-xr-xsecurity-tools/src/main/sh/vespa-curl-wrapper5
-rw-r--r--security-tools/src/test/java/com/yahoo/vespa/security/tool/securityenv/MainTest.java1
-rw-r--r--security-tools/src/test/resources/bash-output.txt1
-rw-r--r--security-tools/src/test/resources/csh-output.txt1
-rw-r--r--security-tools/src/test/resources/expected-help-output.txt2
-rw-r--r--security-tools/src/test/resources/no-security-output.txt1
-rw-r--r--security-utils/src/main/java/com/yahoo/security/SslContextBuilder.java17
-rw-r--r--security-utils/src/main/java/com/yahoo/security/tls/ConfigFileBasedTlsContext.java12
-rw-r--r--security-utils/src/main/java/com/yahoo/security/tls/DefaultTlsContext.java14
-rw-r--r--security-utils/src/main/java/com/yahoo/security/tls/HostnameVerification.java7
-rw-r--r--security-utils/src/main/java/com/yahoo/security/tls/authz/PeerAuthorizerTrustManager.java57
-rw-r--r--security-utils/src/test/java/com/yahoo/security/tls/DefaultTlsContextTest.java4
-rw-r--r--service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModel.java8
-rw-r--r--service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModelManager.java35
-rw-r--r--service-monitor/src/main/java/com/yahoo/vespa/service/health/HealthMonitorManager.java4
-rw-r--r--service-monitor/src/main/java/com/yahoo/vespa/service/manager/MonitorManager.java2
-rw-r--r--service-monitor/src/main/java/com/yahoo/vespa/service/manager/UnionMonitorManager.java4
-rw-r--r--service-monitor/src/main/java/com/yahoo/vespa/service/monitor/DuperModelInfraApi.java3
-rw-r--r--service-monitor/src/main/java/com/yahoo/vespa/service/monitor/DuperModelListener.java (renamed from service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModelListener.java)21
-rw-r--r--service-monitor/src/main/java/com/yahoo/vespa/service/monitor/DuperModelProvider.java6
-rw-r--r--service-monitor/src/main/java/com/yahoo/vespa/service/slobrok/SlobrokMonitorManagerImpl.java4
-rw-r--r--service-monitor/src/test/java/com/yahoo/vespa/service/duper/DuperModelTest.java1
-rw-r--r--service-monitor/src/test/java/com/yahoo/vespa/service/model/ExampleModel.java2
-rw-r--r--storage/src/tests/distributor/garbagecollectiontest.cpp27
-rw-r--r--storage/src/tests/persistence/processalltest.cpp12
-rw-r--r--storage/src/vespa/storage/distributor/bucketdb/bucketdbmetricupdater.cpp10
-rw-r--r--storage/src/vespa/storage/distributor/bucketdb/bucketdbmetricupdater.h8
-rw-r--r--storage/src/vespa/storage/distributor/distributorinterface.h2
-rw-r--r--storage/src/vespa/storage/distributor/distributormetricsset.cpp2
-rw-r--r--storage/src/vespa/storage/distributor/distributormetricsset.h2
-rw-r--r--storage/src/vespa/storage/distributor/externaloperationhandler.h2
-rw-r--r--storage/src/vespa/storage/distributor/idealstatemetricsset.cpp25
-rw-r--r--storage/src/vespa/storage/distributor/idealstatemetricsset.h14
-rw-r--r--storage/src/vespa/storage/distributor/operations/idealstate/garbagecollectionoperation.cpp26
-rw-r--r--storage/src/vespa/storage/distributor/operations/idealstate/garbagecollectionoperation.h2
-rw-r--r--storage/src/vespa/storage/distributor/persistence_operation_metric_set.h4
-rw-r--r--storage/src/vespa/storage/persistence/processallhandler.cpp12
-rw-r--r--storageapi/src/tests/mbusprot/storageprotocoltest.cpp9
-rw-r--r--storageapi/src/vespa/storageapi/mbusprot/protobuf/feed.proto5
-rw-r--r--storageapi/src/vespa/storageapi/mbusprot/protocolserialization7.cpp11
-rw-r--r--storageapi/src/vespa/storageapi/message/removelocation.cpp5
-rw-r--r--storageapi/src/vespa/storageapi/message/removelocation.h9
-rw-r--r--vbench/src/vbench/vbench/vbench.cpp12
-rw-r--r--vespa-http-client/src/main/java/com/yahoo/vespa/http/client/runner/Runner.java13
-rw-r--r--vespa-testrunner-components/src/main/java/com/yahoo/vespa/hosted/testrunner/TestRunnerHandler.java5
-rwxr-xr-xvespabase/src/rhel-prestart.sh52
-rw-r--r--vespaclient/src/perl/lib/Yahoo/Vespa/Http.pm5
-rw-r--r--vespalib/src/tests/net/tls/openssl_impl/openssl_impl_test.cpp154
-rw-r--r--vespalib/src/tests/net/tls/transport_options/transport_options_reading_test.cpp41
-rw-r--r--vespalib/src/vespa/vespalib/data/slime/object_value.h13
-rw-r--r--vespalib/src/vespa/vespalib/geo/zcurve.cpp4
-rw-r--r--vespalib/src/vespa/vespalib/geo/zcurve.h7
-rw-r--r--vespalib/src/vespa/vespalib/net/crypto_engine.h6
-rw-r--r--vespalib/src/vespa/vespalib/net/socket_spec.cpp2
-rw-r--r--vespalib/src/vespa/vespalib/net/socket_spec.h2
-rw-r--r--vespalib/src/vespa/vespalib/net/tls/auto_reloading_tls_crypto_engine.cpp12
-rw-r--r--vespalib/src/vespa/vespalib/net/tls/auto_reloading_tls_crypto_engine.h2
-rw-r--r--vespalib/src/vespa/vespalib/net/tls/crypto_codec.cpp17
-rw-r--r--vespalib/src/vespa/vespalib/net/tls/crypto_codec.h12
-rw-r--r--vespalib/src/vespa/vespalib/net/tls/impl/openssl_crypto_codec_impl.cpp59
-rw-r--r--vespalib/src/vespa/vespalib/net/tls/impl/openssl_crypto_codec_impl.h32
-rw-r--r--vespalib/src/vespa/vespalib/net/tls/maybe_tls_crypto_engine.h2
-rw-r--r--vespalib/src/vespa/vespalib/net/tls/peer_policies.h9
-rw-r--r--vespalib/src/vespa/vespalib/net/tls/tls_crypto_engine.cpp8
-rw-r--r--vespalib/src/vespa/vespalib/net/tls/tls_crypto_engine.h2
-rw-r--r--vespalib/src/vespa/vespalib/net/tls/transport_security_options.cpp42
-rw-r--r--vespalib/src/vespa/vespalib/net/tls/transport_security_options.h33
-rw-r--r--vespalib/src/vespa/vespalib/net/tls/transport_security_options_reading.cpp9
-rw-r--r--vespalib/src/vespa/vespalib/test/make_tls_options_for_testing.cpp8
-rw-r--r--vespalib/src/vespa/vespalib/util/stash.cpp6
-rw-r--r--vespalib/src/vespa/vespalib/util/stash.h18
360 files changed, 4020 insertions, 2148 deletions
diff --git a/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/instanceconfirmation/InstanceValidatorTest.java b/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/instanceconfirmation/InstanceValidatorTest.java
index 2208b470ed8..fdf2bbfccff 100644
--- a/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/instanceconfirmation/InstanceValidatorTest.java
+++ b/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/instanceconfirmation/InstanceValidatorTest.java
@@ -201,7 +201,8 @@ public class InstanceValidatorTest {
ApplicationInfo::getApplicationId,
Function.identity()
)
- ));
+ ),
+ true);
SuperModelProvider superModelProvider = mock(SuperModelProvider.class);
when(superModelProvider.getSuperModel()).thenReturn(superModel);
diff --git a/client/src/main/java/ai/vespa/client/dsl/EndQuery.java b/client/src/main/java/ai/vespa/client/dsl/EndQuery.java
index 4facf6ce0fb..2af1e0bb49d 100644
--- a/client/src/main/java/ai/vespa/client/dsl/EndQuery.java
+++ b/client/src/main/java/ai/vespa/client/dsl/EndQuery.java
@@ -177,7 +177,7 @@ public class EndQuery {
} else if (others.isEmpty()) {
sb.append("order by ").append(orderStr);
} else {
- sb.append("order by ").append(orderStr).append(", ").append(others);
+ sb.append("order by ").append(orderStr).append(" ").append(others);
}
if (groupQueryStr != null) {
diff --git a/client/src/test/groovy/ai/vespa/client/dsl/QTest.groovy b/client/src/test/groovy/ai/vespa/client/dsl/QTest.groovy
index e363a9bcc13..19c87d6aecd 100644
--- a/client/src/test/groovy/ai/vespa/client/dsl/QTest.groovy
+++ b/client/src/test/groovy/ai/vespa/client/dsl/QTest.groovy
@@ -59,7 +59,7 @@ class QTest extends Specification {
.build()
expect:
- q == """yql=select * from sd1 where f1 contains "v1" and f2 contains "v2" or f3 contains "v3" and !(f4 contains "v4") order by f1 desc, f2 asc, limit 2 offset 1 timeout 3;&paramk1=paramv1"""
+ q == """yql=select * from sd1 where f1 contains "v1" and f2 contains "v2" or f3 contains "v3" and !(f4 contains "v4") order by f1 desc, f2 asc limit 2 offset 1 timeout 3;&paramk1=paramv1"""
}
def "matches"() {
diff --git a/config-model-api/abi-spec.json b/config-model-api/abi-spec.json
index 43527335802..90132d6924a 100644
--- a/config-model-api/abi-spec.json
+++ b/config-model-api/abi-spec.json
@@ -990,13 +990,16 @@
],
"methods": [
"public void <init>()",
- "public void <init>(java.util.Map)",
+ "public void <init>(java.util.Map, boolean)",
"public java.util.Map getModelsPerTenant()",
"public java.util.Map getModels()",
+ "public boolean isComplete()",
"public java.util.List getAllApplicationInfos()",
"public java.util.Optional getApplicationInfo(com.yahoo.config.provision.ApplicationId)",
- "public com.yahoo.config.model.api.SuperModel cloneAndSetApplication(com.yahoo.config.model.api.ApplicationInfo)",
- "public com.yahoo.config.model.api.SuperModel cloneAndRemoveApplication(com.yahoo.config.provision.ApplicationId)"
+ "public com.yahoo.config.model.api.SuperModel cloneAndSetApplication(com.yahoo.config.model.api.ApplicationInfo, boolean)",
+ "public com.yahoo.config.model.api.SuperModel cloneAndRemoveApplication(com.yahoo.config.provision.ApplicationId)",
+ "public com.yahoo.config.model.api.SuperModel cloneAsComplete()",
+ "public java.util.Set getApplicationIds()"
],
"fields": []
},
@@ -1010,7 +1013,8 @@
],
"methods": [
"public abstract void applicationActivated(com.yahoo.config.model.api.SuperModel, com.yahoo.config.model.api.ApplicationInfo)",
- "public abstract void applicationRemoved(com.yahoo.config.model.api.SuperModel, com.yahoo.config.provision.ApplicationId)"
+ "public abstract void applicationRemoved(com.yahoo.config.model.api.SuperModel, com.yahoo.config.provision.ApplicationId)",
+ "public abstract void notifyOfCompleteness(com.yahoo.config.model.api.SuperModel)"
],
"fields": []
},
diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/SuperModel.java b/config-model-api/src/main/java/com/yahoo/config/model/api/SuperModel.java
index 1735e08c930..15502dac1f1 100644
--- a/config-model-api/src/main/java/com/yahoo/config/model/api/SuperModel.java
+++ b/config-model-api/src/main/java/com/yahoo/config/model/api/SuperModel.java
@@ -20,13 +20,15 @@ import java.util.Set;
public class SuperModel {
private final Map<ApplicationId, ApplicationInfo> models;
+ private final boolean complete;
public SuperModel() {
- this.models = Collections.emptyMap();
+ this(Collections.emptyMap(), false);
}
- public SuperModel(Map<ApplicationId, ApplicationInfo> models) {
+ public SuperModel(Map<ApplicationId, ApplicationInfo> models, boolean complete) {
this.models = models;
+ this.complete = complete;
}
public Map<TenantName, Set<ApplicationInfo>> getModelsPerTenant() {
@@ -45,6 +47,8 @@ public class SuperModel {
return ImmutableMap.copyOf(models);
}
+ public boolean isComplete() { return complete; }
+
public List<ApplicationInfo> getAllApplicationInfos() {
return new ArrayList<>(models.values());
}
@@ -54,20 +58,22 @@ public class SuperModel {
return applicationInfo == null ? Optional.empty() : Optional.of(applicationInfo);
}
- public SuperModel cloneAndSetApplication(ApplicationInfo application) {
+ public SuperModel cloneAndSetApplication(ApplicationInfo application, boolean complete) {
Map<ApplicationId, ApplicationInfo> newModels = cloneModels(models);
newModels.put(application.getApplicationId(), application);
-
- return new SuperModel(newModels);
+ return new SuperModel(newModels, complete);
}
public SuperModel cloneAndRemoveApplication(ApplicationId applicationId) {
Map<ApplicationId, ApplicationInfo> newModels = cloneModels(models);
newModels.remove(applicationId);
-
- return new SuperModel(newModels);
+ return new SuperModel(newModels, complete);
}
+ public SuperModel cloneAsComplete() { return new SuperModel(models, true); }
+
+ public Set<ApplicationId> getApplicationIds() { return models.keySet(); }
+
private static Map<ApplicationId, ApplicationInfo> cloneModels(Map<ApplicationId, ApplicationInfo> models) {
return new LinkedHashMap<>(models);
}
diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/SuperModelListener.java b/config-model-api/src/main/java/com/yahoo/config/model/api/SuperModelListener.java
index 497c38af908..e66a7e1ef7e 100644
--- a/config-model-api/src/main/java/com/yahoo/config/model/api/SuperModelListener.java
+++ b/config-model-api/src/main/java/com/yahoo/config/model/api/SuperModelListener.java
@@ -17,4 +17,12 @@ public interface SuperModelListener {
* Application has been removed.
*/
void applicationRemoved(SuperModel superModel, ApplicationId id);
+
+ /**
+ * Invoked once all applications that were supposed to be deployed on bootstrap
+ * have been activated (and the respective {@link #applicationActivated(SuperModel, ApplicationInfo)
+ * applicationActivated} have been invoked). The SuperModel is then said to be "complete".
+ * @param superModel
+ */
+ void notifyOfCompleteness(SuperModel superModel);
}
diff --git a/config-model/pom.xml b/config-model/pom.xml
index 25b733985f5..33f6657561c 100644
--- a/config-model/pom.xml
+++ b/config-model/pom.xml
@@ -105,6 +105,10 @@
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpclient</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>org.apache.velocity</groupId>
+ <artifactId>velocity</artifactId>
+ </exclusion>
</exclusions>
</dependency>
<dependency>
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/TensorFieldProcessor.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/TensorFieldProcessor.java
index 9cd7fb24e42..2790f2ddf6e 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/TensorFieldProcessor.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/TensorFieldProcessor.java
@@ -24,16 +24,18 @@ public class TensorFieldProcessor extends Processor {
@Override
public void process(boolean validate, boolean documentsOnly) {
- if ( ! validate) return;
-
for (var field : search.allConcreteFields()) {
if ( field.getDataType() instanceof TensorDataType ) {
- validateIndexingScripsForTensorField(field);
- validateAttributeSettingForTensorField(field);
- processIndexSettingsForTensorField(field);
+ if (validate) {
+ validateIndexingScripsForTensorField(field);
+ validateAttributeSettingForTensorField(field);
+ }
+ processIndexSettingsForTensorField(field, validate);
}
else if (field.getDataType() instanceof CollectionDataType){
- validateDataTypeForCollectionField(field);
+ if (validate) {
+ validateDataTypeForCollectionField(field);
+ }
}
}
}
@@ -68,12 +70,12 @@ public class TensorFieldProcessor extends Processor {
}
}
- private void processIndexSettingsForTensorField(SDField field) {
+ private void processIndexSettingsForTensorField(SDField field, boolean validate) {
if (!field.doesIndexing()) {
return;
}
if (isTensorTypeThatSupportsHnswIndex(field)) {
- if (!field.doesAttributing()) {
+ if (validate && !field.doesAttributing()) {
fail(search, field, "A tensor that has an index must also be an attribute.");
}
var index = field.getIndex(field.getName());
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/LogserverContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/LogserverContainerCluster.java
index 5714d41ef67..38037c8a522 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/admin/LogserverContainerCluster.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/LogserverContainerCluster.java
@@ -4,6 +4,7 @@ package com.yahoo.vespa.model.admin;
import com.yahoo.config.model.deploy.DeployState;
import com.yahoo.config.model.producer.AbstractConfigProducer;
import com.yahoo.container.handler.ThreadpoolConfig;
+import com.yahoo.search.config.QrStartConfig;
import com.yahoo.vespa.model.container.ContainerCluster;
import com.yahoo.vespa.model.container.component.Handler;
@@ -27,6 +28,12 @@ public class LogserverContainerCluster extends ContainerCluster<LogserverContain
builder.maxthreads(10);
}
+ @Override
+ public void getConfig(QrStartConfig.Builder builder) {
+ super.getConfig(builder);
+ builder.jvm.heapsize(384);
+ }
+
protected boolean messageBusEnabled() { return false; }
private void addLogHandler() {
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java
index 071666b5bc7..f81757ac568 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java
@@ -20,6 +20,7 @@ import ai.vespa.metricsproxy.metric.dimensions.PublicDimensions;
import ai.vespa.metricsproxy.rpc.RpcServer;
import ai.vespa.metricsproxy.service.ConfigSentinelClient;
import ai.vespa.metricsproxy.service.SystemPollerProvider;
+import ai.vespa.metricsproxy.telegraf.TelegrafConfig;
import com.yahoo.config.model.deploy.DeployState;
import com.yahoo.config.model.producer.AbstractConfigProducer;
import com.yahoo.config.model.producer.AbstractConfigProducerRoot;
@@ -67,6 +68,7 @@ public class MetricsProxyContainerCluster extends ContainerCluster<MetricsProxyC
ApplicationDimensionsConfig.Producer,
ConsumersConfig.Producer,
MonitoringConfig.Producer,
+ TelegrafConfig.Producer,
ThreadpoolConfig.Producer,
MetricsNodesConfig.Producer
{
@@ -161,6 +163,25 @@ public class MetricsProxyContainerCluster extends ContainerCluster<MetricsProxyC
}
@Override
+ public void getConfig(TelegrafConfig.Builder builder) {
+ var userConsumers = getUserMetricsConsumers();
+ for (var consumer : userConsumers.values()) {
+ for (var cloudWatch : consumer.cloudWatches()) {
+ var cloudWatchBuilder = new TelegrafConfig.CloudWatch.Builder();
+ cloudWatchBuilder
+ .region(cloudWatch.region())
+ .namespace(cloudWatch.namespace())
+ .consumer(cloudWatch.consumer());
+ cloudWatch.hostedAuth().ifPresent(hostedAuth -> cloudWatchBuilder
+ .accessKeyName(hostedAuth.accessKeyName)
+ .secretKeyName(hostedAuth.secretKeyName));
+ cloudWatch.profile().ifPresent(cloudWatchBuilder::profile);
+ builder.cloudWatch(cloudWatchBuilder);
+ }
+ }
+ }
+
+ @Override
public void getConfig(ThreadpoolConfig.Builder builder) {
builder.maxthreads(10);
}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/CloudWatch.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/CloudWatch.java
new file mode 100644
index 00000000000..fd290409ea5
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/CloudWatch.java
@@ -0,0 +1,49 @@
+package com.yahoo.vespa.model.admin.monitoring;
+
+import java.util.Optional;
+
+/**
+ * Helper object for CloudWatch configuration.
+ *
+ * @author gjoranv
+ */
+public class CloudWatch {
+ private final String region;
+ private final String namespace;
+ private final MetricsConsumer consumer;
+
+ private HostedAuth hostedAuth;
+ private String profile;
+
+ public CloudWatch(String region, String namespace, MetricsConsumer consumer) {
+ this.region = region;
+ this.namespace = namespace;
+ this.consumer = consumer;
+ }
+
+ public String region() { return region; }
+ public String namespace() { return namespace; }
+ public String consumer() { return consumer.getId(); }
+
+ public Optional<HostedAuth> hostedAuth() {return Optional.ofNullable(hostedAuth); }
+ public Optional<String> profile() { return Optional.ofNullable(profile); }
+
+ public void setHostedAuth(HostedAuth hostedAuth) {
+ this.hostedAuth = hostedAuth;
+ }
+
+ public void setProfile(String profile) {
+ this.profile = profile;
+ }
+
+ public static class HostedAuth {
+ public final String accessKeyName;
+ public final String secretKeyName;
+
+ public HostedAuth(String accessKeyName, String secretKeyName) {
+ this.accessKeyName = accessKeyName;
+ this.secretKeyName = secretKeyName;
+ }
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/MetricsConsumer.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/MetricsConsumer.java
index 403788e3af6..a8fbcf50b02 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/MetricsConsumer.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/MetricsConsumer.java
@@ -2,9 +2,13 @@
package com.yahoo.vespa.model.admin.monitoring;
import javax.annotation.concurrent.Immutable;
+import java.util.ArrayList;
+import java.util.List;
import java.util.Map;
import java.util.Objects;
+import static java.util.Collections.unmodifiableList;
+
/**
* Represents an arbitrary metric consumer
*
@@ -17,6 +21,8 @@ public class MetricsConsumer {
private final String id;
private final MetricSet metricSet;
+ private final List<CloudWatch> cloudWatches = new ArrayList<>();
+
/**
* @param id the consumer
* @param metricSet the metrics for this consumer
@@ -39,4 +45,12 @@ public class MetricsConsumer {
return metricSet.getMetrics();
}
+ public void addCloudWatch(CloudWatch cloudWatch) {
+ cloudWatches.add(cloudWatch);
+ }
+
+ public List<CloudWatch> cloudWatches() {
+ return unmodifiableList(cloudWatches);
+ }
+
}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/builder/xml/CloudWatchBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/builder/xml/CloudWatchBuilder.java
new file mode 100644
index 00000000000..4b9d5542aa9
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/builder/xml/CloudWatchBuilder.java
@@ -0,0 +1,36 @@
+package com.yahoo.vespa.model.admin.monitoring.builder.xml;
+
+import com.yahoo.vespa.model.admin.monitoring.CloudWatch;
+import com.yahoo.vespa.model.admin.monitoring.CloudWatch.HostedAuth;
+import com.yahoo.vespa.model.admin.monitoring.MetricsConsumer;
+import org.w3c.dom.Element;
+
+import static com.yahoo.config.model.builder.xml.XmlHelper.getOptionalChildValue;
+
+/**
+ * @author gjoranv
+ */
+public class CloudWatchBuilder {
+
+ private static final String REGION_ATTRIBUTE = "region";
+ private static final String NAMESPACE_ATTRIBUTE = "namespace";
+ private static final String ACCESS_KEY_ELEMENT = "access-key-name";
+ private static final String SECRET_KEY_ELEMENT = "secret-key-name";
+ private static final String PROFILE_ELEMENT = "profile";
+
+ public static CloudWatch buildCloudWatch(Element cloudwatchElement, MetricsConsumer consumer) {
+ CloudWatch cloudWatch = new CloudWatch(cloudwatchElement.getAttribute(REGION_ATTRIBUTE),
+ cloudwatchElement.getAttribute(NAMESPACE_ATTRIBUTE),
+ consumer);
+
+ getOptionalChildValue(cloudwatchElement, PROFILE_ELEMENT).ifPresent(cloudWatch::setProfile);
+
+ getOptionalChildValue(cloudwatchElement, ACCESS_KEY_ELEMENT)
+ .ifPresent(accessKey -> cloudWatch.setHostedAuth(
+ new HostedAuth(accessKey,
+ getOptionalChildValue(cloudwatchElement, SECRET_KEY_ELEMENT)
+ .orElseThrow(() -> new IllegalArgumentException("Access key given without a secret key.")))));
+ return cloudWatch;
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/builder/xml/MetricsBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/builder/xml/MetricsBuilder.java
index f029dad01a9..b686288868f 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/builder/xml/MetricsBuilder.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/builder/xml/MetricsBuilder.java
@@ -42,7 +42,11 @@ public class MetricsBuilder {
throwIfIllegalConsumerId(metrics, consumerId);
MetricSet metricSet = buildMetricSet(consumerId, consumerElement);
- metrics.addConsumer(new MetricsConsumer(consumerId, metricSet));
+ var consumer = new MetricsConsumer(consumerId, metricSet);
+ for (Element cloudwatchElement : XML.getChildren(consumerElement, "cloudwatch")) {
+ consumer.addCloudWatch(CloudWatchBuilder.buildCloudWatch(cloudwatchElement, consumer));
+ }
+ metrics.addConsumer(consumer);
}
return metrics;
}
@@ -58,7 +62,7 @@ public class MetricsBuilder {
private MetricSet buildMetricSet(String consumerId, Element consumerElement) {
List<Metric> metrics = XML.getChildren(consumerElement, "metric").stream()
- .map(metricElement -> metricFromElement(metricElement))
+ .map(MetricsBuilder::metricFromElement)
.collect(Collectors.toCollection(LinkedList::new));
List<MetricSet> metricSets = XML.getChildren(consumerElement, "metric-set").stream()
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/EndpointCertificateSecretsValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/EndpointCertificateSecretsValidator.java
index f00ad0f0dbb..4b8bbd4ff08 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/EndpointCertificateSecretsValidator.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/EndpointCertificateSecretsValidator.java
@@ -12,7 +12,7 @@ public class EndpointCertificateSecretsValidator extends Validator {
@Override
public void validate(VespaModel model, DeployState deployState) {
if (deployState.endpointCertificateSecrets().isPresent() && deployState.endpointCertificateSecrets().get() == EndpointCertificateSecrets.MISSING) {
- throw new CertificateNotReadyException("TLS enabled, but could not retrieve certificate yet");
+ throw new CertificateNotReadyException("TLS enabled, but could not yet retrieve certificate for application " + deployState.getProperties().applicationId().serializedForm());
}
}
}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java
index 58679c63565..3632cb08da5 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java
@@ -215,6 +215,7 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat
super.getConfig(builder);
builder.jvm.verbosegc(true)
.availableProcessors(0)
+ .compressedClassSpaceSize(0) //TODO Reduce, next step is 512m
.minHeapsize(1536)
.heapsize(1536);
if (getMemoryPercentage().isPresent()) {
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java
index 6fa446bf365..3b132ab2342 100755
--- a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java
@@ -484,6 +484,7 @@ public abstract class ContainerCluster<CONTAINER extends Container>
builder.jvm
.verbosegc(false)
.availableProcessors(2)
+ .compressedClassSpaceSize(32)
.minHeapsize(32)
.heapsize(512)
.heapSizeAsPercentageOfPhysicalMemory(0)
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/http/AccessControl.java b/config-model/src/main/java/com/yahoo/vespa/model/container/http/AccessControl.java
index 6b1a94e16ae..67c7b67ad9e 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/container/http/AccessControl.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/http/AccessControl.java
@@ -1,7 +1,6 @@
// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.model.container.http;
-import com.google.common.collect.ImmutableList;
import com.yahoo.component.ComponentId;
import com.yahoo.component.ComponentSpecification;
import com.yahoo.config.application.api.DeployLogger;
@@ -30,7 +29,7 @@ public final class AccessControl {
public static final ComponentId ACCESS_CONTROL_CHAIN_ID = ComponentId.fromString("access-control-chain");
- private static final List<String> UNPROTECTED_HANDLERS = ImmutableList.of(
+ public static final List<String> UNPROTECTED_HANDLERS = List.of(
FileStatusHandlerComponent.CLASS,
ContainerCluster.APPLICATION_STATUS_HANDLER_CLASS,
ContainerCluster.BINDINGS_OVERVIEW_HANDLER_CLASS,
diff --git a/config-model/src/main/perl/vespa-deploy b/config-model/src/main/perl/vespa-deploy
index 59a84f5b0c0..a128e4a8d4c 100755
--- a/config-model/src/main/perl/vespa-deploy
+++ b/config-model/src/main/perl/vespa-deploy
@@ -154,7 +154,7 @@ my $command = shift;
$command ||= "help";
# The '--insecure' parameter is sadly required as it is not possible to disable or alter hostname verification with curl
-my $curl_command = $VESPA_HOME . '/libexec/vespa/vespa-curl-wrapper --insecure -A vespa-deploy --silent --show-error --connect-timeout 30 --max-time 1200';
+my $curl_command = $VESPA_HOME . '/libexec/vespa/vespa-curl-wrapper -A vespa-deploy --silent --show-error --connect-timeout 30 --max-time 1200';
my $CURL_PUT = $curl_command . ' --write-out \%{http_code} --request PUT';
my $CURL_GET = $curl_command . ' --request GET';
diff --git a/config-model/src/main/resources/schema/admin.rnc b/config-model/src/main/resources/schema/admin.rnc
index 7a3e2916f94..e3ba7dc500d 100644
--- a/config-model/src/main/resources/schema/admin.rnc
+++ b/config-model/src/main/resources/schema/admin.rnc
@@ -82,10 +82,25 @@ Metrics = element metrics {
element metric {
attribute id { xsd:Name } &
attribute display-name { xsd:Name }?
- }*
+ }* &
+ Cloudwatch?
}+
}
+Cloudwatch = element cloudwatch {
+ attribute region { xsd:Name } &
+ attribute namespace { xsd:string { pattern = "[\w_\-/#:\.]+" } } &
+ (
+ (
+ element access-key-name { xsd:Name } &
+ element secret-key-name { xsd:Name }
+ )
+ |
+ element profile { xsd:Name }
+ )?
+
+}
+
ClusterControllers = element cluster-controllers {
attribute standalone-zookeeper { xsd:string }? &
element cluster-controller {
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsConsumersTest.java b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsConsumersTest.java
new file mode 100644
index 00000000000..b441f1e1993
--- /dev/null
+++ b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsConsumersTest.java
@@ -0,0 +1,244 @@
+package com.yahoo.vespa.model.admin.metricsproxy;
+
+import ai.vespa.metricsproxy.core.ConsumersConfig;
+import com.yahoo.vespa.model.VespaModel;
+import com.yahoo.vespa.model.admin.monitoring.Metric;
+import com.yahoo.vespa.model.admin.monitoring.MetricSet;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+
+import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.TestMode.hosted;
+import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.TestMode.self_hosted;
+import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.checkMetric;
+import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.consumersConfigFromModel;
+import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.consumersConfigFromXml;
+import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.getCustomConsumer;
+import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.getModel;
+import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.servicesWithAdminOnly;
+import static com.yahoo.vespa.model.admin.monitoring.DefaultPublicConsumer.DEFAULT_PUBLIC_CONSUMER_ID;
+import static com.yahoo.vespa.model.admin.monitoring.DefaultPublicMetrics.defaultPublicMetricSet;
+import static com.yahoo.vespa.model.admin.monitoring.DefaultVespaMetrics.defaultVespaMetricSet;
+import static com.yahoo.vespa.model.admin.monitoring.NetworkMetrics.networkMetricSet;
+import static com.yahoo.vespa.model.admin.monitoring.SystemMetrics.systemMetricSet;
+import static com.yahoo.vespa.model.admin.monitoring.VespaMetricSet.vespaMetricSet;
+import static com.yahoo.vespa.model.admin.monitoring.VespaMetricsConsumer.VESPA_CONSUMER_ID;
+import static java.util.Collections.singleton;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * Tests for {@link MetricsProxyContainerCluster} related to metrics consumers.
+ *
+ * @author gjoranv
+ */
+public class MetricsConsumersTest {
+
+ private static int numPublicDefaultMetrics = defaultPublicMetricSet.getMetrics().size();
+ private static int numDefaultVespaMetrics = defaultVespaMetricSet.getMetrics().size();
+ private static int numVespaMetrics = vespaMetricSet.getMetrics().size();
+ private static int numSystemMetrics = systemMetricSet.getMetrics().size();
+ private static int numNetworkMetrics = networkMetricSet.getMetrics().size();
+ private static int numMetricsForVespaConsumer = numVespaMetrics + numSystemMetrics + numNetworkMetrics;
+
+ @Rule
+ public ExpectedException thrown = ExpectedException.none();
+
+ @Test
+ public void default_public_consumer_is_set_up_for_self_hosted() {
+ ConsumersConfig config = consumersConfigFromXml(servicesWithAdminOnly(), self_hosted);
+ assertEquals(2, config.consumer().size());
+ assertEquals(config.consumer(1).name(), DEFAULT_PUBLIC_CONSUMER_ID);
+
+ int numMetricsForPublicDefaultConsumer = defaultPublicMetricSet.getMetrics().size() + numSystemMetrics;
+ assertEquals(numMetricsForPublicDefaultConsumer, config.consumer(1).metric().size());
+ }
+
+ @Test
+ public void vespa_consumer_and_default_public_consumer_is_set_up_for_hosted() {
+ ConsumersConfig config = consumersConfigFromXml(servicesWithAdminOnly(), hosted);
+ assertEquals(2, config.consumer().size());
+ assertEquals(config.consumer(0).name(), VESPA_CONSUMER_ID);
+ assertEquals(config.consumer(1).name(), DEFAULT_PUBLIC_CONSUMER_ID);
+ }
+
+ @Test
+ public void vespa_consumer_is_always_present_and_has_all_vespa_metrics_and_all_system_metrics() {
+ ConsumersConfig config = consumersConfigFromXml(servicesWithAdminOnly(), self_hosted);
+ assertEquals(config.consumer(0).name(), VESPA_CONSUMER_ID);
+ assertEquals(numMetricsForVespaConsumer, config.consumer(0).metric().size());
+ }
+
+ @Test
+ public void vespa_consumer_can_be_amended_via_admin_object() {
+ VespaModel model = getModel(servicesWithAdminOnly(), self_hosted);
+ var additionalMetric = new Metric("additional-metric");
+ model.getAdmin().setAdditionalDefaultMetrics(new MetricSet("amender-metrics", singleton(additionalMetric)));
+
+ ConsumersConfig config = consumersConfigFromModel(model);
+ assertEquals(numMetricsForVespaConsumer + 1, config.consumer(0).metric().size());
+
+ ConsumersConfig.Consumer vespaConsumer = config.consumer(0);
+ assertTrue("Did not contain additional metric", checkMetric(vespaConsumer, additionalMetric));
+ }
+
+ @Test
+ public void vespa_is_a_reserved_consumer_id() {
+ assertReservedConsumerId("Vespa");
+ }
+
+ @Test
+ public void default_is_a_reserved_consumer_id() {
+ assertReservedConsumerId("default");
+ }
+
+ private void assertReservedConsumerId(String consumerId) {
+ String services = String.join("\n",
+ "<services>",
+ " <admin version='2.0'>",
+ " <adminserver hostalias='node1'/>",
+ " <metrics>",
+ " <consumer id='" + consumerId + "'/>",
+ " </metrics>",
+ " </admin>",
+ "</services>"
+ );
+ thrown.expect(IllegalArgumentException.class);
+ thrown.expectMessage("'" + consumerId + "' is not allowed as metrics consumer id");
+ consumersConfigFromXml(services, self_hosted);
+ }
+
+ @Test
+ public void vespa_consumer_id_is_allowed_for_hosted_infrastructure_applications() {
+ String services = String.join("\n",
+ "<services application-type='hosted-infrastructure'>",
+ " <admin version='4.0'>",
+ " <adminserver hostalias='node1'/>",
+ " <metrics>",
+ " <consumer id='Vespa'>",
+ " <metric id='custom.metric1'/>",
+ " </consumer>",
+ " </metrics>",
+ " </admin>",
+ "</services>"
+ );
+ VespaModel hostedModel = getModel(services, hosted);
+ ConsumersConfig config = consumersConfigFromModel(hostedModel);
+ assertEquals(2, config.consumer().size());
+
+ // All default metrics are retained
+ ConsumersConfig.Consumer vespaConsumer = config.consumer(0);
+ assertEquals(numMetricsForVespaConsumer + 1, vespaConsumer.metric().size());
+
+ Metric customMetric1 = new Metric("custom.metric1");
+ assertTrue("Did not contain metric: " + customMetric1, checkMetric(vespaConsumer, customMetric1));
+ }
+
+ @Test
+ public void consumer_id_is_case_insensitive() {
+ String services = String.join("\n",
+ "<services>",
+ " <admin version='2.0'>",
+ " <adminserver hostalias='node1'/>",
+ " <metrics>",
+ " <consumer id='A'/>",
+ " <consumer id='a'/>",
+ " </metrics>",
+ " </admin>",
+ "</services>"
+ );
+ thrown.expect(IllegalArgumentException.class);
+ thrown.expectMessage("'a' is used as id for two metrics consumers");
+ consumersConfigFromXml(services, self_hosted);
+ }
+
+ @Test
+ public void non_existent_metric_set_causes_exception() {
+ String services = String.join("\n",
+ "<services>",
+ " <admin version='2.0'>",
+ " <adminserver hostalias='node1'/>",
+ " <metrics>",
+ " <consumer id='consumer-with-non-existent-default-set'>",
+ " <metric-set id='non-existent'/>",
+ " </consumer>",
+ " </metrics>",
+ " </admin>",
+ "</services>"
+ );
+ thrown.expect(IllegalArgumentException.class);
+ thrown.expectMessage("No such metric-set: non-existent");
+ consumersConfigFromXml(services, self_hosted);
+ }
+
+ @Test
+ public void consumer_with_no_metric_set_has_its_own_metrics_plus_system_metrics_plus_default_vespa_metrics() {
+ String services = String.join("\n",
+ "<services>",
+ " <admin version='2.0'>",
+ " <adminserver hostalias='node1'/>",
+ " <metrics>",
+ " <consumer id='consumer-with-metrics-only'>",
+ " <metric id='custom.metric1'/>",
+ " <metric id='custom.metric2'/>",
+ " </consumer>",
+ " </metrics>",
+ " </admin>",
+ "</services>"
+ );
+ ConsumersConfig.Consumer consumer = getCustomConsumer(services);
+
+ assertEquals(numSystemMetrics + numDefaultVespaMetrics + 2, consumer.metric().size());
+
+ Metric customMetric1 = new Metric("custom.metric1");
+ Metric customMetric2 = new Metric("custom.metric2");
+ assertTrue("Did not contain metric: " + customMetric1, checkMetric(consumer, customMetric1));
+ assertTrue("Did not contain metric: " + customMetric2, checkMetric(consumer, customMetric2));
+ }
+
+ @Test
+ public void consumer_with_default_public_metric_set_has_all_public_metrics_plus_all_system_metrics_plus_its_own() {
+ String services = String.join("\n",
+ "<services>",
+ " <admin version='2.0'>",
+ " <adminserver hostalias='node1'/>",
+ " <metrics>",
+ " <consumer id='consumer-with-public-default-set'>",
+ " <metric-set id='public'/>",
+ " <metric id='custom.metric'/>",
+ " </consumer>",
+ " </metrics>",
+ " </admin>",
+ "</services>"
+ );
+ ConsumersConfig.Consumer consumer = getCustomConsumer(services);
+
+ assertEquals(numPublicDefaultMetrics + numSystemMetrics + 1, consumer.metric().size());
+
+ Metric customMetric = new Metric("custom.metric");
+ assertTrue("Did not contain metric: " + customMetric, checkMetric(consumer, customMetric));
+ }
+
+ @Test
+ public void consumer_with_vespa_metric_set_has_all_vespa_metrics_plus_all_system_metrics_plus_its_own() {
+ String services = String.join("\n",
+ "<services>",
+ " <admin version='2.0'>",
+ " <adminserver hostalias='node1'/>",
+ " <metrics>",
+ " <consumer id='consumer-with-vespa-set'>",
+ " <metric-set id='vespa'/>",
+ " <metric id='my.extra.metric'/>",
+ " </consumer>",
+ " </metrics>",
+ " </admin>",
+ "</services>"
+ );
+ ConsumersConfig.Consumer consumer = getCustomConsumer(services);
+ assertEquals(numVespaMetrics + numSystemMetrics + 1, consumer.metric().size());
+
+ Metric customMetric = new Metric("my.extra.metric");
+ assertTrue("Did not contain metric: " + customMetric, checkMetric(consumer, customMetric));
+ }
+
+}
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerClusterTest.java
index 9265e4437f1..bed77bd5c77 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerClusterTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerClusterTest.java
@@ -5,10 +5,9 @@
package com.yahoo.vespa.model.admin.metricsproxy;
-import ai.vespa.metricsproxy.core.ConsumersConfig;
-import ai.vespa.metricsproxy.http.metrics.MetricsV1Handler;
import ai.vespa.metricsproxy.http.application.ApplicationMetricsHandler;
import ai.vespa.metricsproxy.http.application.MetricsNodesConfig;
+import ai.vespa.metricsproxy.http.metrics.MetricsV1Handler;
import ai.vespa.metricsproxy.http.prometheus.PrometheusHandler;
import ai.vespa.metricsproxy.http.yamas.YamasHandler;
import ai.vespa.metricsproxy.metric.dimensions.ApplicationDimensionsConfig;
@@ -21,13 +20,9 @@ import com.yahoo.container.core.ApplicationMetadataConfig;
import com.yahoo.search.config.QrStartConfig;
import com.yahoo.vespa.model.VespaModel;
import com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyContainerCluster.AppDimensionNames;
-import com.yahoo.vespa.model.admin.monitoring.Metric;
-import com.yahoo.vespa.model.admin.monitoring.MetricSet;
import com.yahoo.vespa.model.container.component.Component;
import com.yahoo.vespa.model.container.component.Handler;
-import org.junit.Rule;
import org.junit.Test;
-import org.junit.rules.ExpectedException;
import java.util.Collection;
@@ -39,26 +34,17 @@ import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.M
import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.MY_TENANT;
import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.TestMode.hosted;
import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.TestMode.self_hosted;
-import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.checkMetric;
-import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.consumersConfigFromModel;
-import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.consumersConfigFromXml;
import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.getApplicationDimensionsConfig;
-import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.getCustomConsumer;
import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.getMetricsNodesConfig;
import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.getModel;
import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.getQrStartConfig;
-import static com.yahoo.vespa.model.admin.monitoring.DefaultPublicConsumer.DEFAULT_PUBLIC_CONSUMER_ID;
-import static com.yahoo.vespa.model.admin.monitoring.DefaultPublicMetrics.defaultPublicMetricSet;
-import static com.yahoo.vespa.model.admin.monitoring.DefaultVespaMetrics.defaultVespaMetricSet;
-import static com.yahoo.vespa.model.admin.monitoring.NetworkMetrics.networkMetricSet;
-import static com.yahoo.vespa.model.admin.monitoring.SystemMetrics.systemMetricSet;
-import static com.yahoo.vespa.model.admin.monitoring.VespaMetricSet.vespaMetricSet;
-import static com.yahoo.vespa.model.admin.monitoring.VespaMetricsConsumer.VESPA_CONSUMER_ID;
-import static java.util.Collections.singleton;
+import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.servicesWithAdminOnly;
import static java.util.stream.Collectors.toList;
import static org.hamcrest.CoreMatchers.endsWith;
import static org.hamcrest.CoreMatchers.hasItem;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotSame;
import static org.junit.Assert.assertThat;
import static org.junit.Assert.assertTrue;
@@ -67,16 +53,6 @@ import static org.junit.Assert.assertTrue;
*/
public class MetricsProxyContainerClusterTest {
- private static int numPublicDefaultMetrics = defaultPublicMetricSet.getMetrics().size();
- private static int numDefaultVespaMetrics = defaultVespaMetricSet.getMetrics().size();
- private static int numVespaMetrics = vespaMetricSet.getMetrics().size();
- private static int numSystemMetrics = systemMetricSet.getMetrics().size();
- private static int numNetworkMetrics = networkMetricSet.getMetrics().size();
- private static int numMetricsForVespaConsumer = numVespaMetrics + numSystemMetrics + numNetworkMetrics;
-
- @Rule
- public ExpectedException thrown = ExpectedException.none();
-
@Test
public void metrics_proxy_bundle_is_included_in_bundles_config() {
VespaModel model = getModel(servicesWithAdminOnly(), self_hosted);
@@ -105,10 +81,11 @@ public class MetricsProxyContainerClusterTest {
assertEquals(512, qrStartConfig.jvm().heapsize());
assertEquals(0, qrStartConfig.jvm().heapSizeAsPercentageOfPhysicalMemory());
assertEquals(2, qrStartConfig.jvm().availableProcessors());
- assertEquals(false, qrStartConfig.jvm().verbosegc());
+ assertFalse(qrStartConfig.jvm().verbosegc());
assertEquals("-XX:+UseG1GC -XX:MaxTenuringThreshold=15", qrStartConfig.jvm().gcopts());
assertEquals(512, qrStartConfig.jvm().stacksize());
assertEquals(0, qrStartConfig.jvm().directMemorySizeCache());
+ assertEquals(32, qrStartConfig.jvm().compressedClassSpaceSize());
assertEquals(75, qrStartConfig.jvm().baseMaxDirectMemorySize());
}
@@ -131,203 +108,6 @@ public class MetricsProxyContainerClusterTest {
}
@Test
- public void default_public_consumer_is_set_up_for_self_hosted() {
- ConsumersConfig config = consumersConfigFromXml(servicesWithAdminOnly(), self_hosted);
- assertEquals(2, config.consumer().size());
- assertEquals(config.consumer(1).name(), DEFAULT_PUBLIC_CONSUMER_ID);
-
- int numMetricsForPublicDefaultConsumer = defaultPublicMetricSet.getMetrics().size() + numSystemMetrics;
- assertEquals(numMetricsForPublicDefaultConsumer, config.consumer(1).metric().size());
- }
-
- @Test
- public void vespa_consumer_and_default_public_consumer_is_set_up_for_hosted() {
- ConsumersConfig config = consumersConfigFromXml(servicesWithAdminOnly(), hosted);
- assertEquals(2, config.consumer().size());
- assertEquals(config.consumer(0).name(), VESPA_CONSUMER_ID);
- assertEquals(config.consumer(1).name(), DEFAULT_PUBLIC_CONSUMER_ID);
- }
-
- @Test
- public void vespa_consumer_is_always_present_and_has_all_vespa_metrics_and_all_system_metrics() {
- ConsumersConfig config = consumersConfigFromXml(servicesWithAdminOnly(), self_hosted);
- assertEquals(config.consumer(0).name(), VESPA_CONSUMER_ID);
- assertEquals(numMetricsForVespaConsumer, config.consumer(0).metric().size());
- }
-
- @Test
- public void vespa_consumer_can_be_amended_via_admin_object() {
- VespaModel model = getModel(servicesWithAdminOnly(), self_hosted);
- var additionalMetric = new Metric("additional-metric");
- model.getAdmin().setAdditionalDefaultMetrics(new MetricSet("amender-metrics", singleton(additionalMetric)));
-
- ConsumersConfig config = consumersConfigFromModel(model);
- assertEquals(numMetricsForVespaConsumer + 1, config.consumer(0).metric().size());
-
- ConsumersConfig.Consumer vespaConsumer = config.consumer(0);
- assertTrue("Did not contain additional metric", checkMetric(vespaConsumer, additionalMetric));
- }
-
- @Test
- public void vespa_is_a_reserved_consumer_id() {
- assertReservedConsumerId("Vespa");
- }
-
- @Test
- public void default_is_a_reserved_consumer_id() {
- assertReservedConsumerId("default");
- }
-
- private void assertReservedConsumerId(String consumerId) {
- String services = String.join("\n",
- "<services>",
- " <admin version='2.0'>",
- " <adminserver hostalias='node1'/>",
- " <metrics>",
- " <consumer id='" + consumerId + "'/>",
- " </metrics>",
- " </admin>",
- "</services>"
- );
- thrown.expect(IllegalArgumentException.class);
- thrown.expectMessage("'" + consumerId + "' is not allowed as metrics consumer id");
- consumersConfigFromXml(services, self_hosted);
- }
-
- @Test
- public void vespa_consumer_id_is_allowed_for_hosted_infrastructure_applications() {
- String services = String.join("\n",
- "<services application-type='hosted-infrastructure'>",
- " <admin version='4.0'>",
- " <adminserver hostalias='node1'/>",
- " <metrics>",
- " <consumer id='Vespa'>",
- " <metric id='custom.metric1'/>",
- " </consumer>",
- " </metrics>",
- " </admin>",
- "</services>"
- );
- VespaModel hostedModel = getModel(services, hosted);
- ConsumersConfig config = consumersConfigFromModel(hostedModel);
- assertEquals(2, config.consumer().size());
-
- // All default metrics are retained
- ConsumersConfig.Consumer vespaConsumer = config.consumer(0);
- assertEquals(numMetricsForVespaConsumer + 1, vespaConsumer.metric().size());
-
- Metric customMetric1 = new Metric("custom.metric1");
- assertTrue("Did not contain metric: " + customMetric1, checkMetric(vespaConsumer, customMetric1));
- }
-
- @Test
- public void consumer_id_is_case_insensitive() {
- String services = String.join("\n",
- "<services>",
- " <admin version='2.0'>",
- " <adminserver hostalias='node1'/>",
- " <metrics>",
- " <consumer id='A'/>",
- " <consumer id='a'/>",
- " </metrics>",
- " </admin>",
- "</services>"
- );
- thrown.expect(IllegalArgumentException.class);
- thrown.expectMessage("'a' is used as id for two metrics consumers");
- consumersConfigFromXml(services, self_hosted);
- }
-
- @Test
- public void non_existent_metric_set_causes_exception() {
- String services = String.join("\n",
- "<services>",
- " <admin version='2.0'>",
- " <adminserver hostalias='node1'/>",
- " <metrics>",
- " <consumer id='consumer-with-non-existent-default-set'>",
- " <metric-set id='non-existent'/>",
- " </consumer>",
- " </metrics>",
- " </admin>",
- "</services>"
- );
- thrown.expect(IllegalArgumentException.class);
- thrown.expectMessage("No such metric-set: non-existent");
- consumersConfigFromXml(services, self_hosted);
- }
-
- @Test
- public void consumer_with_no_metric_set_has_its_own_metrics_plus_system_metrics_plus_default_vespa_metrics() {
- String services = String.join("\n",
- "<services>",
- " <admin version='2.0'>",
- " <adminserver hostalias='node1'/>",
- " <metrics>",
- " <consumer id='consumer-with-metrics-only'>",
- " <metric id='custom.metric1'/>",
- " <metric id='custom.metric2'/>",
- " </consumer>",
- " </metrics>",
- " </admin>",
- "</services>"
- );
- ConsumersConfig.Consumer consumer = getCustomConsumer(services);
-
- assertEquals(numSystemMetrics + numDefaultVespaMetrics + 2, consumer.metric().size());
-
- Metric customMetric1 = new Metric("custom.metric1");
- Metric customMetric2 = new Metric("custom.metric2");
- assertTrue("Did not contain metric: " + customMetric1, checkMetric(consumer, customMetric1));
- assertTrue("Did not contain metric: " + customMetric2, checkMetric(consumer, customMetric2));
- }
-
- @Test
- public void consumer_with_default_public_metric_set_has_all_public_metrics_plus_all_system_metrics_plus_its_own() {
- String services = String.join("\n",
- "<services>",
- " <admin version='2.0'>",
- " <adminserver hostalias='node1'/>",
- " <metrics>",
- " <consumer id='consumer-with-public-default-set'>",
- " <metric-set id='public'/>",
- " <metric id='custom.metric'/>",
- " </consumer>",
- " </metrics>",
- " </admin>",
- "</services>"
- );
- ConsumersConfig.Consumer consumer = getCustomConsumer(services);
-
- assertEquals(numPublicDefaultMetrics + numSystemMetrics + 1, consumer.metric().size());
-
- Metric customMetric = new Metric("custom.metric");
- assertTrue("Did not contain metric: " + customMetric, checkMetric(consumer, customMetric));
- }
-
- @Test
- public void consumer_with_vespa_metric_set_has_all_vespa_metrics_plus_all_system_metrics_plus_its_own() {
- String services = String.join("\n",
- "<services>",
- " <admin version='2.0'>",
- " <adminserver hostalias='node1'/>",
- " <metrics>",
- " <consumer id='consumer-with-vespa-set'>",
- " <metric-set id='vespa'/>",
- " <metric id='my.extra.metric'/>",
- " </consumer>",
- " </metrics>",
- " </admin>",
- "</services>"
- );
- ConsumersConfig.Consumer consumer = getCustomConsumer(services);
- assertEquals(numVespaMetrics + numSystemMetrics + 1, consumer.metric().size());
-
- Metric customMetric = new Metric("my.extra.metric");
- assertTrue("Did not contain metric: " + customMetric, checkMetric(consumer, customMetric));
- }
-
- @Test
public void hosted_application_propagates_application_dimensions() {
VespaModel hostedModel = getModel(servicesWithAdminOnly(), hosted);
ApplicationDimensionsConfig config = getApplicationDimensionsConfig(hostedModel);
@@ -357,16 +137,6 @@ public class MetricsProxyContainerClusterTest {
assertEquals(MetricsV1Handler.VALUES_PATH, node.metricsPath());
}
- private static String servicesWithAdminOnly() {
- return String.join("\n",
- "<services>",
- " <admin version='4.0'>",
- " <adminserver hostalias='node1'/>",
- " </admin>",
- "</services>"
- );
- }
-
private static String servicesWithTwoNodes() {
return String.join("\n",
"<services>",
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyModelTester.java b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyModelTester.java
index 7cbc9db5eb2..8ecb13d7ae5 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyModelTester.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyModelTester.java
@@ -55,6 +55,16 @@ class MetricsProxyModelTester {
: CONTAINER_CONFIG_ID;
}
+ static String servicesWithAdminOnly() {
+ return String.join("\n",
+ "<services>",
+ " <admin version='4.0'>",
+ " <adminserver hostalias='node1'/>",
+ " </admin>",
+ "</services>"
+ );
+ }
+
static boolean checkMetric(ConsumersConfig.Consumer consumer, Metric metric) {
for (ConsumersConfig.Consumer.Metric m : consumer.metric()) {
if (metric.name.equals(m.name()) && metric.outputName.equals(m.outputname()))
@@ -77,32 +87,32 @@ class MetricsProxyModelTester {
}
static ConsumersConfig consumersConfigFromModel(VespaModel model) {
- return new ConsumersConfig((ConsumersConfig.Builder) model.getConfig(new ConsumersConfig.Builder(), CLUSTER_CONFIG_ID));
+ return model.getConfig(ConsumersConfig.class, CLUSTER_CONFIG_ID);
}
static MetricsNodesConfig getMetricsNodesConfig(VespaModel model) {
- return new MetricsNodesConfig((MetricsNodesConfig.Builder) model.getConfig(new MetricsNodesConfig.Builder(), CLUSTER_CONFIG_ID));
+ return model.getConfig(MetricsNodesConfig.class, CLUSTER_CONFIG_ID);
}
static ApplicationDimensionsConfig getApplicationDimensionsConfig(VespaModel model) {
- return new ApplicationDimensionsConfig((ApplicationDimensionsConfig.Builder) model.getConfig(new ApplicationDimensionsConfig.Builder(), CLUSTER_CONFIG_ID));
+ return model.getConfig(ApplicationDimensionsConfig.class, CLUSTER_CONFIG_ID);
}
static QrStartConfig getQrStartConfig(VespaModel model) {
- return new QrStartConfig((QrStartConfig.Builder) model.getConfig(new QrStartConfig.Builder(), CLUSTER_CONFIG_ID));
+ return model.getConfig(QrStartConfig.class, CLUSTER_CONFIG_ID);
}
static NodeDimensionsConfig getNodeDimensionsConfig(VespaModel model, String configId) {
- return new NodeDimensionsConfig((NodeDimensionsConfig.Builder) model.getConfig(new NodeDimensionsConfig.Builder(), configId));
+ return model.getConfig(NodeDimensionsConfig.class, configId);
}
static VespaServicesConfig getVespaServicesConfig(String servicesXml) {
VespaModel model = getModel(servicesXml, self_hosted);
- return new VespaServicesConfig((VespaServicesConfig.Builder) model.getConfig(new VespaServicesConfig.Builder(), CONTAINER_CONFIG_ID));
+ return model.getConfig(VespaServicesConfig.class, CONTAINER_CONFIG_ID);
}
static RpcConnectorConfig getRpcConnectorConfig(VespaModel model) {
- return new RpcConnectorConfig((RpcConnectorConfig.Builder) model.getConfig(new RpcConnectorConfig.Builder(), CONTAINER_CONFIG_ID));
+ return model.getConfig(RpcConnectorConfig.class, CONTAINER_CONFIG_ID);
}
}
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/TelegrafTest.java b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/TelegrafTest.java
new file mode 100644
index 00000000000..144c45a7dd2
--- /dev/null
+++ b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/TelegrafTest.java
@@ -0,0 +1,87 @@
+package com.yahoo.vespa.model.admin.metricsproxy;
+
+import ai.vespa.metricsproxy.telegraf.TelegrafConfig;
+import com.yahoo.vespa.model.VespaModel;
+import org.junit.Test;
+
+import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.CLUSTER_CONFIG_ID;
+import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.TestMode.hosted;
+import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.getModel;
+import static org.junit.Assert.assertEquals;
+
+/**
+ * @author gjoranv
+ */
+public class TelegrafTest {
+
+ @Test
+ public void telegraf_config_is_generated_for_cloudwatch_in_services() {
+ String services = String.join("\n",
+ "<services>",
+ " <admin version='2.0'>",
+ " <adminserver hostalias='node1'/>",
+ " <metrics>",
+ " <consumer id='cloudwatch-consumer'>",
+ " <metric id='my-metric'/>",
+ " <cloudwatch region='us-east-1' namespace='my-namespace' >",
+ " <access-key-name>my-access-key</access-key-name>",
+ " <secret-key-name>my-secret-key</secret-key-name>",
+ " </cloudwatch>",
+ " </consumer>",
+ " </metrics>",
+ " </admin>",
+ "</services>"
+ );
+ VespaModel hostedModel = getModel(services, hosted);
+ TelegrafConfig config = hostedModel.getConfig(TelegrafConfig.class, CLUSTER_CONFIG_ID);
+ var cloudWatch0 = config.cloudWatch(0);
+ assertEquals("cloudwatch-consumer", cloudWatch0.consumer());
+ assertEquals("us-east-1", cloudWatch0.region());
+ assertEquals("my-namespace", cloudWatch0.namespace());
+ assertEquals("my-access-key", cloudWatch0.accessKeyName());
+ assertEquals("my-secret-key", cloudWatch0.secretKeyName());
+ assertEquals("", cloudWatch0.profile());
+ }
+
+ @Test
+ public void multiple_cloudwatches_are_allowed_for_the_same_consumer() {
+ String services = String.join("\n",
+ "<services>",
+ " <admin version='2.0'>",
+ " <adminserver hostalias='node1'/>",
+ " <metrics>",
+ " <consumer id='cloudwatch-consumer'>",
+ " <metric id='my-metric'/>",
+ " <cloudwatch region='us-east-1' namespace='namespace-1' >",
+ " <access-key-name>access-key-1</access-key-name>",
+ " <secret-key-name>secret-key-1</secret-key-name>",
+ " </cloudwatch>",
+ " <cloudwatch region='us-east-1' namespace='namespace-2' >",
+ " <profile>profile-2</profile>",
+ " </cloudwatch>",
+ " </consumer>",
+ " </metrics>",
+ " </admin>",
+ "</services>"
+ );
+ VespaModel hostedModel = getModel(services, hosted);
+ TelegrafConfig config = hostedModel.getConfig(TelegrafConfig.class, CLUSTER_CONFIG_ID);
+
+ var cloudWatch0 = config.cloudWatch(0);
+ assertEquals("cloudwatch-consumer", cloudWatch0.consumer());
+ assertEquals("us-east-1", cloudWatch0.region());
+ assertEquals("namespace-1", cloudWatch0.namespace());
+ assertEquals("access-key-1", cloudWatch0.accessKeyName());
+ assertEquals("secret-key-1", cloudWatch0.secretKeyName());
+ assertEquals("", cloudWatch0.profile());
+
+ var cloudWatch1 = config.cloudWatch(1);
+ assertEquals("cloudwatch-consumer", cloudWatch1.consumer());
+ assertEquals("us-east-1", cloudWatch1.region());
+ assertEquals("namespace-2", cloudWatch1.namespace());
+ assertEquals("", cloudWatch1.accessKeyName());
+ assertEquals("", cloudWatch1.secretKeyName());
+ assertEquals("profile-2", cloudWatch1.profile());
+ }
+
+}
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/EndpointCertificateSecretsValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/EndpointCertificateSecretsValidatorTest.java
index 21df39ebde8..318a0630c4d 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/EndpointCertificateSecretsValidatorTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/EndpointCertificateSecretsValidatorTest.java
@@ -47,7 +47,7 @@ public class EndpointCertificateSecretsValidatorTest {
VespaModel model = new VespaModel(new NullConfigModelRegistry(), deployState);
exceptionRule.expect(CertificateNotReadyException.class);
- exceptionRule.expectMessage("TLS enabled, but could not retrieve certificate yet");
+ exceptionRule.expectMessage("TLS enabled, but could not yet retrieve certificate for application default:default:default");
new EndpointCertificateSecretsValidator().validate(model, deployState);
}
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java
index d1c5e344c24..ce565989c18 100755
--- a/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java
@@ -97,6 +97,7 @@ public class ContainerClusterTest {
cluster.getConfig(qsB);
QrStartConfig qsC= new QrStartConfig(qsB);
assertEquals(expectedMemoryPercentage, qsC.jvm().heapSizeAsPercentageOfPhysicalMemory());
+ assertEquals(0, qsC.jvm().compressedClassSpaceSize());
}
@Test
@@ -156,6 +157,7 @@ public class ContainerClusterTest {
QrStartConfig qrStartConfig = new QrStartConfig(qrBuilder);
assertEquals(32, qrStartConfig.jvm().minHeapsize());
assertEquals(512, qrStartConfig.jvm().heapsize());
+ assertEquals(32, qrStartConfig.jvm().compressedClassSpaceSize());
assertEquals(0, qrStartConfig.jvm().heapSizeAsPercentageOfPhysicalMemory());
ThreadpoolConfig.Builder tpBuilder = new ThreadpoolConfig.Builder();
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/DocprocBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/DocprocBuilderTest.java
index 9ecd33f4273..eda90b03147 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/DocprocBuilderTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/DocprocBuilderTest.java
@@ -28,7 +28,11 @@ import java.util.Map;
import static org.hamcrest.CoreMatchers.is;
import static org.hamcrest.core.IsNull.notNullValue;
-import static org.junit.Assert.*;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.assertThat;
+
/**
* @author einarmr
@@ -204,24 +208,25 @@ public class DocprocBuilderTest extends DomBuilderTest {
@Test
public void testBundlesConfig() {
- assertThat(bundlesConfig.bundle().size(), is(0));
+ assertTrue(bundlesConfig.bundle().isEmpty());
}
@Test
public void testSchemaMappingConfig() {
- assertThat(schemamappingConfig.fieldmapping().size(), is(0));
+ assertTrue(schemamappingConfig.fieldmapping().isEmpty());
}
@Test
public void testQrStartConfig() {
QrStartConfig.Jvm jvm = qrStartConfig.jvm();
- assertThat(jvm.server(), is(true));
- assertThat(jvm.verbosegc(), is(true));
- assertThat(jvm.gcopts(), is("-XX:+UseG1GC -XX:MaxTenuringThreshold=15"));
- assertThat(jvm.minHeapsize(), is(1536));
- assertThat(jvm.heapsize(), is(1536));
- assertThat(jvm.stacksize(), is(512));
- assertThat(qrStartConfig.ulimitv(), is(""));
+ assertTrue(jvm.server());
+ assertTrue(jvm.verbosegc());
+ assertEquals("-XX:+UseG1GC -XX:MaxTenuringThreshold=15", jvm.gcopts());
+ assertEquals(1536, jvm.minHeapsize());
+ assertEquals(1536, jvm.heapsize());
+ assertEquals(512, jvm.stacksize());
+ assertTrue(qrStartConfig.ulimitv().isEmpty());
+ assertEquals(0, jvm.compressedClassSpaceSize());
}
}
diff --git a/config-model/src/test/schema-test-files/services.xml b/config-model/src/test/schema-test-files/services.xml
index 1bf42650123..604e2abbbae 100644
--- a/config-model/src/test/schema-test-files/services.xml
+++ b/config-model/src/test/schema-test-files/services.xml
@@ -15,14 +15,25 @@
<slobrok hostalias="rtc-1" />
</slobroks>
<metrics>
- <consumer id="my-consumer">
+ <consumer id="cloudwatch-hosted">
<metric-set id="my-set" />
<metric id="my-metric"/>
<metric id="my-metric2" display-name="my-metric3"/>
<metric display-name="my-metric4" id="my-metric4.avg"/>
+ <cloudwatch region="us-east1" namespace="my-namespace">
+ <access-key-name>my-access-key</access-key-name>
+ <secret-key-name>my-secret-key</secret-key-name>
+ </cloudwatch>
</consumer>
- <consumer id="my-consumer2">
- <metric-set id="my-set2" />
+ <consumer id="cloudwatch-self-hosted-with-default-auth">
+ <metric-set id="public" />
+ <cloudwatch region="us-east1" namespace="namespace_legal.chars:/#1" />
+ </consumer>
+ <consumer id="cloudwatch-self-hosted-with-profile">
+ <metric id="my-custom-metric" />
+ <cloudwatch region="us-east1" namespace="another-namespace">
+ <profile>profile-in-credentials-file</profile>
+ </cloudwatch>
</consumer>
</metrics>
<logforwarding>
diff --git a/config-provisioning/abi-spec.json b/config-provisioning/abi-spec.json
index 92432d1cab4..9a091f1161c 100644
--- a/config-provisioning/abi-spec.json
+++ b/config-provisioning/abi-spec.json
@@ -507,7 +507,7 @@
],
"methods": [
"public abstract java.util.Optional getDeployment(com.yahoo.config.provision.ApplicationId)",
- "public abstract java.util.Map getSupportedInfraDeployments()"
+ "public abstract void activateAllSupportedInfraApplications(boolean)"
],
"fields": []
},
diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/InfraDeployer.java b/config-provisioning/src/main/java/com/yahoo/config/provision/InfraDeployer.java
index 6fbabfd0c95..363732ee8a7 100644
--- a/config-provisioning/src/main/java/com/yahoo/config/provision/InfraDeployer.java
+++ b/config-provisioning/src/main/java/com/yahoo/config/provision/InfraDeployer.java
@@ -1,7 +1,6 @@
// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.config.provision;
-import java.util.Map;
import java.util.Optional;
/**
@@ -17,6 +16,6 @@ public interface InfraDeployer {
*/
Optional<Deployment> getDeployment(ApplicationId application);
- /** Returns deployments by application id for the supported infrastructure applications in this zone */
- Map<ApplicationId, Deployment> getSupportedInfraDeployments();
+ /** Deploys all supported infrastructure applications in this zone. */
+ void activateAllSupportedInfraApplications(boolean propagateException);
}
diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/zone/RoutingMethod.java b/config-provisioning/src/main/java/com/yahoo/config/provision/zone/RoutingMethod.java
index 892ac639198..02da2ab98e2 100644
--- a/config-provisioning/src/main/java/com/yahoo/config/provision/zone/RoutingMethod.java
+++ b/config-provisioning/src/main/java/com/yahoo/config/provision/zone/RoutingMethod.java
@@ -14,4 +14,12 @@ public enum RoutingMethod {
/** Routing happens through a dedicated layer 4 load balancer */
exclusive,
+ /** Routing happens through a shared layer 4 load balancer */
+ sharedLayer4;
+
+ /** Returns whether this method routes requests directly to the Vespa container cluster */
+ public boolean isDirect() {
+ return this == exclusive || this == sharedLayer4;
+ }
+
}
diff --git a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ConfigProxyRpcServer.java b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ConfigProxyRpcServer.java
index 9d33824e0be..b73cf89d1b4 100644
--- a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ConfigProxyRpcServer.java
+++ b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ConfigProxyRpcServer.java
@@ -162,9 +162,9 @@ public class ConfigProxyRpcServer implements Runnable, TargetWatcher, RpcServer
dispatchRpcRequest(req, () -> {
StringBuilder sb = new StringBuilder();
sb.append("\nDelayed responses queue size: ");
- sb.append(proxyServer.delayedResponses.size());
+ sb.append(proxyServer.delayedResponses().size());
sb.append("\nContents: ");
- for (DelayedResponse delayed : proxyServer.delayedResponses.responses()) {
+ for (DelayedResponse delayed : proxyServer.delayedResponses().responses()) {
sb.append(delayed.getRequest().toString()).append("\n");
}
@@ -357,7 +357,7 @@ public class ConfigProxyRpcServer implements Runnable, TargetWatcher, RpcServer
@Override
public void notifyTargetInvalid(Target target) {
log.log(LogLevel.DEBUG, () -> "Target invalid " + target);
- for (Iterator<DelayedResponse> it = proxyServer.delayedResponses.responses().iterator(); it.hasNext(); ) {
+ for (Iterator<DelayedResponse> it = proxyServer.delayedResponses().responses().iterator(); it.hasNext(); ) {
DelayedResponse delayed = it.next();
JRTServerConfigRequest request = delayed.getRequest();
if (request.getRequest().target().equals(target)) {
diff --git a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ConfigSourceClient.java b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ConfigSourceClient.java
index 2f8a1b463e3..9fb78e7e812 100644
--- a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ConfigSourceClient.java
+++ b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ConfigSourceClient.java
@@ -26,4 +26,6 @@ interface ConfigSourceClient {
void updateSubscribers(RawConfig config);
+ DelayedResponses delayedResponses();
+
}
diff --git a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/MemoryCacheConfigClient.java b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/MemoryCacheConfigClient.java
index f9f5c475723..51446882025 100644
--- a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/MemoryCacheConfigClient.java
+++ b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/MemoryCacheConfigClient.java
@@ -18,6 +18,7 @@ class MemoryCacheConfigClient implements ConfigSourceClient {
private final static Logger log = Logger.getLogger(MemoryCacheConfigClient.class.getName());
private final MemoryCache cache;
+ private final DelayedResponses delayedResponses = new DelayedResponses();
MemoryCacheConfigClient(MemoryCache cache) {
this.cache = cache;
@@ -61,4 +62,9 @@ class MemoryCacheConfigClient implements ConfigSourceClient {
@Override
public void updateSubscribers(RawConfig config) {}
+ @Override
+ public DelayedResponses delayedResponses() {
+ return delayedResponses;
+ }
+
}
diff --git a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ProxyServer.java b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ProxyServer.java
index d77206aee81..545b962f6ff 100644
--- a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ProxyServer.java
+++ b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ProxyServer.java
@@ -1,7 +1,6 @@
// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.config.proxy;
-import com.yahoo.concurrent.DaemonThreadFactory;
import com.yahoo.config.subscription.ConfigSourceSet;
import com.yahoo.jrt.Spec;
import com.yahoo.jrt.Supervisor;
@@ -10,20 +9,15 @@ import com.yahoo.log.LogLevel;
import com.yahoo.log.LogSetup;
import com.yahoo.log.event.Event;
import com.yahoo.vespa.config.RawConfig;
-import com.yahoo.vespa.config.TimingValues;
import com.yahoo.vespa.config.protocol.JRTServerConfigRequest;
import com.yahoo.vespa.config.proxy.filedistribution.FileDistributionAndUrlDownload;
import com.yahoo.yolean.system.CatchSignals;
import java.util.List;
-import java.util.concurrent.Executors;
-import java.util.concurrent.ScheduledExecutorService;
-import java.util.concurrent.ScheduledFuture;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.logging.Logger;
import static com.yahoo.vespa.config.proxy.Mode.ModeName.DEFAULT;
-import static java.util.concurrent.TimeUnit.SECONDS;
/**
* A proxy server that handles RPC config requests. The proxy can run in two modes:
@@ -40,59 +34,34 @@ public class ProxyServer implements Runnable {
private final static Logger log = Logger.getLogger(ProxyServer.class.getName());
private final AtomicBoolean signalCaught = new AtomicBoolean(false);
-
- // Scheduled executor that periodically checks for requests that have timed out and response should be returned to clients
- private final ScheduledExecutorService scheduler = Executors.newScheduledThreadPool(1, new DaemonThreadFactory());
private final Supervisor supervisor = new Supervisor(new Transport(JRT_TRANSPORT_THREADS));
- private ScheduledFuture<?> delayedResponseScheduler;
private final ConfigProxyRpcServer rpcServer;
- final DelayedResponses delayedResponses;
private ConfigSourceSet configSource;
private volatile ConfigSourceClient configClient;
- private final TimingValues timingValues;
private final MemoryCache memoryCache;
- private static final double timingValuesRatio = 0.8;
- private final static TimingValues defaultTimingValues;
private final FileDistributionAndUrlDownload fileDistributionAndUrlDownload;
private volatile Mode mode = new Mode(DEFAULT);
- static {
- // Proxy should time out before clients upon subscription.
- TimingValues tv = new TimingValues();
- tv.setUnconfiguredDelay((long)(tv.getUnconfiguredDelay()* timingValuesRatio)).
- setConfiguredErrorDelay((long)(tv.getConfiguredErrorDelay()* timingValuesRatio)).
- setSubscribeTimeout((long)(tv.getSubscribeTimeout()* timingValuesRatio)).
- setConfiguredErrorTimeout(-1); // Never cache errors
- defaultTimingValues = tv;
- }
-
- ProxyServer(Spec spec, ConfigSourceSet source, TimingValues timingValues,
- MemoryCache memoryCache, ConfigSourceClient configClient) {
- this.delayedResponses = new DelayedResponses();
+ ProxyServer(Spec spec, ConfigSourceSet source, MemoryCache memoryCache, ConfigSourceClient configClient) {
this.configSource = source;
log.log(LogLevel.DEBUG, "Using config source '" + source);
- this.timingValues = timingValues;
this.memoryCache = memoryCache;
this.rpcServer = createRpcServer(spec);
- this.configClient = createClient(rpcServer, delayedResponses, source, timingValues, memoryCache, configClient);
+ this.configClient = (configClient == null) ? createRpcClient(rpcServer, source, memoryCache) : configClient;
this.fileDistributionAndUrlDownload = new FileDistributionAndUrlDownload(supervisor, source);
}
+ @Override
public void run() {
if (rpcServer != null) {
Thread t = new Thread(rpcServer);
t.setName("RpcServer");
t.start();
}
- // Wait for 5 seconds initially, then run every second
- delayedResponseScheduler = scheduler.scheduleAtFixedRate(new DelayedResponseHandler(delayedResponses,
- memoryCache,
- rpcServer),
- 5, 1, SECONDS);
}
RawConfig resolveConfig(JRTServerConfigRequest req) {
@@ -124,7 +93,7 @@ public class ProxyServer implements Runnable {
break;
case DEFAULT:
flush();
- configClient = createRpcClient();
+ configClient = createRpcClient(rpcServer, configSource, memoryCache);
this.mode = new Mode(modeName);
break;
default:
@@ -133,20 +102,12 @@ public class ProxyServer implements Runnable {
log.log(LogLevel.INFO, "Switched from '" + oldMode.name().toLowerCase() + "' mode to '" + getMode().name().toLowerCase() + "' mode");
}
- private ConfigSourceClient createClient(RpcServer rpcServer, DelayedResponses delayedResponses,
- ConfigSourceSet source, TimingValues timingValues,
- MemoryCache memoryCache, ConfigSourceClient client) {
- return (client == null)
- ? new RpcConfigSourceClient(rpcServer, source, memoryCache, timingValues, delayedResponses)
- : client;
- }
-
private ConfigProxyRpcServer createRpcServer(Spec spec) {
return (spec == null) ? null : new ConfigProxyRpcServer(this, supervisor, spec); // TODO: Try to avoid first argument being 'this'
}
- private RpcConfigSourceClient createRpcClient() {
- return new RpcConfigSourceClient(rpcServer, configSource, memoryCache, timingValues, delayedResponses);
+ private static RpcConfigSourceClient createRpcClient(RpcServer rpcServer, ConfigSourceSet source, MemoryCache memoryCache) {
+ return new RpcConfigSourceClient(rpcServer, source, memoryCache);
}
private void setupSignalHandler() {
@@ -181,8 +142,7 @@ public class ProxyServer implements Runnable {
Event.started("configproxy");
ConfigSourceSet configSources = new ConfigSourceSet(properties.configSources);
- ProxyServer proxyServer = new ProxyServer(new Spec(null, port), configSources,
- defaultTimingValues(), new MemoryCache(), null);
+ ProxyServer proxyServer = new ProxyServer(new Spec(null, port), configSources, new MemoryCache(), null);
// catch termination and interrupt signal
proxyServer.setupSignalHandler();
Thread proxyserverThread = new Thread(proxyServer);
@@ -204,14 +164,6 @@ public class ProxyServer implements Runnable {
}
}
- static TimingValues defaultTimingValues() {
- return defaultTimingValues;
- }
-
- TimingValues getTimingValues() {
- return timingValues;
- }
-
// Cancels all config instances and flushes the cache. When this method returns,
// the cache will not be updated again before someone calls getConfig().
private synchronized void flush() {
@@ -222,7 +174,7 @@ public class ProxyServer implements Runnable {
void stop() {
Event.stopping("configproxy", "shutdown");
if (rpcServer != null) rpcServer.shutdown();
- if (delayedResponseScheduler != null) delayedResponseScheduler.cancel(true);
+ if (configClient != null) configClient.cancel();
flush();
fileDistributionAndUrlDownload.close();
}
@@ -242,7 +194,11 @@ public class ProxyServer implements Runnable {
void updateSourceConnections(List<String> sources) {
configSource = new ConfigSourceSet(sources);
flush();
- configClient = createRpcClient();
+ configClient = createRpcClient(rpcServer, configSource, memoryCache);
+ }
+
+ DelayedResponses delayedResponses() {
+ return configClient.delayedResponses();
}
}
diff --git a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/RpcConfigSourceClient.java b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/RpcConfigSourceClient.java
index ee843088086..2a33e8c6928 100644
--- a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/RpcConfigSourceClient.java
+++ b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/RpcConfigSourceClient.java
@@ -11,18 +11,23 @@ import com.yahoo.jrt.Supervisor;
import com.yahoo.jrt.Target;
import com.yahoo.jrt.Transport;
import com.yahoo.log.LogLevel;
-import com.yahoo.vespa.config.*;
+import com.yahoo.vespa.config.ConfigCacheKey;
+import com.yahoo.vespa.config.RawConfig;
+import com.yahoo.vespa.config.TimingValues;
import com.yahoo.vespa.config.protocol.JRTServerConfigRequest;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
-import java.util.Map;
import java.util.concurrent.DelayQueue;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ScheduledFuture;
import java.util.logging.Logger;
+import static java.util.concurrent.TimeUnit.SECONDS;
+
/**
* An Rpc client to a config source
*
@@ -31,6 +36,8 @@ import java.util.logging.Logger;
class RpcConfigSourceClient implements ConfigSourceClient {
private final static Logger log = Logger.getLogger(RpcConfigSourceClient.class.getName());
+ private static final double timingValuesRatio = 0.8;
+
private final Supervisor supervisor = new Supervisor(new Transport());
private final RpcServer rpcServer;
@@ -39,38 +46,37 @@ class RpcConfigSourceClient implements ConfigSourceClient {
private final Object activeSubscribersLock = new Object();
private final MemoryCache memoryCache;
private final DelayedResponses delayedResponses;
- private final TimingValues timingValues;
-
+ private final static TimingValues timingValues;
private final ExecutorService exec;
- private final Map<ConfigSourceSet, JRTConfigRequester> requesterPool;
-
+ private final JRTConfigRequester requester;
+ // Scheduled executor that periodically checks for requests that have timed out and response should be returned to clients
+ private final ScheduledExecutorService scheduler = Executors.newScheduledThreadPool(1, new DaemonThreadFactory());
+ private ScheduledFuture<?> delayedResponseScheduler;
+
+ static {
+ // Proxy should time out before clients upon subscription.
+ TimingValues tv = new TimingValues();
+ tv.setUnconfiguredDelay((long)(tv.getUnconfiguredDelay()* timingValuesRatio)).
+ setConfiguredErrorDelay((long)(tv.getConfiguredErrorDelay()* timingValuesRatio)).
+ setSubscribeTimeout((long)(tv.getSubscribeTimeout()* timingValuesRatio)).
+ setConfiguredErrorTimeout(-1); // Never cache errors
+ timingValues = tv;
+ }
- RpcConfigSourceClient(RpcServer rpcServer,
- ConfigSourceSet configSourceSet,
- MemoryCache memoryCache,
- TimingValues timingValues,
- DelayedResponses delayedResponses) {
+ RpcConfigSourceClient(RpcServer rpcServer, ConfigSourceSet configSourceSet, MemoryCache memoryCache) {
this.rpcServer = rpcServer;
this.configSourceSet = configSourceSet;
this.memoryCache = memoryCache;
- this.delayedResponses = delayedResponses;
- this.timingValues = timingValues;
+ this.delayedResponses = new DelayedResponses();
checkConfigSources();
exec = Executors.newCachedThreadPool(new DaemonThreadFactory("subscriber-"));
- requesterPool = createRequesterPool(configSourceSet, timingValues);
- }
-
- /**
- * Creates a requester (connection) pool of one entry, to be used each time this {@link RpcConfigSourceClient} is used
- * @param ccs a {@link ConfigSourceSet}
- * @param timingValues a {@link TimingValues}
- * @return requester map
- */
- private Map<ConfigSourceSet, JRTConfigRequester> createRequesterPool(ConfigSourceSet ccs, TimingValues timingValues) {
- Map<ConfigSourceSet, JRTConfigRequester> ret = new HashMap<>();
- if (ccs.getSources().isEmpty()) return ret; // unit test, just skip creating any requester
- ret.put(ccs, new JRTConfigRequester(new JRTConnectionPool(ccs), timingValues));
- return ret;
+ requester = JRTConfigRequester.create(configSourceSet, timingValues);
+ // Wait for 5 seconds initially, then run every second
+ delayedResponseScheduler = scheduler.scheduleAtFixedRate(
+ new DelayedResponseHandler(delayedResponses, memoryCache, rpcServer),
+ 5,
+ 1,
+ SECONDS);
}
/**
@@ -152,8 +158,8 @@ class RpcConfigSourceClient implements ConfigSourceClient {
log.log(LogLevel.DEBUG, () -> "Already a subscriber running for: " + configCacheKey);
} else {
log.log(LogLevel.DEBUG, () -> "Could not find good config in cache, creating subscriber for: " + configCacheKey);
- UpstreamConfigSubscriber subscriber = new UpstreamConfigSubscriber(input, this, configSourceSet,
- timingValues, requesterPool, memoryCache);
+ UpstreamConfigSubscriber subscriber =
+ new UpstreamConfigSubscriber(input, this, configSourceSet, timingValues, requester, memoryCache);
try {
subscriber.subscribe();
activeSubscribers.put(configCacheKey, subscriber);
@@ -169,6 +175,8 @@ class RpcConfigSourceClient implements ConfigSourceClient {
@Override
public void cancel() {
shutdownSourceConnections();
+ delayedResponseScheduler.cancel(true);
+ scheduler.shutdown();
}
/**
@@ -183,25 +191,18 @@ class RpcConfigSourceClient implements ConfigSourceClient {
activeSubscribers.clear();
}
exec.shutdown();
- for (JRTConfigRequester requester : requesterPool.values()) {
- requester.close();
- }
+ requester.close();
}
@Override
public String getActiveSourceConnection() {
- if (requesterPool.get(configSourceSet) != null) {
- return requesterPool.get(configSourceSet).getConnectionPool().getCurrent().getAddress();
- } else {
- return "";
- }
+ return requester.getConnectionPool().getCurrent().getAddress();
}
@Override
public List<String> getSourceConnections() {
ArrayList<String> ret = new ArrayList<>();
- final JRTConfigRequester jrtConfigRequester = requesterPool.get(configSourceSet);
- if (jrtConfigRequester != null) {
+ if (configSourceSet != null) {
ret.addAll(configSourceSet.getSources());
}
return ret;
@@ -244,4 +245,9 @@ class RpcConfigSourceClient implements ConfigSourceClient {
log.log(LogLevel.DEBUG, () -> "Finished updating config for " + config.getKey() + "," + config.getGeneration());
}
+ @Override
+ public DelayedResponses delayedResponses() {
+ return delayedResponses;
+ }
+
}
diff --git a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/UpstreamConfigSubscriber.java b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/UpstreamConfigSubscriber.java
index f8df16cb3d2..d8a8c5ce941 100644
--- a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/UpstreamConfigSubscriber.java
+++ b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/UpstreamConfigSubscriber.java
@@ -1,16 +1,15 @@
// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.config.proxy;
-import com.yahoo.config.subscription.ConfigSource;
import com.yahoo.config.subscription.ConfigSourceSet;
import com.yahoo.config.subscription.impl.GenericConfigHandle;
import com.yahoo.config.subscription.impl.GenericConfigSubscriber;
import com.yahoo.config.subscription.impl.JRTConfigRequester;
import com.yahoo.log.LogLevel;
import com.yahoo.vespa.config.ConfigKey;
-import com.yahoo.yolean.Exceptions;
import com.yahoo.vespa.config.RawConfig;
import com.yahoo.vespa.config.TimingValues;
+import com.yahoo.yolean.Exceptions;
import java.util.Map;
import java.util.logging.Logger;
@@ -24,26 +23,26 @@ public class UpstreamConfigSubscriber implements Subscriber {
private final RawConfig config;
private final ConfigSourceClient configSourceClient;
- private final ConfigSource configSourceSet;
+ private final ConfigSourceSet configSourceSet;
private final TimingValues timingValues;
- private final Map<ConfigSourceSet, JRTConfigRequester> requesterPool;
+ private final JRTConfigRequester requester;
private final MemoryCache memoryCache;
private GenericConfigSubscriber subscriber;
private GenericConfigHandle handle;
- UpstreamConfigSubscriber(RawConfig config, ConfigSourceClient configSourceClient, ConfigSource configSourceSet,
- TimingValues timingValues, Map<ConfigSourceSet, JRTConfigRequester> requesterPool,
+ UpstreamConfigSubscriber(RawConfig config, ConfigSourceClient configSourceClient, ConfigSourceSet configSourceSet,
+ TimingValues timingValues, JRTConfigRequester requester,
MemoryCache memoryCache) {
this.config = config;
this.configSourceClient = configSourceClient;
this.configSourceSet = configSourceSet;
this.timingValues = timingValues;
- this.requesterPool = requesterPool;
+ this.requester = requester;
this.memoryCache = memoryCache;
}
void subscribe() {
- subscriber = new GenericConfigSubscriber(requesterPool);
+ subscriber = new GenericConfigSubscriber(Map.of(configSourceSet, requester));
ConfigKey<?> key = config.getKey();
handle = subscriber.subscribe(new ConfigKey<>(key.getName(), key.getConfigId(), key.getNamespace()),
config.getDefContent(), configSourceSet, timingValues);
diff --git a/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/ConfigProxyRpcServerTest.java b/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/ConfigProxyRpcServerTest.java
index dc1c995fbb5..29bd38ea891 100644
--- a/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/ConfigProxyRpcServerTest.java
+++ b/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/ConfigProxyRpcServerTest.java
@@ -264,7 +264,7 @@ public class ConfigProxyRpcServerTest {
}
private static ProxyServer createTestServer(ConfigSourceSet source) {
- return new ProxyServer(null, source, ProxyServer.defaultTimingValues(), new MemoryCache(), null);
+ return new ProxyServer(null, source, new MemoryCache(), null);
}
private static class TestServer implements AutoCloseable {
diff --git a/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/MockConfigSourceClient.java b/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/MockConfigSourceClient.java
index 963c922d5b5..06e55eef4fa 100644
--- a/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/MockConfigSourceClient.java
+++ b/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/MockConfigSourceClient.java
@@ -16,6 +16,7 @@ import java.util.List;
public class MockConfigSourceClient implements ConfigSourceClient{
private final MockConfigSource configSource;
private final MemoryCache memoryCache;
+ private final DelayedResponses delayedResponses = new DelayedResponses();
MockConfigSourceClient(MockConfigSource configSource, MemoryCache memoryCache) {
this.configSource = configSource;
@@ -53,7 +54,9 @@ public class MockConfigSourceClient implements ConfigSourceClient{
}
@Override
- public void updateSubscribers(RawConfig config) {
+ public void updateSubscribers(RawConfig config) { }
+
+ @Override
+ public DelayedResponses delayedResponses() { return delayedResponses; }
- }
}
diff --git a/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/ProxyServerTest.java b/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/ProxyServerTest.java
index 712567774f1..f52598b3ee5 100644
--- a/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/ProxyServerTest.java
+++ b/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/ProxyServerTest.java
@@ -14,7 +14,11 @@ import org.junit.rules.TemporaryFolder;
import java.util.Optional;
import static org.hamcrest.CoreMatchers.is;
-import static org.junit.Assert.*;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertThat;
+import static org.junit.Assert.assertTrue;
/**
* @author hmusum
@@ -23,7 +27,7 @@ public class ProxyServerTest {
private final MemoryCache memoryCache = new MemoryCache();
private final MockConfigSource source = new MockConfigSource();
- private MockConfigSourceClient client = new MockConfigSourceClient(source, memoryCache);
+ private final MockConfigSourceClient client = new MockConfigSourceClient(source, memoryCache);
private ProxyServer proxy;
static final RawConfig fooConfig = ConfigTester.fooConfig;
@@ -54,7 +58,6 @@ public class ProxyServerTest {
public void basic() {
assertTrue(proxy.getMode().isDefault());
assertThat(proxy.getMemoryCache().size(), is(0));
- assertThat(proxy.getTimingValues(), is(ProxyServer.defaultTimingValues()));
ConfigTester tester = new ConfigTester();
final MemoryCache memoryCache = proxy.getMemoryCache();
@@ -222,7 +225,7 @@ public class ProxyServerTest {
private static ProxyServer createTestServer(ConfigSourceSet source,
ConfigSourceClient configSourceClient,
MemoryCache memoryCache) {
- return new ProxyServer(null, source, ProxyServer.defaultTimingValues(), memoryCache, configSourceClient);
+ return new ProxyServer(null, source, memoryCache, configSourceClient);
}
static RawConfig createConfigWithNextConfigGeneration(RawConfig config, int errorCode) {
diff --git a/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/RpcConfigSourceClientTest.java b/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/RpcConfigSourceClientTest.java
index 35f1dd8fcd8..8510b23bbd2 100644
--- a/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/RpcConfigSourceClientTest.java
+++ b/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/RpcConfigSourceClientTest.java
@@ -18,7 +18,6 @@ import static org.junit.Assert.assertEquals;
public class RpcConfigSourceClientTest {
private MockRpcServer rpcServer;
- private DelayedResponses delayedResponses;
private RpcConfigSourceClient rpcConfigSourceClient;
@Rule
@@ -28,10 +27,7 @@ public class RpcConfigSourceClientTest {
@Before
public void setup() {
rpcServer = new MockRpcServer();
- delayedResponses = new DelayedResponses();
- rpcConfigSourceClient =
- new RpcConfigSourceClient(rpcServer, new MockConfigSource(),
- new MemoryCache(), ProxyServer.defaultTimingValues(), delayedResponses);
+ rpcConfigSourceClient = new RpcConfigSourceClient(rpcServer, new MockConfigSource(), new MemoryCache());
}
@Test
@@ -98,7 +94,7 @@ public class RpcConfigSourceClientTest {
}
private void simulateClientRequestingConfig(RawConfig config) {
- delayedResponses.add(new DelayedResponse(JRTServerConfigRequestV3.createFromRequest(JRTConfigRequestFactory.createFromRaw(config, -10L).getRequest())));
+ rpcConfigSourceClient.delayedResponses().add(new DelayedResponse(JRTServerConfigRequestV3.createFromRequest(JRTConfigRequestFactory.createFromRaw(config, -10L).getRequest())));
}
private void configUpdatedSendResponse(RawConfig config) {
diff --git a/config/src/main/java/com/yahoo/config/subscription/ConfigSourceSet.java b/config/src/main/java/com/yahoo/config/subscription/ConfigSourceSet.java
index c799186435c..7472439d6a4 100755
--- a/config/src/main/java/com/yahoo/config/subscription/ConfigSourceSet.java
+++ b/config/src/main/java/com/yahoo/config/subscription/ConfigSourceSet.java
@@ -3,7 +3,11 @@ package com.yahoo.config.subscription;
import com.yahoo.log.LogLevel;
-import java.util.*;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Set;
import java.util.logging.Logger;
diff --git a/config/src/main/java/com/yahoo/config/subscription/ConfigSubscriber.java b/config/src/main/java/com/yahoo/config/subscription/ConfigSubscriber.java
index 2873855d1c2..3891d710fa3 100644
--- a/config/src/main/java/com/yahoo/config/subscription/ConfigSubscriber.java
+++ b/config/src/main/java/com/yahoo/config/subscription/ConfigSubscriber.java
@@ -319,6 +319,7 @@ public class ConfigSubscriber implements AutoCloseable {
@Override
public void close() {
synchronized (monitor) {
+ if (state == State.CLOSED) return;
state = State.CLOSED;
}
for (ConfigHandle<? extends ConfigInstance> h : subscriptionHandles) {
diff --git a/config/src/main/java/com/yahoo/config/subscription/impl/JRTConfigRequester.java b/config/src/main/java/com/yahoo/config/subscription/impl/JRTConfigRequester.java
index 6d08976b61c..49c5dcd343c 100644
--- a/config/src/main/java/com/yahoo/config/subscription/impl/JRTConfigRequester.java
+++ b/config/src/main/java/com/yahoo/config/subscription/impl/JRTConfigRequester.java
@@ -20,7 +20,6 @@ import com.yahoo.yolean.Exceptions;
import java.time.Duration;
import java.time.Instant;
import java.util.concurrent.ScheduledThreadPoolExecutor;
-import java.util.concurrent.ThreadFactory;
import java.util.concurrent.TimeUnit;
import java.util.logging.Level;
import java.util.logging.Logger;
@@ -37,15 +36,17 @@ public class JRTConfigRequester implements RequestWaiter {
private static final Logger log = Logger.getLogger(JRTConfigRequester.class.getName());
public static final ConfigSourceSet defaultSourceSet = ConfigSourceSet.createDefault();
+ private static final JRTManagedConnectionPools managedPool = new JRTManagedConnectionPools();
private static final int TRACELEVEL = 6;
private final TimingValues timingValues;
private int fatalFailures = 0; // independent of transientFailures
private int transientFailures = 0; // independent of fatalFailures
- private final ScheduledThreadPoolExecutor scheduler = new ScheduledThreadPoolExecutor(1, new JRTSourceThreadFactory());
+ private final ScheduledThreadPoolExecutor scheduler;
private Instant suspendWarningLogged = Instant.MIN;
private Instant noApplicationWarningLogged = Instant.MIN;
private static final Duration delayBetweenWarnings = Duration.ofSeconds(60);
private final ConnectionPool connectionPool;
+ private final ConfigSourceSet configSourceSet;
static final float randomFraction = 0.2f;
/* Time to be added to server timeout to create client timeout. This is the time allowed for the server to respond after serverTimeout has elapsed. */
private static final Double additionalTimeForClientTimeout = 10.0;
@@ -56,11 +57,23 @@ public class JRTConfigRequester implements RequestWaiter {
* @param connectionPool the connectionPool this requester should use
* @param timingValues timeouts and delays used when sending JRT config requests
*/
- public JRTConfigRequester(ConnectionPool connectionPool, TimingValues timingValues) {
+ JRTConfigRequester(ConfigSourceSet configSourceSet, ScheduledThreadPoolExecutor scheduler,
+ ConnectionPool connectionPool, TimingValues timingValues) {
+ this.configSourceSet = configSourceSet;
+ this.scheduler = scheduler;
this.connectionPool = connectionPool;
this.timingValues = timingValues;
}
+ /** Only for testing */
+ public JRTConfigRequester(ConnectionPool connectionPool, TimingValues timingValues) {
+ this(null, new ScheduledThreadPoolExecutor(1), connectionPool, timingValues);
+ }
+
+ public static JRTConfigRequester create(ConfigSourceSet sourceSet, TimingValues timingValues) {
+ return managedPool.acquire(sourceSet, timingValues);
+ }
+
/**
* Requests the config for the {@link com.yahoo.config.ConfigInstance} on the given {@link ConfigSubscription}
*
@@ -273,18 +286,8 @@ public class JRTConfigRequester implements RequestWaiter {
// Fake that we have logged to avoid printing warnings after this
suspendWarningLogged = Instant.now();
noApplicationWarningLogged = Instant.now();
-
- connectionPool.close();
- scheduler.shutdown();
- }
-
- private static class JRTSourceThreadFactory implements ThreadFactory {
- @Override
- public Thread newThread(Runnable runnable) {
- Thread t = new Thread(runnable, String.format("jrt-config-requester-%d", System.currentTimeMillis()));
- // We want a daemon thread to avoid hanging threads in case something goes wrong in the config system
- t.setDaemon(true);
- return t;
+ if (configSourceSet != null) {
+ managedPool.release(configSourceSet);
}
}
diff --git a/config/src/main/java/com/yahoo/config/subscription/impl/JRTConfigSubscription.java b/config/src/main/java/com/yahoo/config/subscription/impl/JRTConfigSubscription.java
index 39e6c69f539..a94a135f9d8 100644
--- a/config/src/main/java/com/yahoo/config/subscription/impl/JRTConfigSubscription.java
+++ b/config/src/main/java/com/yahoo/config/subscription/impl/JRTConfigSubscription.java
@@ -28,7 +28,7 @@ import com.yahoo.vespa.config.protocol.Payload;
public class JRTConfigSubscription<T extends ConfigInstance> extends ConfigSubscription<T> {
private JRTConfigRequester requester;
- private TimingValues timingValues;
+ private final TimingValues timingValues;
// Last time we got an OK JRT callback
private Instant lastOK = Instant.MIN;
@@ -156,7 +156,7 @@ public class JRTConfigSubscription<T extends ConfigInstance> extends ConfigSubsc
private JRTConfigRequester getRequester() {
JRTConfigRequester requester = subscriber.requesters().get(sources);
if (requester == null) {
- requester = new JRTConfigRequester(new JRTConnectionPool(sources), timingValues);
+ requester = JRTConfigRequester.create(sources, timingValues);
subscriber.requesters().put(sources, requester);
}
return requester;
diff --git a/config/src/main/java/com/yahoo/config/subscription/impl/JRTManagedConnectionPools.java b/config/src/main/java/com/yahoo/config/subscription/impl/JRTManagedConnectionPools.java
new file mode 100644
index 00000000000..32d2d962e4d
--- /dev/null
+++ b/config/src/main/java/com/yahoo/config/subscription/impl/JRTManagedConnectionPools.java
@@ -0,0 +1,66 @@
+package com.yahoo.config.subscription.impl;
+
+import com.yahoo.config.subscription.ConfigSourceSet;
+import com.yahoo.vespa.config.JRTConnectionPool;
+import com.yahoo.vespa.config.TimingValues;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.ScheduledThreadPoolExecutor;
+import java.util.concurrent.ThreadFactory;
+import java.util.concurrent.TimeUnit;
+
+public class JRTManagedConnectionPools {
+ private static class JRTSourceThreadFactory implements ThreadFactory {
+ @Override
+ public Thread newThread(Runnable runnable) {
+ Thread t = new Thread(runnable, String.format("jrt-config-requester-%d", System.currentTimeMillis()));
+ // We want a daemon thread to avoid hanging threads in case something goes wrong in the config system
+ t.setDaemon(true);
+ return t;
+ }
+ }
+ private static class CountedPool {
+ long count;
+ final JRTConnectionPool pool;
+ final ScheduledThreadPoolExecutor scheduler = new ScheduledThreadPoolExecutor(1, new JRTSourceThreadFactory());
+ CountedPool(JRTConnectionPool requester) {
+ this.pool = requester;
+ count = 0;
+ }
+ }
+
+ private final Map<ConfigSourceSet, CountedPool> pools = new HashMap<>();
+
+ public JRTConfigRequester acquire(ConfigSourceSet sourceSet, TimingValues timingValues) {
+ CountedPool countedPool;
+ synchronized (pools) {
+ countedPool = pools.get(sourceSet);
+ if (countedPool == null) {
+ countedPool = new CountedPool(new JRTConnectionPool(sourceSet));
+ pools.put(sourceSet, countedPool);
+ }
+ countedPool.count++;
+ }
+ return new JRTConfigRequester(sourceSet, countedPool.scheduler, countedPool.pool, timingValues);
+ }
+
+ public synchronized void release(ConfigSourceSet sourceSet) {
+ CountedPool countedPool;
+ synchronized (pools) {
+ countedPool = pools.get(sourceSet);
+ if (countedPool != null)
+ countedPool.count--;
+ if (countedPool == null || countedPool.count > 0) return;
+ pools.remove(sourceSet);
+ }
+
+ countedPool.pool.close();
+ countedPool.scheduler.shutdown();
+ try {
+ countedPool.scheduler.awaitTermination(30, TimeUnit.SECONDS);
+ } catch (InterruptedException e) {
+ throw new RuntimeException("Failed shutting down scheduler:", e);
+ }
+ }
+}
diff --git a/config/src/main/java/com/yahoo/vespa/config/JRTConnectionPool.java b/config/src/main/java/com/yahoo/vespa/config/JRTConnectionPool.java
index a4da996effd..326c1287468 100644
--- a/config/src/main/java/com/yahoo/vespa/config/JRTConnectionPool.java
+++ b/config/src/main/java/com/yahoo/vespa/config/JRTConnectionPool.java
@@ -112,19 +112,6 @@ public class JRTConnectionPool implements ConnectionPool {
return this;
}
- public String getAllSourceAddresses() {
- StringBuilder sb = new StringBuilder();
- synchronized (connections) {
- for (JRTConnection conn : connections.values()) {
- sb.append(conn.getAddress());
- sb.append(",");
- }
- }
- // Remove trailing ","
- sb.deleteCharAt(sb.length() - 1);
- return sb.toString();
- }
-
public String toString() {
StringBuilder sb = new StringBuilder();
synchronized (connections) {
diff --git a/config/src/main/java/com/yahoo/vespa/config/TimingValues.java b/config/src/main/java/com/yahoo/vespa/config/TimingValues.java
index 780cf657009..5d5967e56c4 100644
--- a/config/src/main/java/com/yahoo/vespa/config/TimingValues.java
+++ b/config/src/main/java/com/yahoo/vespa/config/TimingValues.java
@@ -12,12 +12,11 @@ public class TimingValues {
public static final long defaultNextConfigTimeout = 1000;
// See getters below for an explanation of how these values are used and interpreted
// All time values in milliseconds.
- private long successTimeout = 600000;
- private long errorTimeout = 20000;
- private long initialTimeout = 15000;
+ private final long successTimeout;
+ private final long errorTimeout;
+ private final long initialTimeout;
private long subscribeTimeout = 55000;
private long configuredErrorTimeout = -1; // Don't ever timeout (and do not use error response) when we are already configured
- private long nextConfigTimeout = defaultNextConfigTimeout;
private long fixedDelay = 5000;
private long unconfiguredDelay = 1000;
@@ -26,6 +25,9 @@ public class TimingValues {
private final Random rand;
public TimingValues() {
+ successTimeout = 600000;
+ errorTimeout = 20000;
+ initialTimeout = 15000;
this.rand = new Random(System.currentTimeMillis());
}
@@ -100,20 +102,6 @@ public class TimingValues {
}
/**
- * Returns initial timeout to use as server timeout when a config is requested for the first time.
- *
- * @return timeout in milliseconds.
- */
- public long getInitialTimeout() {
- return initialTimeout;
- }
-
- public TimingValues setInitialTimeout(long t) {
- initialTimeout = t;
- return this;
- }
-
- /**
* Returns timeout to use as server timeout when subscribing for the first time.
*
* @return timeout in milliseconds.
@@ -127,38 +115,12 @@ public class TimingValues {
return this;
}
- /**
- * Returns the time to retry getting config from the remote sources, until the next error response will
- * be set as config. Counted from the last ok request was received. A negative value means that
- * we will always retry getting config and never set an error response as config.
- *
- * @return timeout in milliseconds.
- */
- public long getConfiguredErrorTimeout() {
- return configuredErrorTimeout;
- }
-
public TimingValues setConfiguredErrorTimeout(long t) {
configuredErrorTimeout = t;
return this;
}
/**
- * Returns timeout used when calling {@link com.yahoo.config.subscription.ConfigSubscriber#nextConfig()} or
- * {@link com.yahoo.config.subscription.ConfigSubscriber#nextGeneration()}
- *
- * @return timeout in milliseconds.
- */
- public long getNextConfigTimeout() {
- return nextConfigTimeout;
- }
-
- public TimingValues setNextConfigTimeout(long t) {
- nextConfigTimeout = t;
- return this;
- }
-
- /**
* Returns time to wait until next attempt to get config after a failed request when the client has not
* gotten a successful response to a config subscription (i.e, the client has not been configured).
* A negative value means that there will never be a next attempt. If a negative value is set, the
@@ -201,12 +163,6 @@ public class TimingValues {
return maxDelayMultiplier;
}
-
- public TimingValues setSuccessTimeout(long successTimeout) {
- this.successTimeout = successTimeout;
- return this;
- }
-
/**
* Returns fixed delay that is used when retrying getting config no matter if it was a success or an error
* and independent of number of retries.
@@ -228,10 +184,6 @@ public class TimingValues {
return Math.round(val - (val * fraction) + (rand.nextFloat() * 2L * val * fraction));
}
- Random getRandom() {
- return rand;
- }
-
@Override
public String toString() {
return "TimingValues [successTimeout=" + successTimeout
diff --git a/config/src/test/java/com/yahoo/config/subscription/BasicTest.java b/config/src/test/java/com/yahoo/config/subscription/BasicTest.java
index 3b8b7db6487..5b145d40b7f 100644
--- a/config/src/test/java/com/yahoo/config/subscription/BasicTest.java
+++ b/config/src/test/java/com/yahoo/config/subscription/BasicTest.java
@@ -1,13 +1,13 @@
// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.config.subscription;
-import static org.junit.Assert.*;
-import static org.hamcrest.CoreMatchers.is;
import com.yahoo.foo.AppConfig;
import org.junit.Test;
+import static org.junit.Assert.assertEquals;
+
public class BasicTest {
@@ -17,7 +17,8 @@ public class BasicTest {
ConfigHandle<AppConfig> h = s.subscribe(AppConfig.class, "raw:times 0");
s.nextConfig(0);
AppConfig c = h.getConfig();
- assertThat(c.times(), is(0));
+ assertEquals(0, c.times());
+ s.close();
}
@Test
@@ -26,6 +27,7 @@ public class BasicTest {
ConfigHandle<AppConfig> h = s.subscribe(AppConfig.class, "raw:times 2");
s.nextGeneration(0);
AppConfig c = h.getConfig();
- assertThat(c.times(), is(2));
+ assertEquals(2, c.times());
+ s.close();
}
}
diff --git a/config/src/test/java/com/yahoo/config/subscription/ConfigSetSubscriptionTest.java b/config/src/test/java/com/yahoo/config/subscription/ConfigSetSubscriptionTest.java
index 21cdfbe7d30..db30e7b7389 100644
--- a/config/src/test/java/com/yahoo/config/subscription/ConfigSetSubscriptionTest.java
+++ b/config/src/test/java/com/yahoo/config/subscription/ConfigSetSubscriptionTest.java
@@ -128,6 +128,7 @@ public class ConfigSetSubscriptionTest {
assertEquals(hA0.getConfig().times(), 8800);
assertEquals(hA1.getConfig().times(), 890);
assertEquals(hS.getConfig().stringVal(), "new StringVal");
+ subscriber.close();
}
@Test
diff --git a/config/src/test/java/com/yahoo/config/subscription/ConfigSubscriptionTest.java b/config/src/test/java/com/yahoo/config/subscription/ConfigSubscriptionTest.java
index 933a9fd130a..c8d4c081fc9 100644
--- a/config/src/test/java/com/yahoo/config/subscription/ConfigSubscriptionTest.java
+++ b/config/src/test/java/com/yahoo/config/subscription/ConfigSubscriptionTest.java
@@ -60,6 +60,7 @@ public class ConfigSubscriptionTest {
assertEquals(c1, c1);
assertNotEquals(c1, c2);
+ sub.close();
}
@Test
@@ -70,6 +71,7 @@ public class ConfigSubscriptionTest {
sub.nextConfig();
assertTrue(handle.getConfig().boolval());
//assertTrue(sub.getSource() instanceof RawSource);
+ sub.close();
}
// Test that subscription is closed and subscriptionHandles is empty if we get an exception
diff --git a/config/src/test/java/com/yahoo/config/subscription/GenericConfigSubscriberTest.java b/config/src/test/java/com/yahoo/config/subscription/GenericConfigSubscriberTest.java
index e9dc9cf7b98..9c83f2f3c9a 100644
--- a/config/src/test/java/com/yahoo/config/subscription/GenericConfigSubscriberTest.java
+++ b/config/src/test/java/com/yahoo/config/subscription/GenericConfigSubscriberTest.java
@@ -49,14 +49,17 @@ public class GenericConfigSubscriberTest {
public void testGenericRequesterPooling() {
ConfigSourceSet source1 = new ConfigSourceSet("tcp/foo:78");
ConfigSourceSet source2 = new ConfigSourceSet("tcp/bar:79");
- JRTConfigRequester req1 = new JRTConfigRequester(new JRTConnectionPool(source1), JRTConfigRequesterTest.getTestTimingValues());
- JRTConfigRequester req2 = new JRTConfigRequester(new JRTConnectionPool(source2), JRTConfigRequesterTest.getTestTimingValues());
+ JRTConfigRequester req1 = JRTConfigRequester.create(source1, JRTConfigRequesterTest.getTestTimingValues());
+ JRTConfigRequester req2 = JRTConfigRequester.create(source2, JRTConfigRequesterTest.getTestTimingValues());
Map<ConfigSourceSet, JRTConfigRequester> requesters = new LinkedHashMap<>();
requesters.put(source1, req1);
requesters.put(source2, req2);
GenericConfigSubscriber sub = new GenericConfigSubscriber(requesters);
assertEquals(sub.requesters().get(source1).getConnectionPool().getCurrent().getAddress(), "tcp/foo:78");
assertEquals(sub.requesters().get(source2).getConnectionPool().getCurrent().getAddress(), "tcp/bar:79");
+ for (JRTConfigRequester requester : requesters.values()) {
+ requester.close();
+ }
}
@Test(expected=UnsupportedOperationException.class)
diff --git a/config/src/test/java/com/yahoo/config/subscription/impl/JRTConfigRequesterTest.java b/config/src/test/java/com/yahoo/config/subscription/impl/JRTConfigRequesterTest.java
index 757dd99f43b..4211345dff7 100644
--- a/config/src/test/java/com/yahoo/config/subscription/impl/JRTConfigRequesterTest.java
+++ b/config/src/test/java/com/yahoo/config/subscription/impl/JRTConfigRequesterTest.java
@@ -1,10 +1,12 @@
// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.config.subscription.impl;
+import com.yahoo.config.subscription.ConfigSourceSet;
import com.yahoo.foo.SimpletypesConfig;
import com.yahoo.config.subscription.ConfigSubscriber;
import com.yahoo.jrt.Request;
import com.yahoo.vespa.config.ConfigKey;
+import com.yahoo.vespa.config.ConnectionPool;
import com.yahoo.vespa.config.ErrorCode;
import com.yahoo.vespa.config.ErrorType;
import com.yahoo.vespa.config.TimingValues;
@@ -17,6 +19,8 @@ import java.util.Random;
import static org.hamcrest.CoreMatchers.is;
import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNotSame;
+import static org.junit.Assert.assertSame;
import static org.junit.Assert.assertThat;
import static org.junit.Assert.assertTrue;
@@ -349,4 +353,23 @@ public class JRTConfigRequesterTest {
}
}
+ @Test
+ public void testManagedPool() {
+ ConfigSourceSet sourceSet = ConfigSourceSet.createDefault();
+ TimingValues timingValues = new TimingValues();
+ JRTConfigRequester requester1 = JRTConfigRequester.create(sourceSet, timingValues);
+ JRTConfigRequester requester2 = JRTConfigRequester.create(sourceSet, timingValues);
+ assertNotSame(requester1, requester2);
+ assertSame(requester1.getConnectionPool(), requester2.getConnectionPool());
+ ConnectionPool firstPool = requester1.getConnectionPool();
+ requester1.close();
+ requester2.close();
+ requester1 = JRTConfigRequester.create(sourceSet, timingValues);
+ assertNotSame(firstPool, requester1.getConnectionPool());
+ requester2 = JRTConfigRequester.create(new ConfigSourceSet("test-managed-pool-2"), timingValues);
+ assertNotSame(requester1.getConnectionPool(), requester2.getConnectionPool());
+ requester1.close();
+ requester2.close();
+ }
+
}
diff --git a/config/src/tests/trace/trace.cpp b/config/src/tests/trace/trace.cpp
index 41c874eb1d4..9a355f39ecc 100644
--- a/config/src/tests/trace/trace.cpp
+++ b/config/src/tests/trace/trace.cpp
@@ -2,6 +2,7 @@
#include <vespa/vespalib/testkit/test_kit.h>
#include <vespa/config/common/trace.h>
+#include <vespa/vespalib/data/slime/slime.h>
using namespace config;
diff --git a/config/src/vespa/config/common/configdefinition.cpp b/config/src/vespa/config/common/configdefinition.cpp
index 861fc224867..92af068cff5 100644
--- a/config/src/vespa/config/common/configdefinition.cpp
+++ b/config/src/vespa/config/common/configdefinition.cpp
@@ -1,6 +1,7 @@
// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
#include "configdefinition.h"
#include <vespa/vespalib/stllike/asciistream.h>
+#include <vespa/vespalib/data/slime/slime.h>
using namespace vespalib;
using namespace vespalib::slime;
diff --git a/config/src/vespa/config/common/configdefinition.h b/config/src/vespa/config/common/configdefinition.h
index 3d86e5d2567..9154f3ad88a 100644
--- a/config/src/vespa/config/common/configdefinition.h
+++ b/config/src/vespa/config/common/configdefinition.h
@@ -3,8 +3,11 @@
#include <vespa/vespalib/stllike/string.h>
#include <vector>
-#include <vespa/vespalib/data/slime/slime.h>
+namespace vespalib::slime {
+ class Cursor;
+ class Inspector;
+}
namespace config {
/**
diff --git a/config/src/vespa/config/common/trace.cpp b/config/src/vespa/config/common/trace.cpp
index d1bb154eda9..76310d08c7d 100644
--- a/config/src/vespa/config/common/trace.cpp
+++ b/config/src/vespa/config/common/trace.cpp
@@ -2,6 +2,7 @@
#include "trace.h"
#include <vespa/vespalib/trace/slime_trace_serializer.h>
#include <vespa/vespalib/trace/slime_trace_deserializer.h>
+#include <vespa/vespalib/data/slime/slime.h>
using namespace vespalib;
using namespace vespalib::slime;
diff --git a/config/src/vespa/config/common/trace.h b/config/src/vespa/config/common/trace.h
index 772cdb6f31e..c120fe30d12 100644
--- a/config/src/vespa/config/common/trace.h
+++ b/config/src/vespa/config/common/trace.h
@@ -1,11 +1,14 @@
// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
#pragma once
-#include <vespa/vespalib/data/slime/slime.h>
#include <vespa/vespalib/trace/tracenode.h>
#include <vespa/vespalib/stllike/string.h>
-#include <memory>
+#include <vespa/vespalib/data/memory.h>
+namespace vespalib::slime {
+ class Cursor;
+ class Inspector;
+}
namespace config {
/**
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerBootstrap.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerBootstrap.java
index b2a10f4bb21..4e1006213c6 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerBootstrap.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerBootstrap.java
@@ -62,6 +62,7 @@ public class ConfigServerBootstrap extends AbstractComponent implements Runnable
private final StateMonitor stateMonitor;
private final VipStatus vipStatus;
private final ConfigserverConfig configserverConfig;
+ private final SuperModelManager superModelManager;
private final Duration maxDurationOfRedeployment;
private final Duration sleepTimeWhenRedeployingFails;
private final RedeployingApplicationsFails exitIfRedeployingApplicationsFails;
@@ -70,29 +71,32 @@ public class ConfigServerBootstrap extends AbstractComponent implements Runnable
@SuppressWarnings("unused")
@Inject
public ConfigServerBootstrap(ApplicationRepository applicationRepository, RpcServer server,
- VersionState versionState, StateMonitor stateMonitor, VipStatus vipStatus) {
+ VersionState versionState, StateMonitor stateMonitor, VipStatus vipStatus,
+ SuperModelManager superModelManager) {
this(applicationRepository, server, versionState, stateMonitor, vipStatus, BOOTSTRAP_IN_CONSTRUCTOR, EXIT_JVM,
applicationRepository.configserverConfig().hostedVespa()
? VipStatusMode.VIP_STATUS_FILE
- : VipStatusMode.VIP_STATUS_PROGRAMMATICALLY);
+ : VipStatusMode.VIP_STATUS_PROGRAMMATICALLY,
+ superModelManager);
}
// For testing only
ConfigServerBootstrap(ApplicationRepository applicationRepository, RpcServer server, VersionState versionState,
StateMonitor stateMonitor, VipStatus vipStatus, Mode mode, VipStatusMode vipStatusMode) {
- this(applicationRepository, server, versionState, stateMonitor, vipStatus, mode, CONTINUE, vipStatusMode);
+ this(applicationRepository, server, versionState, stateMonitor, vipStatus, mode, CONTINUE, vipStatusMode, null);
}
private ConfigServerBootstrap(ApplicationRepository applicationRepository, RpcServer server,
VersionState versionState, StateMonitor stateMonitor, VipStatus vipStatus,
Mode mode, RedeployingApplicationsFails exitIfRedeployingApplicationsFails,
- VipStatusMode vipStatusMode) {
+ VipStatusMode vipStatusMode, SuperModelManager superModelManager) {
this.applicationRepository = applicationRepository;
this.server = server;
this.versionState = versionState;
this.stateMonitor = stateMonitor;
this.vipStatus = vipStatus;
this.configserverConfig = applicationRepository.configserverConfig();
+ this.superModelManager = superModelManager;
this.maxDurationOfRedeployment = Duration.ofSeconds(configserverConfig.maxDurationOfBootstrap());
this.sleepTimeWhenRedeployingFails = Duration.ofSeconds(configserverConfig.sleepTimeWhenRedeployingFails());
this.exitIfRedeployingApplicationsFails = exitIfRedeployingApplicationsFails;
@@ -208,6 +212,9 @@ public class ConfigServerBootstrap extends AbstractComponent implements Runnable
private boolean redeployAllApplications() throws InterruptedException {
Instant end = Instant.now().plus(maxDurationOfRedeployment);
Set<ApplicationId> applicationsNotRedeployed = applicationRepository.listApplications();
+ if (superModelManager != null) {
+ superModelManager.setBootstrapApplicationSet(applicationsNotRedeployed);
+ }
do {
applicationsNotRedeployed = redeployApplications(applicationsNotRedeployed);
if ( ! applicationsNotRedeployed.isEmpty()) {
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelManager.java b/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelManager.java
index ea835206b7c..ccd92d4a195 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelManager.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelManager.java
@@ -20,6 +20,7 @@ import java.time.Instant;
import java.util.ArrayList;
import java.util.List;
import java.util.Optional;
+import java.util.Set;
/**
* Provides a SuperModel - a model of all application instances, and makes it stays
@@ -38,6 +39,9 @@ public class SuperModelManager implements SuperModelProvider {
private final long masterGeneration; // ConfigserverConfig's generation
private final GenerationCounter generationCounter;
+ // The initial set of applications to be deployed on bootstrap.
+ private Optional<Set<ApplicationId>> bootstrapApplicationSet = Optional.empty();
+
@Inject
public SuperModelManager(ConfigserverConfig configserverConfig,
NodeFlavors nodeFlavors,
@@ -75,6 +79,10 @@ public class SuperModelManager implements SuperModelProvider {
listeners.add(listener);
SuperModel superModel = superModelConfigProvider.getSuperModel();
superModel.getAllApplicationInfos().forEach(application -> listener.applicationActivated(superModel, application));
+
+ if (superModel.isComplete()) {
+ listener.notifyOfCompleteness(superModel);
+ }
}
}
@@ -89,13 +97,19 @@ public class SuperModelManager implements SuperModelProvider {
.getForVersionOrLatest(Optional.empty(), Instant.now())
.toApplicationInfo();
- SuperModel newSuperModel = this.superModelConfigProvider
- .getSuperModel()
- .cloneAndSetApplication(applicationInfo);
+ SuperModel oldSuperModel = superModelConfigProvider.getSuperModel();
+ SuperModel newSuperModel = oldSuperModel
+ .cloneAndSetApplication(applicationInfo, isComplete(oldSuperModel));
+
generationCounter.increment();
makeNewSuperModelConfigProvider(newSuperModel);
- listeners.stream().forEach(listener ->
- listener.applicationActivated(newSuperModel, applicationInfo));
+ listeners.forEach(listener -> listener.applicationActivated(newSuperModel, applicationInfo));
+
+ if (!oldSuperModel.isComplete() && newSuperModel.isComplete()) {
+ for (var listener : listeners) {
+ listener.notifyOfCompleteness(newSuperModel);
+ }
+ }
}
}
@@ -106,11 +120,36 @@ public class SuperModelManager implements SuperModelProvider {
.cloneAndRemoveApplication(applicationId);
generationCounter.increment();
makeNewSuperModelConfigProvider(newSuperModel);
- listeners.stream().forEach(listener ->
- listener.applicationRemoved(newSuperModel, applicationId));
+ listeners.forEach(listener -> listener.applicationRemoved(newSuperModel, applicationId));
}
}
+ public void setBootstrapApplicationSet(Set<ApplicationId> bootstrapApplicationSet) {
+ synchronized (monitor) {
+ this.bootstrapApplicationSet = Optional.of(bootstrapApplicationSet);
+
+ SuperModel superModel = superModelConfigProvider.getSuperModel();
+ if (!superModel.isComplete() && isComplete(superModel)) {
+ // We do NOT increment the generation since completeness is not part of the config:
+ // generationCounter.increment()
+
+ SuperModel newSuperModel = superModel.cloneAsComplete();
+ makeNewSuperModelConfigProvider(newSuperModel);
+ listeners.forEach(listener -> listener.notifyOfCompleteness(newSuperModel));
+ }
+ }
+ }
+
+ /** Returns freshly calculated value of isComplete. */
+ private boolean isComplete(SuperModel currentSuperModel) {
+ if (currentSuperModel.isComplete()) return true;
+ if (bootstrapApplicationSet.isEmpty()) return false;
+
+ Set<ApplicationId> currentApplicationIds = superModelConfigProvider.getSuperModel().getApplicationIds();
+ if (currentApplicationIds.size() < bootstrapApplicationSet.get().size()) return false;
+ return currentApplicationIds.containsAll(bootstrapApplicationSet.get());
+ }
+
private void makeNewSuperModelConfigProvider(SuperModel newSuperModel) {
generation = masterGeneration + generationCounter.get();
superModelConfigProvider = new SuperModelConfigProvider(newSuperModel, zone, flagSource);
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/SuperModelControllerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/SuperModelControllerTest.java
index b7486dc7951..561422c1cf8 100644
--- a/configserver/src/test/java/com/yahoo/vespa/config/server/SuperModelControllerTest.java
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/SuperModelControllerTest.java
@@ -55,7 +55,7 @@ public class SuperModelControllerTest {
ApplicationId app = ApplicationId.from(TenantName.from("a"),
ApplicationName.from("foo"), InstanceName.defaultName());
models.put(app, new ApplicationInfo(app, 4L, new VespaModel(FilesApplicationPackage.fromFile(testApp))));
- SuperModel superModel = new SuperModel(models);
+ SuperModel superModel = new SuperModel(models, true);
handler = new SuperModelController(new SuperModelConfigProvider(superModel, Zone.defaultZone(), new InMemoryFlagSource()), new TestConfigDefinitionRepo(), 2, new UncompressedConfigResponseFactory());
}
@@ -98,7 +98,7 @@ public class SuperModelControllerTest {
models.put(advanced, createApplicationInfo(testApp2, advanced, 4L));
models.put(tooAdvanced, createApplicationInfo(testApp3, tooAdvanced, 4L));
- SuperModel superModel = new SuperModel(models);
+ SuperModel superModel = new SuperModel(models, true);
SuperModelController han = new SuperModelController(new SuperModelConfigProvider(superModel, Zone.defaultZone(), new InMemoryFlagSource()), new TestConfigDefinitionRepo(), 2, new UncompressedConfigResponseFactory());
LbServicesConfig.Builder lb = new LbServicesConfig.Builder();
han.getSuperModel().getConfig(lb);
@@ -126,7 +126,7 @@ public class SuperModelControllerTest {
models.put(advanced, createApplicationInfo(testApp2, advanced, 4L));
models.put(tooAdvanced, createApplicationInfo(testApp3, tooAdvanced, 4L));
- SuperModel superModel = new SuperModel(models);
+ SuperModel superModel = new SuperModel(models, true);
SuperModelController han = new SuperModelController(new SuperModelConfigProvider(superModel, Zone.defaultZone(), new InMemoryFlagSource()), new TestConfigDefinitionRepo(), 2, new UncompressedConfigResponseFactory());
LbServicesConfig.Builder lb = new LbServicesConfig.Builder();
han.getSuperModel().getConfig(lb);
diff --git a/container-accesslogging/pom.xml b/container-accesslogging/pom.xml
index 53c17257992..59dabba9efe 100644
--- a/container-accesslogging/pom.xml
+++ b/container-accesslogging/pom.xml
@@ -65,6 +65,11 @@
<artifactId>jackson-databind</artifactId>
<scope>provided</scope>
</dependency>
+ <dependency>
+ <groupId>org.assertj</groupId>
+ <artifactId>assertj-core</artifactId>
+ <scope>test</scope>
+ </dependency>
</dependencies>
<build>
<plugins>
@@ -73,6 +78,10 @@
<artifactId>bundle-plugin</artifactId>
<extensions>true</extensions>
</plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-compiler-plugin</artifactId>
+ </plugin>
</plugins>
<outputDirectory>${buildOutputDirectory}</outputDirectory>
</build>
diff --git a/container-accesslogging/src/main/java/com/yahoo/container/logging/LogFileHandler.java b/container-accesslogging/src/main/java/com/yahoo/container/logging/LogFileHandler.java
index 82c89276319..a3d34ae6a2c 100644
--- a/container-accesslogging/src/main/java/com/yahoo/container/logging/LogFileHandler.java
+++ b/container-accesslogging/src/main/java/com/yahoo/container/logging/LogFileHandler.java
@@ -11,7 +11,6 @@ import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
-import java.io.OutputStream;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.concurrent.ArrayBlockingQueue;
@@ -41,7 +40,7 @@ public class LogFileHandler extends StreamHandler {
private String filePattern = "./log.%T"; // default to current directory, ms time stamp
private long nextRotationTime = 0;
private FileOutputStream currentOutputStream = null;
- private String fileName;
+ private volatile String fileName;
private String symlinkName = null;
private ArrayBlockingQueue<LogRecord> logQueue = new ArrayBlockingQueue<>(100000);
private LogRecord rotateCmd = new LogRecord(Level.SEVERE, "rotateNow");
diff --git a/container-accesslogging/src/test/java/com/yahoo/container/logging/LogFileHandlerTestCase.java b/container-accesslogging/src/test/java/com/yahoo/container/logging/LogFileHandlerTestCase.java
index 999f430aa7a..d7361eec488 100644
--- a/container-accesslogging/src/test/java/com/yahoo/container/logging/LogFileHandlerTestCase.java
+++ b/container-accesslogging/src/test/java/com/yahoo/container/logging/LogFileHandlerTestCase.java
@@ -2,7 +2,9 @@
package com.yahoo.container.logging;
import com.yahoo.io.IOUtils;
+import org.junit.Rule;
import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
import java.io.File;
import java.io.FileInputStream;
@@ -17,21 +19,23 @@ import java.util.logging.LogRecord;
import java.util.logging.SimpleFormatter;
import java.util.zip.GZIPInputStream;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
+import static org.assertj.core.api.Assertions.assertThat;
/**
* @author Bob Travis
+ * @author bjorncs
*/
-// TODO: Make these tests wait until the right things happen rather than waiting for a predetermined time
-// These tests take too long, and are not cleaning up properly. See how this should be done in YApacheLogTestCase
public class LogFileHandlerTestCase {
+ @Rule
+ public TemporaryFolder temporaryFolder = new TemporaryFolder();
+
@Test
- public void testIt() {
+ public void testIt() throws IOException {
+ File root = temporaryFolder.newFolder("logfilehandlertest");
+
LogFileHandler h = new LogFileHandler();
- h.setFilePattern("./logfilehandlertest.%Y%m%d%H%M%S");
+ h.setFilePattern(root.getAbsolutePath() + "/logfilehandlertest.%Y%m%d%H%M%S");
h.setFormatter(new Formatter() {
public String format(LogRecord r) {
DateFormat df = new SimpleDateFormat("yyyy.MM.dd:HH:mm:ss.SSS");
@@ -43,42 +47,27 @@ public class LogFileHandlerTestCase {
long millisPerDay = 60*60*24*1000;
long tomorrowDays = (now / millisPerDay) +1;
long tomorrowMillis = tomorrowDays * millisPerDay;
- assertEquals (tomorrowMillis, h.getNextRotationTime(now));
+ assertThat(tomorrowMillis).isEqualTo(h.getNextRotationTime(now));
long[] rTimes = {1000, 2000, 10000};
h.setRotationTimes(rTimes);
- assertEquals (tomorrowMillis+1000, h.getNextRotationTime(tomorrowMillis));
- assertEquals (tomorrowMillis+10000, h.getNextRotationTime(tomorrowMillis+3000));
- boolean okToWrite = false; // don't want regular unit tests to create tiles....
- if (okToWrite) {
- LogRecord lr = new LogRecord(Level.INFO, "test");
- h.publish(lr);
- h.publish(new LogRecord(Level.INFO, "another test"));
- h.rotateNow();
- h.publish(lr);
- h.flush();
- }
- }
-
- private boolean delete(String fileOrDir) {
- File file = new File(fileOrDir);
- return file.delete();
- }
-
- private void deleteOnExit(String fileOrDir) {
- new File(fileOrDir).deleteOnExit();
- }
-
- private static void deleteRecursive(String directory) {
- IOUtils.recursiveDeleteDir(new File(directory));
+ assertThat(tomorrowMillis+1000).isEqualTo(h.getNextRotationTime(tomorrowMillis));
+ assertThat(tomorrowMillis+10000).isEqualTo(h.getNextRotationTime(tomorrowMillis+3000));
+ LogRecord lr = new LogRecord(Level.INFO, "test");
+ h.publish(lr);
+ h.publish(new LogRecord(Level.INFO, "another test"));
+ h.rotateNow();
+ h.publish(lr);
+ h.flush();
+ h.shutdown();
}
@Test
- public void testSimpleLogging() {
- String logFilePattern = "./testLogFileG1.txt";
+ public void testSimpleLogging() throws IOException {
+ File logFile = temporaryFolder.newFile("testLogFileG1.txt");
//create logfilehandler
LogFileHandler h = new LogFileHandler();
- h.setFilePattern(logFilePattern);
+ h.setFilePattern(logFile.getAbsolutePath());
h.setFormatter(new SimpleFormatter());
h.setRotationTimes("0 5 ...");
@@ -86,17 +75,16 @@ public class LogFileHandlerTestCase {
LogRecord lr = new LogRecord(Level.INFO, "testDeleteFileFirst1");
h.publish(lr);
h.flush();
-
- new File(logFilePattern).deleteOnExit();
+ h.shutdown();
}
@Test
- public void testDeleteFileDuringLogging() {
- String logFilePattern = "./testLogFileG2.txt";
+ public void testDeleteFileDuringLogging() throws IOException {
+ File logFile = temporaryFolder.newFile("testLogFileG2.txt");
//create logfilehandler
LogFileHandler h = new LogFileHandler();
- h.setFilePattern(logFilePattern);
+ h.setFilePattern(logFile.getAbsolutePath());
h.setFormatter(new SimpleFormatter());
h.setRotationTimes("0 5 ...");
@@ -106,20 +94,20 @@ public class LogFileHandlerTestCase {
h.flush();
//delete log file
- delete(logFilePattern);
+ logFile.delete();
//write log again
lr = new LogRecord(Level.INFO, "testDeleteFileDuringLogging2");
h.publish(lr);
h.flush();
-
- new File(logFilePattern).deleteOnExit();
+ h.shutdown();
}
@Test
- public void testSymlink() {
+ public void testSymlink() throws IOException, InterruptedException {
+ File root = temporaryFolder.newFolder("testlogforsymlinkchecking");
LogFileHandler h = new LogFileHandler();
- h.setFilePattern("./testlogforsymlinkchecking/logfilehandlertest.%Y%m%d%H%M%S%s");
+ h.setFilePattern(root.getAbsolutePath() + "/logfilehandlertest.%Y%m%d%H%M%S%s");
h.setFormatter(new Formatter() {
public String format(LogRecord r) {
DateFormat df = new SimpleDateFormat("yyyy.MM.dd:HH:mm:ss.SSS");
@@ -132,50 +120,43 @@ public class LogFileHandlerTestCase {
h.publish(lr);
String f1 = h.getFileName();
String f2 = null;
- try {
- while (f1 == null) {
- Thread.sleep(1);
- f1 = h.getFileName();
- }
- h.rotateNow();
+ while (f1 == null) {
+ Thread.sleep(1);
+ f1 = h.getFileName();
+ }
+ h.rotateNow();
+ Thread.sleep(1);
+ f2 = h.getFileName();
+ while (f1.equals(f2)) {
Thread.sleep(1);
f2 = h.getFileName();
- while (f1.equals(f2)) {
- Thread.sleep(1);
- f2 = h.getFileName();
- }
- lr = new LogRecord(Level.INFO, "string which is way longer than the word test");
- h.publish(lr);
- Thread.sleep(1000);
- File f = new File(f1);
- long first = f.length();
- f = new File(f2);
- long second = f.length();
- final long secondLength = 72;
- for (int n = 0; n < 20 && second != secondLength; ++n) {
- Thread.sleep(1000);
- second = f.length();
- }
- f = new File("./testlogforsymlinkchecking", "symlink");
- long link = f.length();
- assertEquals(secondLength, link);
- assertEquals(31, first);
- assertEquals(secondLength, second);
- } catch (InterruptedException e) {
- // just let the test pass
}
- deleteOnExit("./testlogforsymlinkchecking");
- deleteOnExit("./testlogforsymlinkchecking/symlink");
- deleteOnExit(f1);
- if (f2 != null)
- deleteOnExit(f2);
+ lr = new LogRecord(Level.INFO, "string which is way longer than the word test");
+ h.publish(lr);
+ h.waitDrained();
+ File f = new File(f1);
+ long first = f.length();
+ f = new File(f2);
+ long second = f.length();
+ final long secondLength = 72;
+ for (int n = 0; n < 20 && second != secondLength; ++n) {
+ Thread.sleep(1);
+ second = f.length();
+ }
+ f = new File(root, "symlink");
+ long link = f.length();
+ assertThat(secondLength).isEqualTo(link);
+ assertThat(31).isEqualTo(first);
+ assertThat(secondLength).isEqualTo(second);
+ h.shutdown();
}
@Test
public void testcompression() throws InterruptedException, IOException {
- IOUtils.recursiveDeleteDir(new File("./testcompression"));
+ File root = temporaryFolder.newFolder("testcompression");
+
LogFileHandler h = new LogFileHandler(true);
- h.setFilePattern("./testcompression/logfilehandlertest.%Y%m%d%H%M%S%s");
+ h.setFilePattern(root.getAbsolutePath() + "/logfilehandlertest.%Y%m%d%H%M%S%s");
h.setFormatter(new Formatter() {
public String format(LogRecord r) {
DateFormat df = new SimpleDateFormat("yyyy.MM.dd:HH:mm:ss.SSS");
@@ -183,28 +164,28 @@ public class LogFileHandlerTestCase {
return ("["+timeStamp+"]" + " " + formatMessage(r) + "\n");
}
} );
- for (int i=0; i < 10000; i++) {
+ int logEntries = 10000;
+ for (int i = 0; i < logEntries; i++) {
LogRecord lr = new LogRecord(Level.INFO, "test");
h.publish(lr);
}
h.waitDrained();
String f1 = h.getFileName();
- assertTrue(f1.startsWith("./testcompression/logfilehandlertest."));
+ assertThat(f1).startsWith(root.getAbsolutePath() + "/logfilehandlertest.");
File uncompressed = new File(f1);
File compressed = new File(f1 + ".gz");
- assertTrue(uncompressed.exists());
- assertFalse(compressed.exists());
+ assertThat(uncompressed).exists();
+ assertThat(compressed).doesNotExist();
String content = IOUtils.readFile(uncompressed);
- assertEquals(310000, content.length());
+ assertThat(content).hasLineCount(logEntries);
h.rotateNow();
while (uncompressed.exists()) {
- Thread.sleep(10);
+ Thread.sleep(1);
}
- assertTrue(compressed.exists());
+ assertThat(compressed).exists();
String unzipped = IOUtils.readAll(new InputStreamReader(new GZIPInputStream(new FileInputStream(compressed))));
- assertEquals(content, unzipped);
-
- IOUtils.recursiveDeleteDir(new File("./testcompression"));
+ assertThat(content).isEqualTo(unzipped);
+ h.shutdown();
}
}
diff --git a/container-core/src/main/java/com/yahoo/container/jdisc/ThreadedRequestHandler.java b/container-core/src/main/java/com/yahoo/container/jdisc/ThreadedRequestHandler.java
index bfcecd61fa4..6c8cee8433c 100644
--- a/container-core/src/main/java/com/yahoo/container/jdisc/ThreadedRequestHandler.java
+++ b/container-core/src/main/java/com/yahoo/container/jdisc/ThreadedRequestHandler.java
@@ -17,6 +17,7 @@ import com.yahoo.jdisc.handler.ResponseDispatch;
import com.yahoo.jdisc.handler.ResponseHandler;
import com.yahoo.log.LogLevel;
+import java.net.URI;
import java.time.Duration;
import java.util.HashMap;
import java.util.Map;
@@ -91,6 +92,11 @@ public abstract class ThreadedRequestHandler extends AbstractRequestHandler {
if (endpoint != null) {
dimensions.put("endpoint", endpoint);
}
+ URI uri = request.getUri();
+ dimensions.put("scheme", uri.getScheme());
+ dimensions.put("port", Integer.toString(uri.getPort()));
+ String handlerClassName = getClass().getName();
+ dimensions.put("handler-name", handlerClassName);
return this.metric.createContext(dimensions);
}
diff --git a/container-disc/src/main/sh/vespa-start-container-daemon.sh b/container-disc/src/main/sh/vespa-start-container-daemon.sh
index 382843b5688..af429d56a75 100755
--- a/container-disc/src/main/sh/vespa-start-container-daemon.sh
+++ b/container-disc/src/main/sh/vespa-start-container-daemon.sh
@@ -64,6 +64,7 @@ configure_memory() {
consider_fallback jvm_heapsize 1536
consider_fallback jvm_stacksize 512
consider_fallback jvm_baseMaxDirectMemorySize 75
+ consider_fallback jvm_compressedClassSpaceSize 32
consider_fallback jvm_directMemorySizeCache 0
# Update jvm_heapsize only if percentage is explicitly set (default is 0).
@@ -80,16 +81,20 @@ configure_memory() {
fi
# Safety measure against bad min vs max heapsize.
- if ((jvm_minHeapsize > jvm_heapsize)); then
+ if ((jvm_minHeapsize > jvm_heapsize)); then
jvm_minHeapsize=${jvm_heapsize}
echo "Misconfigured heap size, jvm_minHeapsize(${jvm_minHeapsize} is larger than jvm_heapsize(${jvm_heapsize}). It has been capped."
- fi
+ fi
maxDirectMemorySize=$(( jvm_baseMaxDirectMemorySize + jvm_heapsize / 8 + jvm_directMemorySizeCache ))
memory_options="-Xms${jvm_minHeapsize}m -Xmx${jvm_heapsize}m"
memory_options="${memory_options} -XX:ThreadStackSize=${jvm_stacksize}"
- memory_options="${memory_options} -XX:MaxDirectMemorySize=${maxDirectMemorySize}m"
+ memory_options="${memory_options} -XX:MaxDirectMemorySize=${maxDirectMemorySize}m"
+
+ if ((jvm_compressedClassSpaceSize != 0)); then
+ memory_options="${memory_options} -XX:CompressedClassSpaceSize=${jvm_compressedClassSpaceSize}m"
+ fi
if [ "${VESPA_USE_HUGEPAGES}" ]; then
memory_options="${memory_options} -XX:+UseLargePages"
diff --git a/container-search/src/main/resources/configdefinitions/qr-start.def b/container-search/src/main/resources/configdefinitions/qr-start.def
index 031877ada81..95e9d4575dd 100644
--- a/container-search/src/main/resources/configdefinitions/qr-start.def
+++ b/container-search/src/main/resources/configdefinitions/qr-start.def
@@ -20,6 +20,9 @@ jvm.minHeapsize int default=1536 restart
## Stack size (in kilobytes)
jvm.stacksize int default=512 restart
+## CompressedOOps size in megabytes
+jvm.compressedClassSpaceSize int default=32 restart
+
## Base value of maximum direct memory size (in megabytes)
jvm.baseMaxDirectMemorySize int default=75 restart
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/EndpointCertificateMetadata.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/EndpointCertificateMetadata.java
index 0aa0df8ae2b..171c5caa756 100644
--- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/EndpointCertificateMetadata.java
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/EndpointCertificateMetadata.java
@@ -18,25 +18,23 @@ public class EndpointCertificateMetadata {
private final int version;
private final Optional<String> request_id;
private final Optional<List<String>> requestedDnsSans;
+ private final Optional<String> issuer;
public EndpointCertificateMetadata(String keyName, String certName, int version) {
- this.keyName = keyName;
- this.certName = certName;
- this.version = version;
- this.request_id = Optional.empty();
- this.requestedDnsSans = Optional.empty();
+ this(keyName, certName, version, Optional.empty(), Optional.empty(), Optional.empty());
+ }
+
+ public EndpointCertificateMetadata(String keyName, String certName, int version, String request_id, List<String> requestedDnsSans) {
+ this(keyName, certName, version, Optional.of(request_id), Optional.of(requestedDnsSans), Optional.empty());
}
- public EndpointCertificateMetadata(String keyName, String certName, int version, Optional<String> request_id, Optional<List<String>> requestedDnsSans) {
+ public EndpointCertificateMetadata(String keyName, String certName, int version, Optional<String> request_id, Optional<List<String>> requestedDnsSans, Optional<String> issuer) {
this.keyName = keyName;
this.certName = certName;
this.version = version;
this.request_id = request_id;
this.requestedDnsSans = requestedDnsSans;
- }
-
- public EndpointCertificateMetadata(String keyName, String certName, int version, String request_id, List<String> requestedDnsSans) {
- this(keyName, certName, version, Optional.of(request_id), Optional.of(requestedDnsSans));
+ this.issuer = issuer;
}
public String keyName() {
@@ -59,6 +57,10 @@ public class EndpointCertificateMetadata {
return requestedDnsSans;
}
+ public Optional<String> issuer() {
+ return issuer;
+ }
+
@Override
public String toString() {
return "EndpointCertificateMetadata{" +
@@ -67,6 +69,7 @@ public class EndpointCertificateMetadata {
", version=" + version +
", request_id=" + request_id +
", requestedDnsSans=" + requestedDnsSans +
+ ", issuer=" + issuer +
'}';
}
@@ -79,11 +82,12 @@ public class EndpointCertificateMetadata {
keyName.equals(that.keyName) &&
certName.equals(that.certName) &&
request_id.equals(that.request_id) &&
- requestedDnsSans.equals(that.requestedDnsSans);
+ requestedDnsSans.equals(that.requestedDnsSans) &&
+ issuer.equals(that.issuer);
}
@Override
public int hashCode() {
- return Objects.hash(keyName, certName, version, request_id, requestedDnsSans);
+ return Objects.hash(keyName, certName, version, request_id, requestedDnsSans, issuer);
}
}
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/EndpointCertificateMock.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/EndpointCertificateMock.java
index 8e81400f3c8..c38ea158507 100644
--- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/EndpointCertificateMock.java
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/EndpointCertificateMock.java
@@ -7,6 +7,7 @@ import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.Optional;
import java.util.UUID;
/**
@@ -21,7 +22,7 @@ public class EndpointCertificateMock implements EndpointCertificateProvider {
}
@Override
- public EndpointCertificateMetadata requestCaSignedCertificate(ApplicationId applicationId, List<String> dnsNames) {
+ public EndpointCertificateMetadata requestCaSignedCertificate(ApplicationId applicationId, List<String> dnsNames, Optional<EndpointCertificateMetadata> currentMetadata) {
this.dnsNames.put(applicationId, dnsNames);
String endpointCertificatePrefix = String.format("vespa.tls.%s.%s@%s", applicationId.tenant(),
applicationId.application(),
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/EndpointCertificateProvider.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/EndpointCertificateProvider.java
index 97d2bdb3343..9c5c25c1c71 100644
--- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/EndpointCertificateProvider.java
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/EndpointCertificateProvider.java
@@ -4,6 +4,7 @@ package com.yahoo.vespa.hosted.controller.api.integration.certificates;
import com.yahoo.config.provision.ApplicationId;
import java.util.List;
+import java.util.Optional;
/**
* Generates an endpoint certificate for an application instance.
@@ -12,7 +13,7 @@ import java.util.List;
*/
public interface EndpointCertificateProvider {
- EndpointCertificateMetadata requestCaSignedCertificate(ApplicationId applicationId, List<String> dnsNames);
+ EndpointCertificateMetadata requestCaSignedCertificate(ApplicationId applicationId, List<String> dnsNames, Optional<EndpointCertificateMetadata> currentMetadata);
List<EndpointCertificateMetadata> listCertificates();
}
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/PathGroup.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/PathGroup.java
index 67a6faac606..4c91238453a 100644
--- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/PathGroup.java
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/PathGroup.java
@@ -179,8 +179,7 @@ enum PathGroup {
"/application/v4/tenant/"),
/** Paths which contain (not very strictly) classified information about, e.g., customers. */
- classifiedInfo("/cost/v1/{*}",
- "/",
+ classifiedInfo("/",
"/d/{*}"),
/** Same as classifiedInfo, but with optional /api prefix */
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java
index 17c9e852bd9..e3d41873a8e 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java
@@ -49,7 +49,7 @@ import com.yahoo.vespa.hosted.controller.concurrent.Once;
import com.yahoo.vespa.hosted.controller.deployment.DeploymentTrigger;
import com.yahoo.vespa.hosted.controller.deployment.Run;
import com.yahoo.vespa.hosted.controller.deployment.Versions;
-import com.yahoo.vespa.hosted.controller.endpointcertificates.EndpointCertificateManager;
+import com.yahoo.vespa.hosted.controller.certificate.EndpointCertificateManager;
import com.yahoo.vespa.hosted.controller.persistence.CuratorDb;
import com.yahoo.vespa.hosted.controller.security.AccessControl;
import com.yahoo.vespa.hosted.controller.security.Credentials;
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java
index 74304f2e49d..cf272d94dcd 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java
@@ -237,7 +237,8 @@ public class RoutingController {
}
}
- private boolean supportsRoutingMethod(RoutingMethod routingMethod, ZoneId zone) {
+ /** Returns whether given routingMethod is supported by zone */
+ public boolean supportsRoutingMethod(RoutingMethod routingMethod, ZoneId zone) {
return controller.zoneRegistry().zones().routingMethod(routingMethod).ids().contains(zone);
}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java
index c39255fd7a8..5fa463233fd 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java
@@ -1,15 +1,13 @@
// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.hosted.controller.application;
-import com.google.common.hash.Hashing;
-import com.google.common.io.BaseEncoding;
import com.yahoo.config.provision.ApplicationId;
import com.yahoo.config.provision.ClusterSpec;
import com.yahoo.config.provision.SystemName;
+import com.yahoo.config.provision.zone.RoutingMethod;
import com.yahoo.config.provision.zone.ZoneId;
import java.net.URI;
-import java.nio.charset.Charset;
import java.util.Objects;
/**
@@ -20,28 +18,29 @@ import java.util.Objects;
*/
public class Endpoint {
- public static final String YAHOO_DNS_SUFFIX = ".vespa.yahooapis.com";
- public static final String OATH_DNS_SUFFIX = ".vespa.oath.cloud";
- public static final String PUBLIC_DNS_SUFFIX = ".public.vespa.oath.cloud";
- public static final String PUBLIC_CD_DNS_SUFFIX = ".public-cd.vespa.oath.cloud";
+ private static final String YAHOO_DNS_SUFFIX = ".vespa.yahooapis.com";
+ private static final String OATH_DNS_SUFFIX = ".vespa.oath.cloud";
+ private static final String PUBLIC_DNS_SUFFIX = ".public.vespa.oath.cloud";
+ private static final String PUBLIC_CD_DNS_SUFFIX = ".public-cd.vespa.oath.cloud";
private final URI url;
private final Scope scope;
private final boolean legacy;
- private final boolean directRouting;
+ private final RoutingMethod routingMethod;
private final boolean tls;
private final boolean wildcard;
private Endpoint(String name, ApplicationId application, ZoneId zone, SystemName system, Port port, boolean legacy,
- boolean directRouting, boolean wildcard) {
+ RoutingMethod routingMethod, boolean wildcard) {
Objects.requireNonNull(name, "name must be non-null");
Objects.requireNonNull(application, "application must be non-null");
Objects.requireNonNull(system, "system must be non-null");
Objects.requireNonNull(port, "port must be non-null");
- this.url = createUrl(name, application, zone, system, port, legacy, directRouting);
+ Objects.requireNonNull(routingMethod, "routingMethod must be non-null");
+ this.url = createUrl(name, application, zone, system, port, legacy, routingMethod);
this.scope = zone == null ? Scope.global : Scope.zone;
this.legacy = legacy;
- this.directRouting = directRouting;
+ this.routingMethod = routingMethod;
this.tls = port.tls;
this.wildcard = wildcard;
}
@@ -67,12 +66,9 @@ public class Endpoint {
return legacy;
}
- /**
- * Returns whether this endpoint supports direct routing. Direct routing means that this endpoint is served by an
- * exclusive load balancer instead of a shared routing layer.
- */
- public boolean directRouting() {
- return directRouting;
+ /** Returns the routing used for this */
+ public RoutingMethod routingMethod() {
+ return routingMethod;
}
/** Returns whether this endpoint supports TLS connections */
@@ -100,13 +96,18 @@ public class Endpoint {
@Override
public String toString() {
- return String.format("endpoint %s [scope=%s, legacy=%s, directRouting=%s]", url, scope, legacy, directRouting);
+ return String.format("endpoint %s [scope=%s, legacy=%s, routingMethod=%s]", url, scope, legacy, routingMethod);
+ }
+
+ /** Returns the DNS suffix used for endpoints in given system */
+ public static String dnsSuffix(SystemName system) {
+ return dnsSuffix(system, false);
}
private static URI createUrl(String name, ApplicationId application, ZoneId zone, SystemName system,
- Port port, boolean legacy, boolean directRouting) {
+ Port port, boolean legacy, RoutingMethod routingMethod) {
String scheme = port.tls ? "https" : "http";
- String separator = separator(system, directRouting, port.tls);
+ String separator = separator(system, routingMethod, port.tls);
String portPart = port.isDefault() ? "" : ":" + port.port;
return URI.create(scheme + "://" +
sanitize(namePart(name, separator)) +
@@ -126,9 +127,9 @@ public class Endpoint {
return part.replace('_', '-');
}
- private static String separator(SystemName system, boolean directRouting, boolean tls) {
+ private static String separator(SystemName system, RoutingMethod routingMethod, boolean tls) {
if (!tls) return ".";
- if (directRouting) return ".";
+ if (routingMethod.isDirect()) return ".";
if (system.isPublic()) return ".";
return "--";
}
@@ -214,13 +215,6 @@ public class Endpoint {
}
- /** Create a DNS name based on a hash of the ApplicationId. This should always be less than 64 characters long. */
- public static String createHashedCn(ApplicationId application, SystemName system) {
- var hashCode = Hashing.sha1().hashString(application.serializedForm(), Charset.defaultCharset());
- var base32encoded = BaseEncoding.base32().omitPadding().lowerCase().encode(hashCode.asBytes());
- return 'v' + base32encoded + dnsSuffix(system, false);
- }
-
/** Build an endpoint for given application */
public static EndpointBuilder of(ApplicationId application) {
return new EndpointBuilder(application);
@@ -234,8 +228,8 @@ public class Endpoint {
private ClusterSpec.Id cluster;
private EndpointId endpointId;
private Port port;
+ private RoutingMethod routingMethod = RoutingMethod.shared;
private boolean legacy = false;
- private boolean directRouting = false;
private boolean wildcard = false;
private EndpointBuilder(ApplicationId application) {
@@ -292,9 +286,9 @@ public class Endpoint {
return this;
}
- /** Enables direct routing support for this */
- public EndpointBuilder directRouting() {
- this.directRouting = true;
+ /** Sets the routing method for this */
+ public EndpointBuilder routingMethod(RoutingMethod method) {
+ this.routingMethod = method;
return this;
}
@@ -310,13 +304,13 @@ public class Endpoint {
} else {
throw new IllegalArgumentException("Must set either cluster, rotation or wildcard target");
}
- if (system.isPublic() && !directRouting) {
- throw new IllegalArgumentException("Public system only supports direct routing endpoints");
+ if (system.isPublic() && routingMethod != RoutingMethod.exclusive) {
+ throw new IllegalArgumentException("Public system only supports routing method " + RoutingMethod.exclusive);
}
- if (directRouting && !port.isDefault()) {
- throw new IllegalArgumentException("Direct routing endpoints only support default port");
+ if (routingMethod.isDirect() && !port.isDefault()) {
+ throw new IllegalArgumentException("Routing method " + routingMethod + " can only use default port");
}
- return new Endpoint(name, application, zone, system, port, legacy, directRouting, wildcard);
+ return new Endpoint(name, application, zone, system, port, legacy, routingMethod, wildcard);
}
}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/endpointcertificates/EndpointCertificateManager.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificateManager.java
index d915da21603..e22d381e615 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/endpointcertificates/EndpointCertificateManager.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificateManager.java
@@ -1,8 +1,12 @@
-package com.yahoo.vespa.hosted.controller.endpointcertificates;
+package com.yahoo.vespa.hosted.controller.certificate;
import com.google.common.collect.Sets;
+import com.google.common.hash.Hashing;
+import com.google.common.io.BaseEncoding;
import com.yahoo.config.provision.ApplicationId;
import com.yahoo.config.provision.ClusterSpec;
+import com.yahoo.config.provision.SystemName;
+import com.yahoo.config.provision.zone.RoutingMethod;
import com.yahoo.config.provision.zone.ZoneApi;
import com.yahoo.config.provision.zone.ZoneId;
import com.yahoo.container.jdisc.secretstore.SecretNotFoundException;
@@ -23,6 +27,7 @@ import com.yahoo.vespa.hosted.controller.application.Endpoint;
import com.yahoo.vespa.hosted.controller.application.EndpointId;
import com.yahoo.vespa.hosted.controller.persistence.CuratorDb;
+import java.nio.charset.Charset;
import java.security.cert.X509Certificate;
import java.time.Clock;
import java.time.Instant;
@@ -142,7 +147,7 @@ public class EndpointCertificateManager {
if (storedMetaData.requestedDnsSans().isPresent() && storedMetaData.request_id().isPresent())
return;
- var hashedCn = Endpoint.createHashedCn(applicationId, zoneRegistry.system()); // use as join key
+ var hashedCn = commonNameHashOf(applicationId, zoneRegistry.system()); // use as join key
EndpointCertificateMetadata providerMetadata = sanToEndpointCertificate.get(hashedCn);
if(providerMetadata == null) {
@@ -156,7 +161,8 @@ public class EndpointCertificateManager {
storedMetaData.certName(),
storedMetaData.version(),
providerMetadata.request_id(),
- providerMetadata.requestedDnsSans());
+ providerMetadata.requestedDnsSans(),
+ Optional.empty());
if (mode == BackfillMode.DRYRUN) {
log.log(LogLevel.INFO, "Would update stored metadata " + storedMetaData + " with data from provider: " + backfilledMetadata);
@@ -176,7 +182,7 @@ public class EndpointCertificateManager {
private EndpointCertificateMetadata provisionEndpointCertificate(Instance instance) {
List<ZoneId> zones = zoneRegistry.zones().controllerUpgraded().zones().stream().map(ZoneApi::getId).collect(Collectors.toUnmodifiableList());
EndpointCertificateMetadata provisionedCertificateMetadata = endpointCertificateProvider
- .requestCaSignedCertificate(instance.id(), dnsNamesOf(instance.id(), zones));
+ .requestCaSignedCertificate(instance.id(), dnsNamesOf(instance.id(), zones), Optional.empty());
curator.writeEndpointCertificateMetadata(instance.id(), provisionedCertificateMetadata);
return provisionedCertificateMetadata;
}
@@ -232,7 +238,7 @@ public class EndpointCertificateManager {
// We add first an endpoint name based on a hash of the applicationId,
// as the certificate provider requires the first CN to be < 64 characters long.
- endpointDnsNames.add(Endpoint.createHashedCn(applicationId, zoneRegistry.system()));
+ endpointDnsNames.add(commonNameHashOf(applicationId, zoneRegistry.system()));
var globalDefaultEndpoint = Endpoint.of(applicationId).named(EndpointId.defaultId());
var rotationEndpoints = Endpoint.of(applicationId).wildcard();
@@ -243,7 +249,7 @@ public class EndpointCertificateManager {
));
Stream.concat(Stream.of(globalDefaultEndpoint, rotationEndpoints), zoneLocalEndpoints)
- .map(Endpoint.EndpointBuilder::directRouting)
+ .map(endpoint -> endpoint.routingMethod(RoutingMethod.exclusive))
.map(endpoint -> endpoint.on(Endpoint.Port.tls()))
.map(endpointBuilder -> endpointBuilder.in(zoneRegistry.system()))
.map(Endpoint::dnsName).forEach(endpointDnsNames::add);
@@ -251,4 +257,11 @@ public class EndpointCertificateManager {
return Collections.unmodifiableList(endpointDnsNames);
}
+ /** Create a common name based on a hash of the ApplicationId. This should always be less than 64 characters long. */
+ private static String commonNameHashOf(ApplicationId application, SystemName system) {
+ var hashCode = Hashing.sha1().hashString(application.serializedForm(), Charset.defaultCharset());
+ var base32encoded = BaseEncoding.base32().omitPadding().lowerCase().encode(hashCode.asBytes());
+ return 'v' + base32encoded + Endpoint.dnsSuffix(system);
+ }
+
}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java
index 9d666c6f7b5..4d09765f78b 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java
@@ -28,7 +28,6 @@ import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId;
import com.yahoo.vespa.hosted.controller.api.identifiers.Hostname;
import com.yahoo.vespa.hosted.controller.api.integration.LogEntry;
import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerException;
-import com.yahoo.vespa.hosted.controller.api.integration.configserver.Log;
import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node;
import com.yahoo.vespa.hosted.controller.api.integration.configserver.PrepareResponse;
import com.yahoo.vespa.hosted.controller.api.integration.configserver.ServiceConvergence;
@@ -38,6 +37,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId;
import com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterCloud;
import com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterId;
import com.yahoo.vespa.hosted.controller.api.integration.organization.DeploymentFailureMails;
+import com.yahoo.vespa.hosted.controller.api.integration.organization.Mail;
import com.yahoo.vespa.hosted.controller.application.ApplicationPackage;
import com.yahoo.vespa.hosted.controller.application.Deployment;
import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId;
@@ -76,7 +76,6 @@ import static com.yahoo.vespa.hosted.controller.api.integration.configserver.Nod
import static com.yahoo.vespa.hosted.controller.api.integration.configserver.Node.State.reserved;
import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.aborted;
import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.deploymentFailed;
-import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.endpointCertificateTimeout;
import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.error;
import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.installationFailed;
import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.outOfCapacity;
@@ -672,22 +671,36 @@ public class InternalStepRunner implements StepRunner {
return;
try {
- if (run.status() == outOfCapacity && run.id().type().isProduction())
- controller.serviceRegistry().mailer().send(mails.outOfCapacity(run.id(), recipients));
- if (run.status() == deploymentFailed)
- controller.serviceRegistry().mailer().send(mails.deploymentFailure(run.id(), recipients));
- if (run.status() == installationFailed)
- controller.serviceRegistry().mailer().send(mails.installationFailure(run.id(), recipients));
- if (run.status() == testFailure)
- controller.serviceRegistry().mailer().send(mails.testFailure(run.id(), recipients));
- if (run.status() == error)
- controller.serviceRegistry().mailer().send(mails.systemError(run.id(), recipients));
+ mailOf(run, recipients).ifPresent(controller.serviceRegistry().mailer()::send);
}
catch (RuntimeException e) {
logger.log(INFO, "Exception trying to send mail for " + run.id(), e);
}
}
+ private Optional<Mail> mailOf(Run run, List<String> recipients) {
+ switch (run.status()) {
+ case running:
+ case aborted:
+ case success:
+ return Optional.empty();
+ case outOfCapacity:
+ return run.id().type().isProduction() ? Optional.of(mails.outOfCapacity(run.id(), recipients)) : Optional.empty();
+ case deploymentFailed:
+ return Optional.of(mails.deploymentFailure(run.id(), recipients));
+ case installationFailed:
+ return Optional.of(mails.installationFailure(run.id(), recipients));
+ case testFailure:
+ return Optional.of(mails.testFailure(run.id(), recipients));
+ case error:
+ case endpointCertificateTimeout:
+ return Optional.of(mails.systemError(run.id(), recipients));
+ default:
+ logger.log(WARNING, "Don't know what mail to send for run status '" + run.status() + "'");
+ return Optional.of(mails.systemError(run.id(), recipients));
+ }
+ }
+
/** Returns the deployment of the real application in the zone of the given job, if it exists. */
private Optional<Deployment> deployment(ApplicationId id, JobType type) {
return Optional.ofNullable(application(id).deployments().get(type.zone(controller.system())));
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CostReportMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CostReportMaintainer.java
index ff88805f957..5b792f384e5 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CostReportMaintainer.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CostReportMaintainer.java
@@ -1,12 +1,11 @@
// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.hosted.controller.maintenance;
-import com.yahoo.config.provision.CloudName;
import com.yahoo.config.provision.SystemName;
import com.yahoo.vespa.hosted.controller.Controller;
import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeRepository;
import com.yahoo.vespa.hosted.controller.api.integration.resource.CostReportConsumer;
-import com.yahoo.vespa.hosted.controller.restapi.cost.CostCalculator;
+import com.yahoo.vespa.hosted.controller.metric.CostCalculator;
import java.time.Clock;
import java.time.Duration;
@@ -34,7 +33,8 @@ public class CostReportMaintainer extends Maintainer {
@Override
protected void maintain() {
- consumer.consume(CostCalculator.resourceShareByPropertyToCsv(nodeRepository, controller(), clock, consumer.fixedAllocations(), CloudName.from("yahoo")));
+ var csv = CostCalculator.resourceShareByPropertyToCsv(nodeRepository, controller(), clock, consumer.fixedAllocations());
+ consumer.consume(csv);
}
}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/cost/CostCalculator.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/metric/CostCalculator.java
index ec01b3817a7..4cc4ee0386c 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/cost/CostCalculator.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/metric/CostCalculator.java
@@ -1,5 +1,5 @@
// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.hosted.controller.restapi.cost;
+package com.yahoo.vespa.hosted.controller.metric;
import com.yahoo.config.provision.CloudName;
import com.yahoo.config.provision.Environment;
@@ -28,12 +28,12 @@ import static com.yahoo.yolean.Exceptions.uncheck;
public class CostCalculator {
private static final double SELF_HOSTED_DISCOUNT = .5;
+ private static final CloudName cloudName = CloudName.from("yahoo");
public static String resourceShareByPropertyToCsv(NodeRepository nodeRepository,
Controller controller,
Clock clock,
- Map<Property, ResourceAllocation> fixedAllocations,
- CloudName cloudName) {
+ Map<Property, ResourceAllocation> fixedAllocations) {
var date = DateTimeFormatter.ofPattern("yyyy-MM-dd").withZone(ZoneId.of("UTC")).format(clock.instant());
@@ -61,14 +61,12 @@ public class CostCalculator {
}
// Add fixed allocations from config
- if (cloudName.equals(CloudName.from("yahoo"))) {
- for (var kv : fixedAllocations.entrySet()) {
- var property = kv.getKey();
- var allocation = allocationByProperty.getOrDefault(property, ResourceAllocation.ZERO);
- var discountedFixedAllocation = kv.getValue().multiply(SELF_HOSTED_DISCOUNT);
- allocationByProperty.put(property, allocation.plus(discountedFixedAllocation));
- totalAllocation = totalAllocation.plus(discountedFixedAllocation);
- }
+ for (var kv : fixedAllocations.entrySet()) {
+ var property = kv.getKey();
+ var allocation = allocationByProperty.getOrDefault(property, ResourceAllocation.ZERO);
+ var discountedFixedAllocation = kv.getValue().multiply(SELF_HOSTED_DISCOUNT);
+ allocationByProperty.put(property, allocation.plus(discountedFixedAllocation));
+ totalAllocation = totalAllocation.plus(discountedFixedAllocation);
}
return toCsv(allocationByProperty, date, totalAllocation);
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java
index ad2835e301f..eb86b1028e2 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java
@@ -42,7 +42,6 @@ import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
-import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.NavigableMap;
@@ -521,8 +520,7 @@ public class CuratorDb {
}
public Optional<EndpointCertificateMetadata> readEndpointCertificateMetadata(ApplicationId applicationId) {
- Optional<String> zkData = curator.getData(endpointCertificatePath(applicationId)).map(String::new);
- return zkData.map(EndpointCertificateMetadataSerializer::fromJsonOrTlsSecretsKeysString);
+ return curator.getData(endpointCertificatePath(applicationId)).map(String::new).map(EndpointCertificateMetadataSerializer::fromJsonString);
}
public Map<ApplicationId, EndpointCertificateMetadata> readAllEndpointCertificateMetadata() {
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/EndpointCertificateMetadataSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/EndpointCertificateMetadataSerializer.java
index 653f224a02b..501d3a06d42 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/EndpointCertificateMetadataSerializer.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/EndpointCertificateMetadataSerializer.java
@@ -4,6 +4,7 @@ import com.yahoo.slime.Cursor;
import com.yahoo.slime.Inspector;
import com.yahoo.slime.Slime;
import com.yahoo.slime.SlimeUtils;
+import com.yahoo.slime.Type;
import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificateMetadata;
import java.util.List;
@@ -33,6 +34,7 @@ public class EndpointCertificateMetadataSerializer {
private final static String versionField = "version";
private final static String requestIdField = "requestId";
private final static String requestedDnsSansField = "requestedDnsSans";
+ private final static String issuerField = "issuer";
public static Slime toSlime(EndpointCertificateMetadata metadata) {
Slime slime = new Slime();
@@ -51,46 +53,31 @@ public class EndpointCertificateMetadataSerializer {
}
public static EndpointCertificateMetadata fromSlime(Inspector inspector) {
- switch (inspector.type()) {
- case STRING: // TODO: Remove once all are transmitted and stored as JSON
- return new EndpointCertificateMetadata(
- inspector.asString() + "-key",
- inspector.asString() + "-cert",
- 0
- );
- case OBJECT: {
- Optional<String> request_id = inspector.field(requestIdField).valid() ?
- Optional.of(inspector.field(requestIdField).asString()) :
- Optional.empty();
+ if (inspector.type() != Type.OBJECT)
+ throw new IllegalArgumentException("Unknown format encountered for endpoint certificate metadata!");
+ Optional<String> request_id = inspector.field(requestIdField).valid() ?
+ Optional.of(inspector.field(requestIdField).asString()) :
+ Optional.empty();
- Optional<List<String>> requestedDnsSans = inspector.field(requestedDnsSansField).valid() ?
- Optional.of(IntStream.range(0, inspector.field(requestedDnsSansField).entries())
- .mapToObj(i -> inspector.field(requestedDnsSansField).entry(i).asString()).collect(Collectors.toList())) :
- Optional.empty();
+ Optional<List<String>> requestedDnsSans = inspector.field(requestedDnsSansField).valid() ?
+ Optional.of(IntStream.range(0, inspector.field(requestedDnsSansField).entries())
+ .mapToObj(i -> inspector.field(requestedDnsSansField).entry(i).asString()).collect(Collectors.toList())) :
+ Optional.empty();
- return new EndpointCertificateMetadata(
- inspector.field(keyNameField).asString(),
- inspector.field(certNameField).asString(),
- Math.toIntExact(inspector.field(versionField).asLong()),
- request_id,
- requestedDnsSans
- );
- }
+ Optional<String> issuer = inspector.field(issuerField).valid() ?
+ Optional.of(inspector.field(issuerField).asString()) :
+ Optional.empty();
- default:
- throw new IllegalArgumentException("Unknown format encountered for endpoint certificate metadata!");
- }
+ return new EndpointCertificateMetadata(
+ inspector.field(keyNameField).asString(),
+ inspector.field(certNameField).asString(),
+ Math.toIntExact(inspector.field(versionField).asLong()),
+ request_id,
+ requestedDnsSans,
+ issuer);
}
- public static EndpointCertificateMetadata fromTlsSecretsKeysString(String tlsSecretsKeys) {
- return fromSlime(new Slime().setString(tlsSecretsKeys));
- }
-
- public static EndpointCertificateMetadata fromJsonOrTlsSecretsKeysString(String zkdata) {
- if (zkdata.strip().startsWith("{")) {
- return fromSlime(SlimeUtils.jsonToSlime(zkdata).get());
- } else {
- return fromTlsSecretsKeysString(zkdata);
- }
+ public static EndpointCertificateMetadata fromJsonString(String zkdata) {
+ return fromSlime(SlimeUtils.jsonToSlime(zkdata).get());
}
}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java
index 073306719f3..94d7b120406 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java
@@ -1061,6 +1061,9 @@ public class ApplicationApiHandler extends LoggingRequestHandler {
// Add zone endpoints defined by routing policies
var endpointArray = response.setArray("endpoints");
for (var policy : controller.routingController().policies().get(deploymentId).values()) {
+ // TODO(mpolden): Always add endpoints from all policies, independent of routing method. This allows removal
+ // of RoutingGenerator and eliminates the external call to the routing layer below.
+ if (!controller.routingController().supportsRoutingMethod(RoutingMethod.exclusive, deployment.zone())) continue;
if (!policy.status().isActive()) continue;
{
var endpointObject = endpointArray.addObject();
@@ -1157,6 +1160,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler {
deployment.activity().lastWritesPerSecond().ifPresent(value -> activity.setDouble("lastWritesPerSecond", value));
// Cost
+ // TODO(mpolden): Unused, remove this field and related code.
DeploymentCost appCost = new DeploymentCost(Map.of());
Cursor costObject = response.setObject("cost");
toSlime(appCost, costObject);
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java
index 32c2d6ec3d1..0d1dca391bd 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java
@@ -554,15 +554,16 @@ class JobControllerApiHandlerHelper {
private static String nameOf(RunStatus status) {
switch (status) {
- case running: return "running";
- case aborted: return "aborted";
- case error: return "error";
- case testFailure: return "testFailure";
- case outOfCapacity: return "outOfCapacity";
- case installationFailed: return "installationFailed";
- case deploymentFailed: return "deploymentFailed";
- case success: return "success";
- default: throw new IllegalArgumentException("Unexpected status '" + status + "'");
+ case running: return "running";
+ case aborted: return "aborted";
+ case error: return "error";
+ case testFailure: return "testFailure";
+ case endpointCertificateTimeout: return "endpointCertificateTimeout";
+ case outOfCapacity: return "outOfCapacity";
+ case installationFailed: return "installationFailed";
+ case deploymentFailed: return "deploymentFailed";
+ case success: return "success";
+ default: throw new IllegalArgumentException("Unexpected status '" + status + "'");
}
}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/cost/CostApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/cost/CostApiHandler.java
deleted file mode 100644
index 8bf3f4a6fd0..00000000000
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/cost/CostApiHandler.java
+++ /dev/null
@@ -1,52 +0,0 @@
-// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.hosted.controller.restapi.cost;
-
-import com.yahoo.config.provision.CloudName;
-import com.yahoo.container.jdisc.HttpRequest;
-import com.yahoo.container.jdisc.HttpResponse;
-import com.yahoo.container.jdisc.LoggingRequestHandler;
-import com.yahoo.restapi.Path;
-import com.yahoo.vespa.hosted.controller.Controller;
-import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeRepository;
-import com.yahoo.vespa.hosted.controller.api.integration.resource.CostReportConsumer;
-import com.yahoo.restapi.ErrorResponse;
-import com.yahoo.restapi.StringResponse;
-
-import java.time.Clock;
-import java.util.Optional;
-
-import static com.yahoo.jdisc.http.HttpRequest.Method.GET;
-
-/**
- * @author ldalves
- */
-public class CostApiHandler extends LoggingRequestHandler {
-
- private final Controller controller;
- private final NodeRepository nodeRepository;
- private final CostReportConsumer costReportConsumer;
-
- public CostApiHandler(Context ctx, Controller controller) {
- super(ctx);
- this.controller = controller;
- this.nodeRepository = controller.serviceRegistry().configServer().nodeRepository();
- this.costReportConsumer = controller.serviceRegistry().costReportConsumer();
- }
-
- @Override
- public HttpResponse handle(HttpRequest request) {
- if (request.getMethod() != GET) {
- return ErrorResponse.methodNotAllowed("Method '" + request.getMethod() + "' is not supported");
- }
-
- Path path = new Path(request.getUri());
-
- if (path.matches("/cost/v1/csv")) {
- Optional<String> cloudProperty = Optional.ofNullable(request.getProperty("cloud"));
- CloudName cloud = cloudProperty.map(CloudName::from).orElse(CloudName.defaultName());
- return new StringResponse(CostCalculator.resourceShareByPropertyToCsv(nodeRepository, controller, Clock.systemUTC(), costReportConsumer.fixedAllocations(), cloud));
- }
-
- return ErrorResponse.notFoundError("Nothing at " + path);
- }
-}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/cost/package-info.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/cost/package-info.java
deleted file mode 100644
index a96ae5488fa..00000000000
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/cost/package-info.java
+++ /dev/null
@@ -1,5 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-@ExportPackage
-package com.yahoo.vespa.hosted.controller.restapi.cost;
-
-import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicy.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicy.java
index 9ef2d519c05..32ec9f359a9 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicy.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicy.java
@@ -5,6 +5,7 @@ import com.google.common.collect.ImmutableSortedSet;
import com.yahoo.config.provision.ApplicationId;
import com.yahoo.config.provision.HostName;
import com.yahoo.config.provision.SystemName;
+import com.yahoo.config.provision.zone.RoutingMethod;
import com.yahoo.vespa.hosted.controller.application.Endpoint;
import com.yahoo.vespa.hosted.controller.application.Endpoint.Port;
import com.yahoo.vespa.hosted.controller.application.EndpointId;
@@ -70,7 +71,10 @@ public class RoutingPolicy {
/** Returns the endpoint of this */
public Endpoint endpointIn(SystemName system) {
- return Endpoint.of(id.owner()).target(id.cluster(), id.zone()).on(Port.tls()).directRouting().in(system);
+ return Endpoint.of(id.owner()).target(id.cluster(), id.zone())
+ .routingMethod(RoutingMethod.exclusive)
+ .on(Port.tls())
+ .in(system);
}
/** Returns global endpoints which this is a member of */
@@ -100,7 +104,10 @@ public class RoutingPolicy {
/** Creates a global endpoint for given application */
public static Endpoint globalEndpointOf(ApplicationId application, EndpointId endpointId, SystemName system) {
- return Endpoint.of(application).named(endpointId).on(Port.tls()).directRouting().in(system);
+ return Endpoint.of(application).named(endpointId)
+ .on(Port.tls())
+ .routingMethod(RoutingMethod.exclusive)
+ .in(system);
}
}
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/EndpointTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/EndpointTest.java
index ea97e3e6c71..81bbbb479b2 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/EndpointTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/EndpointTest.java
@@ -4,6 +4,7 @@ package com.yahoo.vespa.hosted.controller.application;
import com.yahoo.config.provision.ApplicationId;
import com.yahoo.config.provision.ClusterSpec;
import com.yahoo.config.provision.SystemName;
+import com.yahoo.config.provision.zone.RoutingMethod;
import com.yahoo.config.provision.zone.ZoneId;
import com.yahoo.vespa.hosted.controller.application.Endpoint.Port;
import org.junit.Test;
@@ -47,23 +48,23 @@ public class EndpointTest {
// Main endpoint with direct routing and default TLS port
"https://a1.t1.global.vespa.oath.cloud/",
- Endpoint.of(app1).named(endpointId).on(Port.tls()).directRouting().in(SystemName.main),
+ Endpoint.of(app1).named(endpointId).on(Port.tls()).routingMethod(RoutingMethod.exclusive).in(SystemName.main),
// Main endpoint with custom rotation name
"https://r1.a1.t1.global.vespa.oath.cloud/",
- Endpoint.of(app1).named(EndpointId.of("r1")).on(Port.tls()).directRouting().in(SystemName.main),
+ Endpoint.of(app1).named(EndpointId.of("r1")).on(Port.tls()).routingMethod(RoutingMethod.exclusive).in(SystemName.main),
// Main endpoint for custom instance in default rotation
"https://i2.a2.t2.global.vespa.oath.cloud/",
- Endpoint.of(app2).named(endpointId).on(Port.tls()).directRouting().in(SystemName.main),
+ Endpoint.of(app2).named(endpointId).on(Port.tls()).routingMethod(RoutingMethod.exclusive).in(SystemName.main),
// Main endpoint for custom instance with custom rotation name
"https://r2.i2.a2.t2.global.vespa.oath.cloud/",
- Endpoint.of(app2).named(EndpointId.of("r2")).on(Port.tls()).directRouting().in(SystemName.main),
+ Endpoint.of(app2).named(EndpointId.of("r2")).on(Port.tls()).routingMethod(RoutingMethod.exclusive).in(SystemName.main),
// Main endpoint in public system
"https://a1.t1.global.public.vespa.oath.cloud/",
- Endpoint.of(app1).named(endpointId).on(Port.tls()).directRouting().in(SystemName.Public)
+ Endpoint.of(app1).named(endpointId).on(Port.tls()).routingMethod(RoutingMethod.exclusive).in(SystemName.Public)
);
tests.forEach((expected, endpoint) -> assertEquals(expected, endpoint.url().toString()));
}
@@ -95,23 +96,23 @@ public class EndpointTest {
// Main endpoint with direct routing and default TLS port
"https://a1.t1.global.vespa.oath.cloud/",
- Endpoint.of(app1).named(endpointId).on(Port.tls()).directRouting().in(SystemName.main),
+ Endpoint.of(app1).named(endpointId).on(Port.tls()).routingMethod(RoutingMethod.exclusive).in(SystemName.main),
// Main endpoint with custom rotation name
"https://r1.a1.t1.global.vespa.oath.cloud/",
- Endpoint.of(app1).named(EndpointId.of("r1")).on(Port.tls()).directRouting().in(SystemName.main),
+ Endpoint.of(app1).named(EndpointId.of("r1")).on(Port.tls()).routingMethod(RoutingMethod.exclusive).in(SystemName.main),
// Main endpoint for custom instance in default rotation
"https://i2.a2.t2.global.vespa.oath.cloud/",
- Endpoint.of(app2).named(endpointId).on(Port.tls()).directRouting().in(SystemName.main),
+ Endpoint.of(app2).named(endpointId).on(Port.tls()).routingMethod(RoutingMethod.exclusive).in(SystemName.main),
// Main endpoint for custom instance with custom rotation name
"https://r2.i2.a2.t2.global.vespa.oath.cloud/",
- Endpoint.of(app2).named(EndpointId.of("r2")).on(Port.tls()).directRouting().in(SystemName.main),
+ Endpoint.of(app2).named(EndpointId.of("r2")).on(Port.tls()).routingMethod(RoutingMethod.exclusive).in(SystemName.main),
// Main endpoint in public system
"https://a1.t1.global.public.vespa.oath.cloud/",
- Endpoint.of(app1).named(endpointId).on(Port.tls()).directRouting().in(SystemName.Public)
+ Endpoint.of(app1).named(endpointId).on(Port.tls()).routingMethod(RoutingMethod.exclusive).in(SystemName.Public)
);
tests.forEach((expected, endpoint) -> assertEquals(expected, endpoint.url().toString()));
}
@@ -153,11 +154,15 @@ public class EndpointTest {
// Non-default cluster in public
"https://c1.a1.t1.us-north-1.public.vespa.oath.cloud/",
- Endpoint.of(app1).target(ClusterSpec.Id.from("c1"), prodZone).on(Port.tls()).directRouting().in(SystemName.Public),
+ Endpoint.of(app1).target(ClusterSpec.Id.from("c1"), prodZone).on(Port.tls()).routingMethod(RoutingMethod.exclusive).in(SystemName.Public),
// Non-default cluster and instance in public
"https://c2.i2.a2.t2.us-north-1.public.vespa.oath.cloud/",
- Endpoint.of(app2).target(ClusterSpec.Id.from("c2"), prodZone).on(Port.tls()).directRouting().in(SystemName.Public)
+ Endpoint.of(app2).target(ClusterSpec.Id.from("c2"), prodZone).on(Port.tls()).routingMethod(RoutingMethod.exclusive).in(SystemName.Public),
+
+ // Endpoint in main using shared layer 4
+ "https://a1.t1.us-north-1.vespa.oath.cloud/",
+ Endpoint.of(app1).target(cluster, prodZone).on(Port.tls()).routingMethod(RoutingMethod.sharedLayer4).in(SystemName.main)
);
tests.forEach((expected, endpoint) -> assertEquals(expected, endpoint.url().toString()));
}
@@ -173,7 +178,7 @@ public class EndpointTest {
"https://a1.t1.global.public.vespa.oath.cloud/",
Endpoint.of(app1)
.named(EndpointId.defaultId())
- .directRouting()
+ .routingMethod(RoutingMethod.exclusive)
.on(Port.tls())
.in(SystemName.Public),
@@ -181,7 +186,7 @@ public class EndpointTest {
"https://*.a1.t1.global.public.vespa.oath.cloud/",
Endpoint.of(app1)
.wildcard()
- .directRouting()
+ .routingMethod(RoutingMethod.exclusive)
.on(Port.tls())
.in(SystemName.Public),
@@ -189,7 +194,7 @@ public class EndpointTest {
"https://a1.t1.us-north-1.public.vespa.oath.cloud/",
Endpoint.of(app1)
.target(defaultCluster, prodZone)
- .directRouting()
+ .routingMethod(RoutingMethod.exclusive)
.on(Port.tls())
.in(SystemName.Public),
@@ -197,7 +202,7 @@ public class EndpointTest {
"https://*.a1.t1.us-north-1.public.vespa.oath.cloud/",
Endpoint.of(app1)
.wildcard(prodZone)
- .directRouting()
+ .routingMethod(RoutingMethod.exclusive)
.on(Port.tls())
.in(SystemName.Public),
@@ -205,7 +210,7 @@ public class EndpointTest {
"https://a1.t1.us-north-2.test.public.vespa.oath.cloud/",
Endpoint.of(app1)
.target(defaultCluster, testZone)
- .directRouting()
+ .routingMethod(RoutingMethod.exclusive)
.on(Port.tls())
.in(SystemName.Public),
@@ -213,7 +218,7 @@ public class EndpointTest {
"https://*.a1.t1.us-north-2.test.public.vespa.oath.cloud/",
Endpoint.of(app1)
.wildcard(testZone)
- .directRouting()
+ .routingMethod(RoutingMethod.exclusive)
.on(Port.tls())
.in(SystemName.Public)
);
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/endpointcertificates/EndpointCertificateManagerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificateManagerTest.java
index 3f8e91dec58..93f081be63e 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/endpointcertificates/EndpointCertificateManagerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificateManagerTest.java
@@ -1,4 +1,4 @@
-package com.yahoo.vespa.hosted.controller.endpointcertificates;
+package com.yahoo.vespa.hosted.controller.certificate;
import com.yahoo.config.provision.ApplicationId;
import com.yahoo.config.provision.SystemName;
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/EndpointCertificateMetadataSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/EndpointCertificateMetadataSerializerTest.java
index 7428b9901a2..5f8a3eaa98a 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/EndpointCertificateMetadataSerializerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/EndpointCertificateMetadataSerializerTest.java
@@ -29,17 +29,10 @@ public class EndpointCertificateMetadataSerializerTest {
}
@Test
- public void deserializeFromString() {
- assertEquals(
- new EndpointCertificateMetadata("foo-key", "foo-cert", 0),
- EndpointCertificateMetadataSerializer.fromJsonOrTlsSecretsKeysString("foo"));
- }
-
- @Test
public void deserializeFromJson() {
assertEquals(
sample,
- EndpointCertificateMetadataSerializer.fromJsonOrTlsSecretsKeysString(
+ EndpointCertificateMetadataSerializer.fromJsonString(
"{\"keyName\":\"keyName\",\"certName\":\"certName\",\"version\":1}"));
}
@@ -47,7 +40,7 @@ public class EndpointCertificateMetadataSerializerTest {
public void deserializeFromJsonWithRequestMetadata() {
assertEquals(
sampleWithRequestMetadata,
- EndpointCertificateMetadataSerializer.fromJsonOrTlsSecretsKeysString(
+ EndpointCertificateMetadataSerializer.fromJsonString(
"{\"keyName\":\"keyName\",\"certName\":\"certName\",\"version\":1,\"requestId\":\"requestId\",\"requestedDnsSans\":[\"SAN1\",\"SAN2\"]}"));
}
} \ No newline at end of file
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java
index da770c9c023..9e03a236f4a 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java
@@ -92,9 +92,6 @@ public class ControllerContainerTest {
" <handler id='com.yahoo.vespa.hosted.controller.restapi.os.OsApiHandler'>\n" +
" <binding>http://*/os/v1/*</binding>\n" +
" </handler>\n" +
- " <handler id='com.yahoo.vespa.hosted.controller.restapi.cost.CostApiHandler'>\n" +
- " <binding>http://*/cost/v1/*</binding>\n" +
- " </handler>\n" +
" <handler id='com.yahoo.vespa.hosted.controller.restapi.zone.v2.ZoneApiHandler'>\n" +
" <binding>http://*/zone/v2</binding>\n" +
" <binding>http://*/zone/v2/*</binding>\n" +
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java
index 88faf54ea28..442770ba23e 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java
@@ -13,6 +13,7 @@ import com.yahoo.config.provision.ClusterSpec;
import com.yahoo.config.provision.Environment;
import com.yahoo.config.provision.RegionName;
import com.yahoo.config.provision.TenantName;
+import com.yahoo.config.provision.zone.RoutingMethod;
import com.yahoo.config.provision.zone.ZoneId;
import com.yahoo.vespa.athenz.api.AthenzDomain;
import com.yahoo.vespa.athenz.api.AthenzIdentity;
@@ -58,6 +59,7 @@ import com.yahoo.vespa.hosted.controller.deployment.DeploymentContext;
import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester;
import com.yahoo.vespa.hosted.controller.deployment.DeploymentTrigger;
import com.yahoo.vespa.hosted.controller.integration.ConfigServerMock;
+import com.yahoo.vespa.hosted.controller.integration.ZoneApiMock;
import com.yahoo.vespa.hosted.controller.maintenance.JobControl;
import com.yahoo.vespa.hosted.controller.maintenance.RotationStatusUpdater;
import com.yahoo.vespa.hosted.controller.metric.ApplicationMetrics;
@@ -82,6 +84,7 @@ import java.time.temporal.ChronoUnit;
import java.util.ArrayList;
import java.util.Base64;
import java.util.Collections;
+import java.util.EnumSet;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -1475,14 +1478,17 @@ public class ApplicationApiTest extends ControllerContainerTest {
@Test
public void applicationWithRoutingPolicy() {
var app = deploymentTester.newDeploymentContext(createTenantAndApplication());
+ var zone = ZoneId.from(Environment.prod, RegionName.from("us-west-1"));
+ deploymentTester.controllerTester().zoneRegistry().setRoutingMethod(ZoneApiMock.from(zone),
+ EnumSet.of(RoutingMethod.exclusive, RoutingMethod.shared));
ApplicationPackage applicationPackage = new ApplicationPackageBuilder()
.environment(Environment.prod)
.instances("instance1")
- .region("us-west-1")
+ .region(zone.region().value())
.build();
app.submit(applicationPackage).deploy();
- app.addRoutingPolicy(ZoneId.from(Environment.prod, RegionName.from("us-west-1")), true);
- app.addRoutingPolicy(ZoneId.from(Environment.prod, RegionName.from("us-west-1")), false);
+ app.addRoutingPolicy(zone, true);
+ app.addRoutingPolicy(zone, false);
// GET application
tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1", GET)
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/cost/CostApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/cost/CostApiTest.java
deleted file mode 100644
index f992a54a114..00000000000
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/cost/CostApiTest.java
+++ /dev/null
@@ -1,49 +0,0 @@
-// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.hosted.controller.restapi.cost;
-
-import com.yahoo.application.container.handler.Request;
-import com.yahoo.config.provision.CloudName;
-import com.yahoo.config.provision.SystemName;
-import com.yahoo.config.provision.zone.ZoneApi;
-import com.yahoo.vespa.athenz.api.AthenzIdentity;
-import com.yahoo.vespa.athenz.api.AthenzUser;
-import com.yahoo.vespa.hosted.controller.integration.ZoneApiMock;
-import com.yahoo.vespa.hosted.controller.restapi.ContainerTester;
-import com.yahoo.vespa.hosted.controller.restapi.ControllerContainerTest;
-import org.junit.Before;
-import org.junit.Test;
-
-/**
- * @author andreer
- */
-public class CostApiTest extends ControllerContainerTest {
-
- private static final String responses = "src/test/java/com/yahoo/vespa/hosted/controller/restapi/cost/responses/";
- private static final AthenzIdentity operator = AthenzUser.fromUserId("operatorUser");
- private static final CloudName cloud1 = CloudName.from("yahoo");
- private static final CloudName cloud2 = CloudName.from("cloud2");
- private static final ZoneApi zone1 = ZoneApiMock.newBuilder().withId("prod.us-east-3").with(cloud1).build();
- private static final ZoneApi zone2 = ZoneApiMock.newBuilder().withId("prod.us-west-1").with(cloud1).build();
- private static final ZoneApi zone3 = ZoneApiMock.newBuilder().withId("prod.eu-west-1").with(cloud2).build();
-
- private ContainerTester tester;
-
- @Before
- public void before() {
- tester = new ContainerTester(container, responses);
- tester.serviceRegistry().zoneRegistry().setSystemName(SystemName.cd)
- .setZones(zone1, zone2, zone3);
- }
-
- @Test
- public void test_api() {
- assertResponse(new Request("http://localhost:8080/cost/v1/csv"),
- "Date,Property,Reserved Cpu Cores,Reserved Memory GB,Reserved Disk Space GB,Usage Fraction\n", 200);
- }
-
- private void assertResponse(Request request, String body, int statusCode) {
- addIdentityToRequest(request, operator);
- tester.assertResponse(request, body, statusCode);
- }
-
-}
diff --git a/default_build_settings.cmake b/default_build_settings.cmake
index e29e4c32017..a23a8f1489a 100644
--- a/default_build_settings.cmake
+++ b/default_build_settings.cmake
@@ -80,11 +80,17 @@ function(setup_vespa_default_build_settings_fedora_32)
set(DEFAULT_VESPA_LLVM_VERSION "10" PARENT_SCOPE)
endfunction()
-function(setup_vespa_default_build_settings_ubuntu_18_10)
- message("-- Setting up default build settings for ubuntu 18.10")
- set(DEFAULT_EXTRA_LINK_DIRECTORY "${VESPA_DEPS}/lib" "/usr/lib/llvm-6.0/lib" PARENT_SCOPE)
- set(DEFAULT_EXTRA_INCLUDE_DIRECTORY "${VESPA_DEPS}/include" "/usr/lib/llvm-6.0/include" PARENT_SCOPE)
- set(DEFAULT_VESPA_LLVM_VERSION "6.0" PARENT_SCOPE)
+function(setup_vespa_default_build_settings_fedora_33)
+ message("-- Setting up default build settings for fedora 33")
+ set(DEFAULT_EXTRA_INCLUDE_DIRECTORY "${VESPA_DEPS}/include" "/usr/include/openblas" PARENT_SCOPE)
+ set(DEFAULT_VESPA_LLVM_VERSION "10" PARENT_SCOPE)
+endfunction()
+
+function(setup_vespa_default_build_settings_ubuntu_19_10)
+ message("-- Setting up default build settings for ubuntu 19.10")
+ set(DEFAULT_EXTRA_LINK_DIRECTORY "${VESPA_DEPS}/lib" "/usr/lib/llvm-9/lib" PARENT_SCOPE)
+ set(DEFAULT_EXTRA_INCLUDE_DIRECTORY "${VESPA_DEPS}/include" "/usr/lib/llvm-9/include" PARENT_SCOPE)
+ set(DEFAULT_VESPA_LLVM_VERSION "9" PARENT_SCOPE)
endfunction()
function(vespa_use_default_vespa_unprivileged)
@@ -173,8 +179,10 @@ function(vespa_use_default_build_settings)
setup_vespa_default_build_settings_fedora_31()
elseif(VESPA_OS_DISTRO_COMBINED STREQUAL "fedora 32")
setup_vespa_default_build_settings_fedora_32()
- elseif(VESPA_OS_DISTRO_COMBINED STREQUAL "ubuntu 18.10")
- setup_vespa_default_build_settings_ubuntu_18_10()
+ elseif(VESPA_OS_DISTRO_COMBINED STREQUAL "fedora 33")
+ setup_vespa_default_build_settings_fedora_33()
+ elseif(VESPA_OS_DISTRO_COMBINED STREQUAL "ubuntu 19.10")
+ setup_vespa_default_build_settings_ubuntu_19_10()
else()
message(FATAL_ERROR "-- Unkonwn vespa build platform ${VESPA_OS_DISTRO_COMBINED}")
endif()
diff --git a/dist/vespa.spec b/dist/vespa.spec
index c54e4442167..8ce494fd7cb 100644
--- a/dist/vespa.spec
+++ b/dist/vespa.spec
@@ -60,27 +60,37 @@ BuildRequires: vespa-protobuf-devel >= 3.7.0-4
%if 0%{?fedora}
BuildRequires: cmake >= 3.9.1
BuildRequires: maven
-BuildRequires: vespa-protobuf-devel >= 3.7.0-4
BuildRequires: openssl-devel
%if 0%{?fc29}
+BuildRequires: vespa-protobuf-devel >= 3.7.0-4
BuildRequires: llvm-devel >= 7.0.0
BuildRequires: boost-devel >= 1.66
BuildRequires: gtest-devel
BuildRequires: gmock-devel
%endif
%if 0%{?fc30}
+BuildRequires: vespa-protobuf-devel >= 3.7.0-4
BuildRequires: llvm-devel >= 8.0.0
BuildRequires: boost-devel >= 1.69
BuildRequires: gtest-devel
BuildRequires: gmock-devel
%endif
%if 0%{?fc31}
+BuildRequires: vespa-protobuf-devel >= 3.7.0-4
BuildRequires: llvm-devel >= 9.0.0
BuildRequires: boost-devel >= 1.69
BuildRequires: gtest-devel
BuildRequires: gmock-devel
%endif
%if 0%{?fc32}
+BuildRequires: protobuf-devel
+BuildRequires: llvm-devel >= 10.0.0
+BuildRequires: boost-devel >= 1.69
+BuildRequires: gtest-devel
+BuildRequires: gmock-devel
+%endif
+%if 0%{?fc33}
+BuildRequires: protobuf-devel
BuildRequires: llvm-devel >= 10.0.0
BuildRequires: boost-devel >= 1.69
BuildRequires: gtest-devel
@@ -159,21 +169,29 @@ Requires: openssl-libs
%define _extra_include_directory %{_vespa_deps_prefix}/include;/usr/include/openblas
%endif
%if 0%{?fedora}
-Requires: vespa-protobuf >= 3.7.0-4
Requires: openssl-libs
%if 0%{?fc29}
+Requires: vespa-protobuf >= 3.7.0-4
Requires: llvm-libs >= 7.0.0
%define _vespa_llvm_version 7
%endif
%if 0%{?fc30}
+Requires: vespa-protobuf >= 3.7.0-4
Requires: llvm-libs >= 8.0.0
%define _vespa_llvm_version 8
%endif
%if 0%{?fc31}
+Requires: vespa-protobuf >= 3.7.0-4
Requires: llvm-libs >= 9.0.0
%define _vespa_llvm_version 9
%endif
%if 0%{?fc32}
+Requires: protobuf
+Requires: llvm-libs >= 10.0.0
+%define _vespa_llvm_version 10
+%endif
+%if 0%{?fc33}
+Requires: protobuf
Requires: llvm-libs >= 10.0.0
%define _vespa_llvm_version 10
%endif
diff --git a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/CreateContainerCommandImpl.java b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/CreateContainerCommandImpl.java
index 32302a98757..ecfe9b2468a 100644
--- a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/CreateContainerCommandImpl.java
+++ b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/CreateContainerCommandImpl.java
@@ -34,6 +34,7 @@ class CreateContainerCommandImpl implements Docker.CreateContainerCommand {
private final Map<String, String> labels = new HashMap<>();
private final List<String> environmentAssignments = new ArrayList<>();
private final List<String> volumeBindSpecs = new ArrayList<>();
+ private final List<String> dnsOptions = new ArrayList<>();
private final List<Ulimit> ulimits = new ArrayList<>();
private final Set<Capability> addCapabilities = new HashSet<>();
private final Set<Capability> dropCapabilities = new HashSet<>();
@@ -96,6 +97,12 @@ class CreateContainerCommandImpl implements Docker.CreateContainerCommand {
}
@Override
+ public Docker.CreateContainerCommand withDnsOption(String dnsOption) {
+ dnsOptions.add(dnsOption);
+ return this;
+ }
+
+ @Override
public Docker.CreateContainerCommand withPrivileged(boolean privileged) {
this.privileged = privileged;
return this;
@@ -171,6 +178,7 @@ class CreateContainerCommandImpl implements Docker.CreateContainerCommand {
.withPidsLimit(-1L)
.withCapAdd(addCapabilities.toArray(new Capability[0]))
.withCapDrop(dropCapabilities.toArray(new Capability[0]))
+ .withDnsOptions(dnsOptions)
.withPrivileged(privileged);
containerResources.ifPresent(cr -> hostConfig
@@ -241,6 +249,7 @@ class CreateContainerCommandImpl implements Docker.CreateContainerCommand {
toRepeatedOption("--cap-add", addCapabilitiesList),
toRepeatedOption("--cap-drop", dropCapabilitiesList),
toRepeatedOption("--security-opt", securityOpts),
+ toRepeatedOption("--dns-option", dnsOptions),
toOptionalOption("--net", networkMode),
toOptionalOption("--ip", ipv4Address),
toOptionalOption("--ip6", ipv6Address),
diff --git a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/Docker.java b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/Docker.java
index 4e7ef5a1ff6..648c94d71ab 100644
--- a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/Docker.java
+++ b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/Docker.java
@@ -53,6 +53,7 @@ public interface Docker {
CreateContainerCommand withAddCapability(String capabilityName);
CreateContainerCommand withDropCapability(String capabilityName);
CreateContainerCommand withSecurityOpts(String securityOpt);
+ CreateContainerCommand withDnsOption(String dnsOption);
CreateContainerCommand withPrivileged(boolean privileged);
void create();
diff --git a/eval/src/vespa/eval/eval/interpreted_function.cpp b/eval/src/vespa/eval/eval/interpreted_function.cpp
index 0a630a3e20a..ec28604fd87 100644
--- a/eval/src/vespa/eval/eval/interpreted_function.cpp
+++ b/eval/src/vespa/eval/eval/interpreted_function.cpp
@@ -3,21 +3,16 @@
#include "interpreted_function.h"
#include "node_visitor.h"
#include "node_traverser.h"
-#include "check_type.h"
-#include "tensor_spec.h"
-#include "operation.h"
#include "tensor_nodes.h"
#include "tensor_engine.h"
+#include "make_tensor_function.h"
+#include "compile_tensor_function.h"
#include <vespa/vespalib/util/classname.h>
#include <vespa/eval/eval/llvm/compile_cache.h>
#include <vespa/vespalib/util/benchmark_timer.h>
#include <set>
-#include "make_tensor_function.h"
-#include "compile_tensor_function.h"
-
-namespace vespalib {
-namespace eval {
+namespace vespalib::eval {
namespace {
@@ -42,11 +37,12 @@ InterpretedFunction::State::State(const TensorEngine &engine_in)
params(nullptr),
stash(),
stack(),
- program_offset(0)
+ program_offset(0),
+ if_cnt(0)
{
}
-InterpretedFunction::State::~State() {}
+InterpretedFunction::State::~State() = default;
void
InterpretedFunction::State::init(const LazyParams &params_in) {
@@ -82,7 +78,7 @@ InterpretedFunction::InterpretedFunction(const TensorEngine &engine, const nodes
_program = compile_tensor_function(optimized, _stash);
}
-InterpretedFunction::~InterpretedFunction() {}
+InterpretedFunction::~InterpretedFunction() = default;
const Value &
InterpretedFunction::eval(Context &ctx, const LazyParams &params) const
@@ -123,5 +119,4 @@ InterpretedFunction::detect_issues(const Function &function)
return Function::Issues(std::move(checker.issues));
}
-} // namespace vespalib::eval
-} // namespace vespalib
+}
diff --git a/eval/src/vespa/eval/eval/interpreted_function.h b/eval/src/vespa/eval/eval/interpreted_function.h
index e3e8d18b44f..e638ccffcea 100644
--- a/eval/src/vespa/eval/eval/interpreted_function.h
+++ b/eval/src/vespa/eval/eval/interpreted_function.h
@@ -7,8 +7,7 @@
#include "lazy_params.h"
#include <vespa/vespalib/util/stash.h>
-namespace vespalib {
-namespace eval {
+namespace vespalib::eval {
namespace nodes { struct Node; }
struct TensorEngine;
@@ -107,5 +106,4 @@ public:
static Function::Issues detect_issues(const Function &function);
};
-} // namespace vespalib::eval
-} // namespace vespalib
+}
diff --git a/eval/src/vespa/eval/eval/lazy_params.cpp b/eval/src/vespa/eval/eval/lazy_params.cpp
index aec8cf78059..2c00c4c312b 100644
--- a/eval/src/vespa/eval/eval/lazy_params.cpp
+++ b/eval/src/vespa/eval/eval/lazy_params.cpp
@@ -1,19 +1,16 @@
// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
#include "lazy_params.h"
-#include <assert.h>
+#include <vespa/vespalib/util/stash.h>
+#include <cassert>
namespace vespalib::eval {
-LazyParams::~LazyParams()
-{
-}
+LazyParams::~LazyParams() = default;
//-----------------------------------------------------------------------------
-SimpleObjectParams::~SimpleObjectParams()
-{
-}
+SimpleObjectParams::~SimpleObjectParams() = default;
const Value &
SimpleObjectParams::resolve(size_t idx, Stash &) const
@@ -24,9 +21,7 @@ SimpleObjectParams::resolve(size_t idx, Stash &) const
//-----------------------------------------------------------------------------
-SimpleParams::~SimpleParams()
-{
-}
+SimpleParams::~SimpleParams() = default;
const Value &
SimpleParams::resolve(size_t idx, Stash &stash) const
diff --git a/eval/src/vespa/eval/eval/tensor_function.h b/eval/src/vespa/eval/eval/tensor_function.h
index 4b862e9ec6a..55d27fb74ea 100644
--- a/eval/src/vespa/eval/eval/tensor_function.h
+++ b/eval/src/vespa/eval/eval/tensor_function.h
@@ -2,20 +2,17 @@
#pragma once
-#include <memory>
-#include <vector>
-#include <variant>
-#include <vespa/vespalib/stllike/asciistream.h>
-#include <vespa/vespalib/stllike/string.h>
-#include <vespa/vespalib/util/arrayref.h>
-#include <vespa/vespalib/util/overload.h>
#include "tensor_spec.h"
#include "lazy_params.h"
#include "value_type.h"
#include "value.h"
#include "aggr.h"
-
#include "interpreted_function.h"
+#include <vespa/vespalib/stllike/asciistream.h>
+#include <vespa/vespalib/stllike/string.h>
+#include <vespa/vespalib/util/arrayref.h>
+#include <vespa/vespalib/util/overload.h>
+#include <variant>
namespace vespalib {
diff --git a/eval/src/vespa/eval/eval/value.h b/eval/src/vespa/eval/eval/value.h
index 6701173bcd3..cad76c93c5c 100644
--- a/eval/src/vespa/eval/eval/value.h
+++ b/eval/src/vespa/eval/eval/value.h
@@ -2,13 +2,11 @@
#pragma once
-#include <vespa/vespalib/stllike/string.h>
-#include <memory>
-#include <vespa/vespalib/util/stash.h>
#include "value_type.h"
+#include <vespa/vespalib/util/traits.h>
+#include <memory>
-namespace vespalib {
-namespace eval {
+namespace vespalib::eval {
class Tensor;
@@ -40,7 +38,6 @@ public:
static const ValueType &double_type() { return _type; }
};
-} // namespace vespalib::eval
-} // namespace vespalib
+}
VESPA_CAN_SKIP_DESTRUCTION(::vespalib::eval::DoubleValue);
diff --git a/fbench/CMakeLists.txt b/fbench/CMakeLists.txt
index 5cc56786227..3da632d98a6 100644
--- a/fbench/CMakeLists.txt
+++ b/fbench/CMakeLists.txt
@@ -15,6 +15,7 @@ vespa_define_module(
TESTS
src/test
+ src/test/authority
)
vespa_install_script(util/resultfilter.pl vespa-fbench-result-filter.pl bin)
diff --git a/fbench/src/fbench/fbench.cpp b/fbench/src/fbench/fbench.cpp
index 91475ce2125..593ae30a0e5 100644
--- a/fbench/src/fbench/fbench.cpp
+++ b/fbench/src/fbench/fbench.cpp
@@ -86,10 +86,13 @@ FBench::init_crypto_engine(const std::string &ca_certs_file_name,
return false;
}
bool load_failed = false;
- vespalib::net::tls::TransportSecurityOptions
- tls_opts(maybe_load(ca_certs_file_name, load_failed),
- maybe_load(cert_chain_file_name, load_failed),
- maybe_load(private_key_file_name, load_failed));
+ auto ts_builder = vespalib::net::tls::TransportSecurityOptions::Params().
+ ca_certs_pem(maybe_load(ca_certs_file_name, load_failed)).
+ cert_chain_pem(maybe_load(cert_chain_file_name, load_failed)).
+ private_key_pem(maybe_load(private_key_file_name, load_failed)).
+ authorized_peers(vespalib::net::tls::AuthorizedPeers::allow_all_authenticated()).
+ disable_hostname_validation(true); // TODO configurable or default false!
+ vespalib::net::tls::TransportSecurityOptions tls_opts(std::move(ts_builder));
if (load_failed) {
fprintf(stderr, "failed to load transport security options\n");
return false;
diff --git a/fbench/src/httpclient/CMakeLists.txt b/fbench/src/httpclient/CMakeLists.txt
index 5f3333128b3..a28f3666383 100644
--- a/fbench/src/httpclient/CMakeLists.txt
+++ b/fbench/src/httpclient/CMakeLists.txt
@@ -3,5 +3,6 @@ vespa_add_library(fbench_httpclient STATIC
SOURCES
httpclient.cpp
DEPENDS
+ fbench_util
fastos
)
diff --git a/fbench/src/httpclient/httpclient.cpp b/fbench/src/httpclient/httpclient.cpp
index 99134a6e297..9615a6e6df7 100644
--- a/fbench/src/httpclient/httpclient.cpp
+++ b/fbench/src/httpclient/httpclient.cpp
@@ -1,6 +1,7 @@
// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
#include "httpclient.h"
#include <vespa/vespalib/net/socket_spec.h>
+#include <util/authority.h>
#include <cassert>
#include <cstring>
@@ -29,7 +30,8 @@ HTTPClient::HTTPClient(vespalib::CryptoEngine::SP engine, const char *hostname,
_keepAlive(keepAlive),
_headerBenchmarkdataCoverage(headerBenchmarkdataCoverage),
_extraHeaders(extraHeaders),
- _authority(authority),
+ _sni_spec(make_sni_spec(authority, hostname, port, _engine->use_tls_when_client())),
+ _host_header_value(make_host_header_value(_sni_spec, _engine->use_tls_when_client())),
_reuseCount(0),
_bufsize(10240),
_buf(new char[_bufsize]),
@@ -51,11 +53,6 @@ HTTPClient::HTTPClient(vespalib::CryptoEngine::SP engine, const char *hostname,
_dataDone(false),
_reader(NULL)
{
- if (_authority == "") {
- char tmp[1024];
- snprintf(tmp, 1024, "%s:%d", hostname, port);
- _authority = tmp;
- }
}
bool
@@ -70,8 +67,7 @@ HTTPClient::connect_socket()
if (!handle.valid()) {
return false;
}
- _socket = vespalib::SyncCryptoSocket::create_client(*_engine, std::move(handle),
- vespalib::SocketSpec::from_host_port(_hostname, _port));
+ _socket = vespalib::SyncCryptoSocket::create_client(*_engine, std::move(handle), _sni_spec);
return bool(_socket);
}
@@ -153,14 +149,14 @@ HTTPClient::Connect(const char *url, bool usePost, const char *content, int cLen
"Content-Length: %d\r\n"
"%s"
"\r\n",
- url, _authority.c_str(), cLen, headers.c_str());
+ url, _host_header_value.c_str(), cLen, headers.c_str());
} else {
snprintf(req, req_max,
"GET %s HTTP/1.1\r\n"
"Host: %s\r\n"
"%s"
"\r\n",
- url, _authority.c_str(), headers.c_str());
+ url, _host_header_value.c_str(), headers.c_str());
}
// try to reuse connection if keep-alive is enabled
diff --git a/fbench/src/httpclient/httpclient.h b/fbench/src/httpclient/httpclient.h
index 9c3ccd437d1..cad01826db7 100644
--- a/fbench/src/httpclient/httpclient.h
+++ b/fbench/src/httpclient/httpclient.h
@@ -6,6 +6,7 @@
#include <vespa/vespalib/net/sync_crypto_socket.h>
#include <vespa/vespalib/net/crypto_engine.h>
#include <vespa/vespalib/net/socket_address.h>
+#include <vespa/vespalib/net/socket_spec.h>
/**
* This class implements a HTTP client that may be used to fetch
@@ -99,7 +100,8 @@ protected:
bool _keepAlive;
bool _headerBenchmarkdataCoverage;
std::string _extraHeaders;
- std::string _authority;
+ vespalib::SocketSpec _sni_spec;
+ std::string _host_header_value;
uint64_t _reuseCount;
size_t _bufsize;
diff --git a/fbench/src/test/authority/CMakeLists.txt b/fbench/src/test/authority/CMakeLists.txt
new file mode 100644
index 00000000000..00f804f43f6
--- /dev/null
+++ b/fbench/src/test/authority/CMakeLists.txt
@@ -0,0 +1,10 @@
+# Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_executable(fbench_authority_test_app TEST
+ SOURCES
+ authority_test.cpp
+ DEPENDS
+ fbench_util
+ vespalib
+ gtest
+)
+vespa_add_test(NAME fbench_authority_test_app COMMAND fbench_authority_test_app)
diff --git a/fbench/src/test/authority/authority_test.cpp b/fbench/src/test/authority/authority_test.cpp
new file mode 100644
index 00000000000..de723a8730f
--- /dev/null
+++ b/fbench/src/test/authority/authority_test.cpp
@@ -0,0 +1,87 @@
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <util/authority.h>
+#include <vespa/vespalib/gtest/gtest.h>
+
+using vespalib::SocketSpec;
+
+//-----------------------------------------------------------------------------
+
+TEST(MakeSNISpecTest, host_port_is_parsed_as_expected) {
+ EXPECT_EQ(make_sni_spec("my_host:123", "fallback", 456, false).host(), "my_host");
+ EXPECT_EQ(make_sni_spec("my_host:123", "fallback", 456, true).host(), "my_host");
+ EXPECT_EQ(make_sni_spec("my_host:123", "fallback", 456, false).port(), 123);
+ EXPECT_EQ(make_sni_spec("my_host:123", "fallback", 456, true).port(), 123);
+}
+
+TEST(MakeSNISpecTest, user_info_is_stripped) {
+ EXPECT_EQ(make_sni_spec("myuser:deprecated@my_host:123", "fallback", 456, false).host(), "my_host");
+ EXPECT_EQ(make_sni_spec("myuser:deprecated@my_host:123", "fallback", 456, true).host(), "my_host");
+ EXPECT_EQ(make_sni_spec("myuser:deprecated@my_host:123", "fallback", 456, false).port(), 123);
+ EXPECT_EQ(make_sni_spec("myuser:deprecated@my_host:123", "fallback", 456, true).port(), 123);
+}
+
+TEST(MakeSNISpecTest, port_can_be_skipped) {
+ EXPECT_EQ(make_sni_spec("my_host", "fallback", 456, false).host(), "my_host");
+ EXPECT_EQ(make_sni_spec("my_host", "fallback", 456, true).host(), "my_host");
+ EXPECT_EQ(make_sni_spec("my_host", "fallback", 456, false).port(), 80);
+ EXPECT_EQ(make_sni_spec("my_host", "fallback", 456, true).port(), 443);
+}
+
+TEST(MakeSNISpecTest, quoted_ip_addresses_work_as_expected) {
+ EXPECT_EQ(make_sni_spec("[::1]:123", "fallback", 456, false).host(), "::1");
+ EXPECT_EQ(make_sni_spec("[::1]:123", "fallback", 456, true).host(), "::1");
+ EXPECT_EQ(make_sni_spec("[::1]:123", "fallback", 456, false).port(), 123);
+ EXPECT_EQ(make_sni_spec("[::1]:123", "fallback", 456, true).port(), 123);
+ EXPECT_EQ(make_sni_spec("[::1]", "fallback", 456, false).host(), "::1");
+ EXPECT_EQ(make_sni_spec("[::1]", "fallback", 456, true).host(), "::1");
+ EXPECT_EQ(make_sni_spec("[::1]", "fallback", 456, false).port(), 80);
+ EXPECT_EQ(make_sni_spec("[::1]", "fallback", 456, true).port(), 443);
+}
+
+TEST(MakeSNISpecTest, supplied_host_port_is_used_as_fallback) {
+ EXPECT_EQ(make_sni_spec("", "fallback", 456, false).host(), "fallback");
+ EXPECT_EQ(make_sni_spec("", "fallback", 456, true).host(), "fallback");
+ EXPECT_EQ(make_sni_spec("", "fallback", 456, false).port(), 456);
+ EXPECT_EQ(make_sni_spec("", "fallback", 456, true).port(), 456);
+}
+
+//-----------------------------------------------------------------------------
+
+TEST(MakeHostHeaderValueTest, host_port_is_formatted_as_expected) {
+ auto my_spec = SocketSpec::from_host_port("myhost", 123);
+ EXPECT_EQ(make_host_header_value(my_spec, false), "myhost:123");
+ EXPECT_EQ(make_host_header_value(my_spec, true), "myhost:123");
+}
+
+TEST(MakeHostHeaderValueTest, inappropriate_spec_gives_empty_host_value) {
+ std::vector<SocketSpec> bad_specs = {
+ SocketSpec::invalid,
+ SocketSpec::from_port(123),
+ SocketSpec::from_name("foo"),
+ SocketSpec::from_path("bar")
+ };
+ for (const auto &spec: bad_specs) {
+ EXPECT_EQ(make_host_header_value(spec, false), "");
+ EXPECT_EQ(make_host_header_value(spec, true), "");
+ }
+}
+
+TEST(MakeHostHeaderValueTest, default_port_is_omitted) {
+ auto spec1 = SocketSpec::from_host_port("myhost", 80);
+ auto spec2 = SocketSpec::from_host_port("myhost", 443);
+ EXPECT_EQ(make_host_header_value(spec1, false), "myhost");
+ EXPECT_EQ(make_host_header_value(spec1, true), "myhost:80");
+ EXPECT_EQ(make_host_header_value(spec2, false), "myhost:443");
+ EXPECT_EQ(make_host_header_value(spec2, true), "myhost");
+}
+
+TEST(MakeHostHeaderValueTest, ipv6_addresses_are_quoted) {
+ auto my_spec = SocketSpec::from_host_port("::1", 123);
+ EXPECT_EQ(make_host_header_value(my_spec, false), "[::1]:123");
+ EXPECT_EQ(make_host_header_value(my_spec, true), "[::1]:123");
+}
+
+//-----------------------------------------------------------------------------
+
+GTEST_MAIN_RUN_ALL_TESTS()
diff --git a/fbench/src/util/CMakeLists.txt b/fbench/src/util/CMakeLists.txt
index 47cc46ffc8f..3cdff26ce16 100644
--- a/fbench/src/util/CMakeLists.txt
+++ b/fbench/src/util/CMakeLists.txt
@@ -1,8 +1,9 @@
# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
vespa_add_library(fbench_util STATIC
SOURCES
+ authority.cpp
+ clientstatus.cpp
filereader.cpp
timer.cpp
- clientstatus.cpp
DEPENDS
)
diff --git a/fbench/src/util/authority.cpp b/fbench/src/util/authority.cpp
new file mode 100644
index 00000000000..6247c72d9b0
--- /dev/null
+++ b/fbench/src/util/authority.cpp
@@ -0,0 +1,42 @@
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include "authority.h"
+#include <vespa/vespalib/util/stringfmt.h>
+#include <cassert>
+
+namespace {
+
+int default_port(bool use_https) { return use_https ? 443 : 80; }
+
+}
+
+vespalib::SocketSpec make_sni_spec(const std::string &authority, const char *hostname, int port, bool use_https) {
+ if (authority.empty()) {
+ return vespalib::SocketSpec::from_host_port(hostname, port);
+ }
+ auto split = authority.rfind('@');
+ std::string spec_str = (split == std::string::npos) ? authority : authority.substr(split + 1);
+ auto a = spec_str.rfind(':');
+ auto b = spec_str.rfind(']');
+ bool has_port = (a != std::string::npos) && ((b == std::string::npos) || (a > b));
+ if (has_port) {
+ spec_str = "tcp/" + spec_str;
+ } else {
+ spec_str = vespalib::make_string("tcp/%s:%d", spec_str.c_str(), default_port(use_https));
+ }
+ // use SocketSpec parser to ensure ipv6 addresses are dequoted
+ return vespalib::SocketSpec(spec_str);
+}
+
+std::string make_host_header_value(const vespalib::SocketSpec &sni_spec, bool use_https) {
+ if (sni_spec.host().empty()) {
+ return "";
+ }
+ if (sni_spec.port() == default_port(use_https)) {
+ return sni_spec.host();
+ }
+ // use SocketSpec formatter to ensure ipv6 addresses are quoted
+ std::string spec_str = sni_spec.spec();
+ assert(spec_str.find("tcp/") == 0);
+ return spec_str.substr(4);
+}
diff --git a/fbench/src/util/authority.h b/fbench/src/util/authority.h
new file mode 100644
index 00000000000..49dab4a29fd
--- /dev/null
+++ b/fbench/src/util/authority.h
@@ -0,0 +1,30 @@
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <vespa/vespalib/net/socket_spec.h>
+
+/**
+ * Assemble an SNI (Server Name Indication) spec that will be used
+ * when handshaking over TLS. The authority will be used if
+ * non-empty. Hostname/port will be used as fall-back. Note that the
+ * SNI spec will also be used to generate the Host header used in
+ * subsequent HTTP requests.
+ *
+ * @return sni spec
+ * @param authority user-provided authority
+ * @param hostname name of the host we are connecting to
+ * @param port which port we are connecting to
+ * @param use_https are we using https? (TLS)
+ **/
+vespalib::SocketSpec make_sni_spec(const std::string &authority, const char *hostname, int port, bool use_https);
+
+/**
+ * Use an SNI spec to generate a matching Host header to be used in
+ * HTTP requests. Note that default port numbers will be omitted.
+ *
+ * @return host header value
+ * @param sni_spec SNI spec
+ * @param use_https are we using https? (TLS)
+ **/
+std::string make_host_header_value(const vespalib::SocketSpec &sni_spec, bool use_https);
diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java
index fc54c61fca8..bb4e281a85f 100644
--- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java
+++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java
@@ -74,7 +74,7 @@ public class Flags {
HOSTNAME, NODE_TYPE);
public static final UnboundStringFlag DOCKER_VERSION = defineStringFlag(
- "docker-version", "1.13.1-91.git07f3374",
+ "docker-version", "1.13.1-102.git7f2769b",
"The version of the docker to use of the format VERSION-REL: The YUM package to be installed will be " +
"2:docker-VERSION-REL.el7.centos.x86_64 in AWS (and without '.centos' otherwise). " +
"If docker-version is not of this format, it must be parseable by YumPackageName::fromString.",
@@ -126,14 +126,6 @@ public class Flags {
"scheduled evenly distributed in the 1x-2x range (and naturally guaranteed at the 2x boundary).",
"Takes effect on next run of NodeRebooter");
- public static final UnboundBooleanFlag ENABLE_LARGE_ORCHESTRATOR_LOCKS = defineFeatureFlag(
- "enable-large-orchestrator-locks", true,
- "If enabled, the orchestrator will accumulate application locks during probe in batch suspension, " +
- "and release them in reverse order only after the non-probe is complete. Can be set depending on " +
- "parent hostname.",
- "Takes immediate effect for new batch suspensions.",
- HOSTNAME);
-
public static final UnboundBooleanFlag RETIRE_WITH_PERMANENTLY_DOWN = defineFeatureFlag(
"retire-with-permanently-down", false,
"If enabled, retirement will end with setting the host status to PERMANENTLY_DOWN, " +
diff --git a/fnet/src/tests/connect/connect_test.cpp b/fnet/src/tests/connect/connect_test.cpp
index d94b6759077..62000efb682 100644
--- a/fnet/src/tests/connect/connect_test.cpp
+++ b/fnet/src/tests/connect/connect_test.cpp
@@ -65,6 +65,8 @@ struct BlockingCryptoEngine : public CryptoEngine {
Gate handshake_work_enter;
Gate handshake_work_exit;
Gate handshake_socket_deleted;
+ bool use_tls_when_client() const override { return false; }
+ bool always_use_tls_when_server() const override { return false; }
CryptoSocket::UP create_client_crypto_socket(SocketHandle socket, const SocketSpec &) override {
return std::make_unique<BlockingCryptoSocket>(std::move(socket),
handshake_work_enter, handshake_work_exit, handshake_socket_deleted);
diff --git a/fnet/src/tests/frt/values/values_test.cpp b/fnet/src/tests/frt/values/values_test.cpp
index 5bc6b0e2dce..3b36e8989c1 100644
--- a/fnet/src/tests/frt/values/values_test.cpp
+++ b/fnet/src/tests/frt/values/values_test.cpp
@@ -3,6 +3,7 @@
#include <vespa/fnet/frt/values.h>
#include <vespa/fnet/databuffer.h>
#include <vespa/fnet/info.h>
+#include <vespa/vespalib/util/stash.h>
using vespalib::Stash;
diff --git a/fnet/src/vespa/fnet/frt/rpcrequest.h b/fnet/src/vespa/fnet/frt/rpcrequest.h
index cc871e7ac0c..eaa34a46b7a 100644
--- a/fnet/src/vespa/fnet/frt/rpcrequest.h
+++ b/fnet/src/vespa/fnet/frt/rpcrequest.h
@@ -5,6 +5,7 @@
#include "values.h"
#include "error.h"
#include <vespa/fnet/context.h>
+#include <vespa/vespalib/util/stash.h>
#include <atomic>
class FNETConnection;
diff --git a/fnet/src/vespa/fnet/frt/values.cpp b/fnet/src/vespa/fnet/frt/values.cpp
index a5f59df19b2..3b37aa9a1bc 100644
--- a/fnet/src/vespa/fnet/frt/values.cpp
+++ b/fnet/src/vespa/fnet/frt/values.cpp
@@ -3,6 +3,7 @@
#include "values.h"
#include <vespa/fnet/databuffer.h>
#include <vespa/vespalib/util/stringfmt.h>
+#include <vespa/vespalib/util/stash.h>
#include <cassert>
static_assert(sizeof(uint8_t) == 1, "uint8_t must be 1 byte.");
@@ -81,7 +82,7 @@ FRT_Values::FRT_Values(Stash &stash)
_stash(stash)
{ }
-FRT_Values::~FRT_Values() { }
+FRT_Values::~FRT_Values() = default;
LocalBlob::LocalBlob(const char *data, uint32_t len) :
_data(Alloc::alloc(len)),
@@ -294,7 +295,7 @@ FRT_Values::AddSharedData(FRT_ISharedBlob *blob) {
}
void
-FRT_Values::AddData(vespalib::alloc::Alloc buf, uint32_t len) {
+FRT_Values::AddData(vespalib::alloc::Alloc && buf, uint32_t len) {
AddSharedData(&_stash.create<LocalBlob>(std::move(buf), len));
}
diff --git a/fnet/src/vespa/fnet/frt/values.h b/fnet/src/vespa/fnet/frt/values.h
index e00aec8423c..2aa7551c423 100644
--- a/fnet/src/vespa/fnet/frt/values.h
+++ b/fnet/src/vespa/fnet/frt/values.h
@@ -3,9 +3,10 @@
#pragma once
#include "isharedblob.h"
-#include <vespa/vespalib/util/stash.h>
#include <cstring>
+namespace vespalib { class Stash; }
+namespace vespalib::alloc { class Alloc; }
namespace fnet {
char * copyString(char *dst, const char *src, size_t len);
char * copyData(char *dst, const void *src, size_t len);
@@ -216,7 +217,7 @@ public:
char *AddString(uint32_t len);
FRT_StringValue *AddStringArray(uint32_t len);
void AddSharedData(FRT_ISharedBlob *blob);
- void AddData(Alloc buf, uint32_t len);
+ void AddData(Alloc && buf, uint32_t len);
void AddData(const char *buf, uint32_t len);
char *AddData(uint32_t len);
FRT_DataValue *AddDataArray(uint32_t len);
diff --git a/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/ssl/impl/TlsContextBasedProviderTest.java b/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/ssl/impl/TlsContextBasedProviderTest.java
index 88db5c99de9..eb292199ea2 100644
--- a/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/ssl/impl/TlsContextBasedProviderTest.java
+++ b/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/ssl/impl/TlsContextBasedProviderTest.java
@@ -5,6 +5,7 @@ import com.yahoo.security.KeyUtils;
import com.yahoo.security.X509CertificateBuilder;
import com.yahoo.security.tls.AuthorizationMode;
import com.yahoo.security.tls.DefaultTlsContext;
+import com.yahoo.security.tls.HostnameVerification;
import com.yahoo.security.tls.PeerAuthentication;
import com.yahoo.security.tls.TlsContext;
import com.yahoo.security.tls.policy.AuthorizedPeers;
@@ -51,7 +52,7 @@ public class TlsContextBasedProviderTest {
BigInteger.ONE)
.build();
return new DefaultTlsContext(
- List.of(certificate), keyPair.getPrivate(), List.of(certificate), new AuthorizedPeers(Set.of()), AuthorizationMode.ENFORCE, PeerAuthentication.NEED);
+ List.of(certificate), keyPair.getPrivate(), List.of(certificate), new AuthorizedPeers(Set.of()), AuthorizationMode.ENFORCE, PeerAuthentication.NEED, HostnameVerification.ENABLED);
}
private static class SimpleTlsContextBasedProvider extends TlsContextBasedProvider {
diff --git a/jrt/src/com/yahoo/jrt/TlsCryptoEngine.java b/jrt/src/com/yahoo/jrt/TlsCryptoEngine.java
index 7474220d4e7..a140e87713c 100644
--- a/jrt/src/com/yahoo/jrt/TlsCryptoEngine.java
+++ b/jrt/src/com/yahoo/jrt/TlsCryptoEngine.java
@@ -21,7 +21,8 @@ public class TlsCryptoEngine implements CryptoEngine {
@Override
public TlsCryptoSocket createClientCryptoSocket(SocketChannel channel, Spec spec) {
- SSLEngine sslEngine = tlsContext.createSslEngine();
+ String peerHost = spec.host() != null ? spec.host() : "localhost"; // Use localhost for wildcard address
+ SSLEngine sslEngine = tlsContext.createSslEngine(peerHost, spec.port());
sslEngine.setUseClientMode(true);
return new TlsCryptoSocket(channel, sslEngine);
}
diff --git a/jrt/tests/com/yahoo/jrt/CryptoUtils.java b/jrt/tests/com/yahoo/jrt/CryptoUtils.java
index e7e4eea568d..95ea581cb90 100644
--- a/jrt/tests/com/yahoo/jrt/CryptoUtils.java
+++ b/jrt/tests/com/yahoo/jrt/CryptoUtils.java
@@ -5,6 +5,7 @@ import com.yahoo.security.KeyUtils;
import com.yahoo.security.X509CertificateBuilder;
import com.yahoo.security.tls.AuthorizationMode;
import com.yahoo.security.tls.DefaultTlsContext;
+import com.yahoo.security.tls.HostnameVerification;
import com.yahoo.security.tls.PeerAuthentication;
import com.yahoo.security.tls.TlsContext;
import com.yahoo.security.tls.policy.AuthorizedPeers;
@@ -35,21 +36,23 @@ class CryptoUtils {
static final KeyPair keyPair = KeyUtils.generateKeypair(EC);
static final X509Certificate certificate = X509CertificateBuilder
- .fromKeypair(keyPair, new X500Principal("CN=dummy"), EPOCH, Instant.now().plus(1, DAYS), SHA256_WITH_ECDSA, generateRandomSerialNumber())
+ .fromKeypair(keyPair, new X500Principal("CN=localhost"), EPOCH, Instant.now().plus(1, DAYS), SHA256_WITH_ECDSA, generateRandomSerialNumber())
.build();
static final AuthorizedPeers authorizedPeers = new AuthorizedPeers(
singleton(
new PeerPolicy(
- "dummy-policy",
+ "localhost-policy",
singleton(
- new Role("dummy-role")),
+ new Role("localhost-role")),
singletonList(
new RequiredPeerCredential(
- Field.CN, new HostGlobPattern("dummy"))))));
+ Field.CN, new HostGlobPattern("localhost"))))));
static TlsContext createTestTlsContext() {
- return new DefaultTlsContext(singletonList(certificate), keyPair.getPrivate(), singletonList(certificate), authorizedPeers, AuthorizationMode.ENFORCE, PeerAuthentication.NEED);
+ return new DefaultTlsContext(
+ singletonList(certificate), keyPair.getPrivate(), singletonList(certificate), authorizedPeers,
+ AuthorizationMode.ENFORCE, PeerAuthentication.NEED, HostnameVerification.ENABLED);
}
}
diff --git a/metrics-proxy/pom.xml b/metrics-proxy/pom.xml
index f72ad75c6af..355f420c2a4 100644
--- a/metrics-proxy/pom.xml
+++ b/metrics-proxy/pom.xml
@@ -132,6 +132,10 @@
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpclient</artifactId>
</dependency>
+ <dependency>
+ <groupId>org.apache.velocity</groupId>
+ <artifactId>velocity</artifactId>
+ </dependency>
<!-- test scope -->
diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/telegraf/Telegraf.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/telegraf/Telegraf.java
new file mode 100644
index 00000000000..2afc0267434
--- /dev/null
+++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/telegraf/Telegraf.java
@@ -0,0 +1,95 @@
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package ai.vespa.metricsproxy.telegraf;
+
+import com.google.inject.Inject;
+import com.yahoo.component.AbstractComponent;
+import com.yahoo.log.LogLevel;
+import com.yahoo.system.execution.ProcessExecutor;
+import com.yahoo.system.execution.ProcessResult;
+import com.yahoo.vespa.defaults.Defaults;
+import org.apache.velocity.VelocityContext;
+import org.apache.velocity.app.VelocityEngine;
+
+import java.io.FileWriter;
+import java.io.InputStreamReader;
+import java.io.Reader;
+import java.io.Writer;
+import java.util.logging.Logger;
+
+import static com.yahoo.yolean.Exceptions.uncheck;
+
+/**
+ * @author olaa
+ */
+public class Telegraf extends AbstractComponent {
+
+ private static final String TELEGRAF_CONFIG_PATH = "/etc/telegraf/telegraf.conf";
+ private static final String TELEGRAF_CONFIG_TEMPLATE_PATH = "templates/telegraf.conf.vm";
+ private static final String TELEGRAF_LOG_FILE_PATH = Defaults.getDefaults().underVespaHome("logs/telegraf/telegraf.log");
+ private final TelegrafRegistry telegrafRegistry;
+
+ private static final Logger logger = Logger.getLogger(Telegraf.class.getName());
+
+ @Inject
+ public Telegraf(TelegrafRegistry telegrafRegistry, TelegrafConfig telegrafConfig) {
+ this.telegrafRegistry = telegrafRegistry;
+ telegrafRegistry.addInstance(this);
+ writeConfig(telegrafConfig, uncheck(() -> new FileWriter(TELEGRAF_CONFIG_PATH)));
+ restartTelegraf();
+ }
+
+ protected static void writeConfig(TelegrafConfig telegrafConfig, Writer writer) {
+ VelocityContext context = new VelocityContext();
+ context.put("logFilePath", TELEGRAF_LOG_FILE_PATH);
+ context.put("intervalSeconds", telegrafConfig.intervalSeconds());
+ context.put("cloudwatchPlugins", telegrafConfig.cloudWatch());
+ // TODO: Add node cert if hosted
+
+ VelocityEngine velocityEngine = new VelocityEngine();
+ velocityEngine.init();
+ velocityEngine.evaluate(context, writer, "TelegrafConfigWriter", getTemplateReader());
+ uncheck(writer::close);
+ }
+
+ private void restartTelegraf() {
+ executeCommand("service telegraf restart");
+ }
+
+ private void stopTelegraf() {
+ executeCommand("service telegraf stop");
+ }
+
+ private void executeCommand(String command) {
+ logger.info(String.format("Running command: %s", command));
+ ProcessExecutor processExecutor = new ProcessExecutor
+ .Builder(10)
+ .successExitCodes(0)
+ .build();
+ ProcessResult processResult = uncheck(() -> processExecutor.execute(command))
+ .orElseThrow(() -> new RuntimeException("Timed out running command: " + command));
+
+ logger.log(LogLevel.DEBUG, () -> String.format("Exit code: %d\nstdOut: %s\nstdErr: %s",
+ processResult.exitCode,
+ processResult.stdOut,
+ processResult.stdErr));
+
+ if (!processResult.stdErr.isBlank())
+ logger.warning(String.format("stdErr not empty: %s", processResult.stdErr));
+ }
+
+ @SuppressWarnings("ConstantConditions")
+ private static Reader getTemplateReader() {
+ return new InputStreamReader(Telegraf.class.getClassLoader()
+ .getResourceAsStream(TELEGRAF_CONFIG_TEMPLATE_PATH)
+ );
+
+ }
+
+ @Override
+ public void deconstruct() {
+ telegrafRegistry.removeInstance(this);
+ if (telegrafRegistry.isEmpty()) {
+ stopTelegraf();
+ }
+ }
+}
diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/telegraf/TelegrafRegistry.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/telegraf/TelegrafRegistry.java
new file mode 100644
index 00000000000..23da51ea082
--- /dev/null
+++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/telegraf/TelegrafRegistry.java
@@ -0,0 +1,33 @@
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package ai.vespa.metricsproxy.telegraf;
+
+import com.yahoo.log.LogLevel;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.logging.Logger;
+
+/**
+ * @author olaa
+ */
+public class TelegrafRegistry {
+
+ private static final List<Telegraf> telegrafInstances = Collections.synchronizedList(new ArrayList<>());
+
+ private static final Logger logger = Logger.getLogger(TelegrafRegistry.class.getName());
+
+ public void addInstance(Telegraf telegraf) {
+ logger.log(LogLevel.DEBUG, () -> "Adding Telegraf instance to registry: " + telegraf.hashCode());
+ telegrafInstances.add(telegraf);
+ }
+
+ public void removeInstance(Telegraf telegraf) {
+ logger.log(LogLevel.DEBUG, () -> "Removing Telegraf instance from registry: " + telegraf.hashCode());
+ telegrafInstances.remove(telegraf);
+ }
+
+ public boolean isEmpty() {
+ return telegrafInstances.isEmpty();
+ }
+}
diff --git a/metrics-proxy/src/main/resources/configdefinitions/telegraf.def b/metrics-proxy/src/main/resources/configdefinitions/telegraf.def
index f3b5db35d52..6abbd7921b5 100644
--- a/metrics-proxy/src/main/resources/configdefinitions/telegraf.def
+++ b/metrics-proxy/src/main/resources/configdefinitions/telegraf.def
@@ -5,9 +5,8 @@ package=ai.vespa.metricsproxy.telegraf
intervalSeconds int default=60
-# The consumer to get metrics for
-vespa.consumer string default="default"
-
+# The Vespa metrics consumer to get metrics for
+cloudWatch[].consumer string
cloudWatch[].region string default="us-east-1"
cloudWatch[].namespace string
diff --git a/metrics-proxy/src/main/resources/templates/telegraf.conf.vm b/metrics-proxy/src/main/resources/templates/telegraf.conf.vm
new file mode 100644
index 00000000000..e99bab8b02d
--- /dev/null
+++ b/metrics-proxy/src/main/resources/templates/telegraf.conf.vm
@@ -0,0 +1,44 @@
+# Configuration for telegraf agent
+[agent]
+ interval = "${intervalSeconds}s"
+ round_interval = true
+ metric_batch_size = 1000
+ metric_buffer_limit = 10000
+ collection_jitter = "0s"
+ flush_interval = "${intervalSeconds}s"
+ flush_jitter = "0s"
+ precision = ""
+ logtarget = "file"
+ logfile = "$logFilePath"
+ logfile_rotation_interval = "1d"
+ logfile_rotation_max_size = "20MB"
+ logfile_rotation_max_archives = 5
+
+#foreach( $cloudwatch in $cloudwatchPlugins )
+# Configuration for AWS CloudWatch output.
+[[outputs.cloudwatch]]
+ region = "$cloudwatch.region()"
+ namespace = "$cloudwatch.namespace()"
+#if( $cloudwatch.accessKeyName() != "" )
+ access_key = "$cloudwatch.accessKeyName()"
+ secret_key = "$cloudwatch.secretKeyName()"
+#elseif( $cloudwatch.profile() != "" )
+ profile = "$cloudwatch.profile()"
+#end
+ tagexclude = ["vespa_consumer"]
+ [outputs.cloudwatch.tagpass]
+ vespa_consumer = ["$cloudwatch.consumer()"]
+
+# Configuration for Vespa input plugin
+[[inputs.vespa]]
+ url = "http://localhost:19092/metrics/v2/values?consumer=$cloudwatch.consumer()"
+ [inputs.vespa.tags]
+ vespa_consumer = "$cloudwatch.consumer()"
+#* TODO: Add node cert if hosted
+#if( $isHosted )
+ tls_cert = "${VESPA_CERTIFICATE_PATH}"
+ tls_key = "${VESPA_KEY_PATH}"
+ insecure_skip_verify = true
+#end
+*###
+#end \ No newline at end of file
diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/telegraf/TelegrafTest.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/telegraf/TelegrafTest.java
new file mode 100644
index 00000000000..9ad31a0d9e8
--- /dev/null
+++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/telegraf/TelegrafTest.java
@@ -0,0 +1,42 @@
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package ai.vespa.metricsproxy.telegraf;
+
+import ai.vespa.metricsproxy.TestUtil;
+import org.junit.Test;
+
+import java.io.StringWriter;
+
+import static org.junit.Assert.*;
+
+/**
+ * @author olaa
+ */
+public class TelegrafTest {
+
+ @Test
+ public void test_writing_correct_telegraf_plugin_config() {
+ TelegrafConfig telegrafConfig = new TelegrafConfig.Builder()
+ .cloudWatch(
+ new TelegrafConfig.CloudWatch.Builder()
+ .accessKeyName("accessKey1")
+ .namespace("namespace1")
+ .secretKeyName("secretKey1")
+ .region("us-east-1")
+ .consumer("consumer1")
+ )
+ .cloudWatch(
+ new TelegrafConfig.CloudWatch.Builder()
+ .namespace("namespace2")
+ .profile("awsprofile")
+ .region("us-east-2")
+ .consumer("consumer2")
+ )
+ .intervalSeconds(300)
+ .build();
+ StringWriter stringWriter = new StringWriter();
+ Telegraf.writeConfig(telegrafConfig, stringWriter);
+ String expectedConfig = TestUtil.getFileContents( "telegraf-config-with-two-cloudwatch-plugins.txt");
+ assertEquals(expectedConfig, stringWriter.toString());
+ }
+
+}
diff --git a/metrics-proxy/src/test/resources/telegraf-config-with-two-cloudwatch-plugins.txt b/metrics-proxy/src/test/resources/telegraf-config-with-two-cloudwatch-plugins.txt
new file mode 100644
index 00000000000..accd2cc87eb
--- /dev/null
+++ b/metrics-proxy/src/test/resources/telegraf-config-with-two-cloudwatch-plugins.txt
@@ -0,0 +1,46 @@
+# Configuration for telegraf agent
+[agent]
+ interval = "300s"
+ round_interval = true
+ metric_batch_size = 1000
+ metric_buffer_limit = 10000
+ collection_jitter = "0s"
+ flush_interval = "300s"
+ flush_jitter = "0s"
+ precision = ""
+ logtarget = "file"
+ logfile = "/opt/vespa/logs/telegraf/telegraf.log"
+ logfile_rotation_interval = "1d"
+ logfile_rotation_max_size = "20MB"
+ logfile_rotation_max_archives = 5
+
+# Configuration for AWS CloudWatch output.
+[[outputs.cloudwatch]]
+ region = "us-east-1"
+ namespace = "namespace1"
+ access_key = "accessKey1"
+ secret_key = "secretKey1"
+ tagexclude = ["vespa_consumer"]
+ [outputs.cloudwatch.tagpass]
+ vespa_consumer = ["consumer1"]
+
+# Configuration for Vespa input plugin
+[[inputs.vespa]]
+ url = "http://localhost:19092/metrics/v2/values?consumer=consumer1"
+ [inputs.vespa.tags]
+ vespa_consumer = "consumer1"
+# Configuration for AWS CloudWatch output.
+[[outputs.cloudwatch]]
+ region = "us-east-2"
+ namespace = "namespace2"
+ profile = "awsprofile"
+ tagexclude = ["vespa_consumer"]
+ [outputs.cloudwatch.tagpass]
+ vespa_consumer = ["consumer2"]
+
+# Configuration for Vespa input plugin
+[[inputs.vespa]]
+ url = "http://localhost:19092/metrics/v2/values?consumer=consumer2"
+ [inputs.vespa.tags]
+ vespa_consumer = "consumer2"
+
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImpl.java
index c790e73037e..782f8592350 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImpl.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImpl.java
@@ -66,6 +66,9 @@ public class DockerOperationsImpl implements DockerOperations {
.withHostName(context.node().hostname())
.withResources(containerResources)
.withManagedBy(MANAGER_NAME)
+ // The inet6 option is needed to prefer AAAA records with gethostbyname(3), used by (at least) a yca package
+ // TODO: Try to remove this
+ .withDnsOption("inet6")
.withUlimit("nofile", 262_144, 262_144)
// The nproc aka RLIMIT_NPROC resource limit works as follows:
// - A process has a (soft) nproc limit, either inherited by the parent or changed with setrlimit(2).
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java
index 2f92ef8affe..3b6145aa6a8 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java
@@ -76,6 +76,7 @@ public class NodeAgentImpl implements NodeAgent {
private boolean hasResumedNode = false;
private boolean hasStartedServices = true;
private Optional<Instant> firstSuccessfulHealthCheckInstant = Optional.empty();
+ private boolean suspendedInOrchestrator = false;
private int numberOfUnhandledException = 0;
private long currentRebootGeneration = 0;
@@ -397,6 +398,7 @@ public class NodeAgentImpl implements NodeAgent {
public void converge(NodeAgentContext context) {
try {
doConverge(context);
+ context.log(logger, LogLevel.INFO, "Converged");
} catch (ConvergenceException e) {
context.log(logger, e.getMessage());
} catch (ContainerNotFoundException e) {
@@ -411,7 +413,7 @@ public class NodeAgentImpl implements NodeAgent {
}
}
- // Public for testing
+ // Non-private for testing
void doConverge(NodeAgentContext context) {
NodeSpec node = context.node();
Optional<Container> container = getContainer(context);
@@ -488,8 +490,11 @@ public class NodeAgentImpl implements NodeAgent {
// - Slobrok and internal orchestrator state is used to determine whether
// to allow upgrade (suspend).
updateNodeRepoWithCurrentAttributes(context);
- context.log(logger, "Call resume against Orchestrator");
- orchestrator.resume(context.hostname().value());
+ if (suspendedInOrchestrator || node.allowedToBeDown().orElse(false)) {
+ context.log(logger, "Call resume against Orchestrator");
+ orchestrator.resume(context.hostname().value());
+ suspendedInOrchestrator = false;
+ }
break;
case provisioned:
nodeRepository.setNodeState(context.hostname().value(), NodeState.dirty);
@@ -562,6 +567,7 @@ public class NodeAgentImpl implements NodeAgent {
context.log(logger, "Ask Orchestrator for permission to suspend node");
try {
orchestrator.suspend(context.hostname().value());
+ suspendedInOrchestrator = true;
} catch (OrchestratorException e) {
// Ensure the ACLs are up to date: The reason we're unable to suspend may be because some other
// node is unable to resume because the ACL rules of SOME Docker container is wrong...
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerMock.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerMock.java
index e2ad9e3de97..69bc9f5e092 100644
--- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerMock.java
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerMock.java
@@ -181,6 +181,11 @@ public class DockerMock implements Docker {
}
@Override
+ public CreateContainerCommand withDnsOption(String dnsOption) {
+ return this;
+ }
+
+ @Override
public CreateContainerCommand withPrivileged(boolean privileged) {
return this;
}
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java
index f65ad379a63..06d15a3f577 100644
--- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java
@@ -80,6 +80,7 @@ public class NodeAgentImplTest {
.wantedDockerImage(dockerImage).currentDockerImage(dockerImage)
.wantedVespaVersion(vespaVersion).currentVespaVersion(vespaVersion)
.wantedRestartGeneration(1).currentRestartGeneration(1)
+ .allowedToBeDown(false)
.build();
NodeAgentContext context = createContext(node);
@@ -98,7 +99,7 @@ public class NodeAgentImplTest {
// TODO: Verify this isn't run unless 1st time
inOrder.verify(dockerOperations, never()).startServices(eq(context));
inOrder.verify(dockerOperations, times(1)).resumeNode(eq(context));
- inOrder.verify(orchestrator).resume(hostName);
+ inOrder.verify(orchestrator, never()).resume(hostName);
}
@Test
@@ -188,7 +189,7 @@ public class NodeAgentImplTest {
inOrder.verify(healthChecker, times(1)).verifyHealth(eq(context));
inOrder.verify(nodeRepository).updateNodeAttributes(
hostName, new NodeAttributes().withDockerImage(dockerImage).withVespaVersion(dockerImage.tagAsVersion()));
- inOrder.verify(orchestrator).resume(hostName);
+ inOrder.verify(orchestrator, never()).resume(hostName);
}
@Test
@@ -224,7 +225,8 @@ public class NodeAgentImplTest {
.state(NodeState.active)
.wantedDockerImage(dockerImage).currentDockerImage(dockerImage)
.wantedVespaVersion(vespaVersion).currentVespaVersion(vespaVersion)
- .wantedRestartGeneration(1).currentRestartGeneration(1);
+ .wantedRestartGeneration(1).currentRestartGeneration(1)
+ .allowedToBeDown(false);
NodeAgentContext firstContext = createContext(specBuilder.build());
NodeAgentImpl nodeAgent = makeNodeAgent(dockerImage, true);
@@ -232,17 +234,20 @@ public class NodeAgentImplTest {
when(dockerOperations.pullImageAsyncIfNeeded(any())).thenReturn(true);
when(storageMaintainer.getDiskUsageFor(any())).thenReturn(Optional.of(201326592000L));
+ InOrder inOrder = inOrder(orchestrator, dockerOperations);
+
nodeAgent.doConverge(firstContext);
+ inOrder.verify(orchestrator, never()).resume(any(String.class));
+
NodeAgentContext secondContext = createContext(specBuilder.diskGb(200).build());
nodeAgent.doConverge(secondContext);
+ inOrder.verify(orchestrator, never()).resume(any(String.class));
+
NodeAgentContext thirdContext = new NodeAgentContextImpl.Builder(specBuilder.vcpu(5).build()).cpuSpeedUp(1.25).build();
nodeAgent.doConverge(thirdContext);
ContainerResources resourcesAfterThird = ContainerResources.from(0, 4, 16);
mockGetContainer(dockerImage, resourcesAfterThird, true);
- InOrder inOrder = inOrder(orchestrator, dockerOperations);
- inOrder.verify(orchestrator).resume(any(String.class));
- inOrder.verify(orchestrator).resume(any(String.class));
inOrder.verify(orchestrator).suspend(any(String.class));
inOrder.verify(dockerOperations).updateContainer(eq(thirdContext), eq(resourcesAfterThird));
inOrder.verify(dockerOperations, never()).removeContainer(any(), any());
@@ -254,7 +259,7 @@ public class NodeAgentImplTest {
inOrder.verify(orchestrator, never()).suspend(any(String.class));
inOrder.verify(dockerOperations, never()).updateContainer(eq(thirdContext), any());
inOrder.verify(dockerOperations, never()).removeContainer(any(), any());
- inOrder.verify(orchestrator).resume(any(String.class));
+ inOrder.verify(orchestrator, never()).resume(any(String.class));
// Set the feature flag
flagSource.withDoubleFlag(Flags.CONTAINER_CPU_CAP.id(), 2.3);
@@ -537,6 +542,7 @@ public class NodeAgentImplTest {
.wantedDockerImage(dockerImage).currentDockerImage(dockerImage)
.currentVespaVersion(vespaVersion)
.wantedRestartGeneration(1).currentRestartGeneration(1)
+ .allowedToBeDown(true)
.build();
NodeAgentContext context = createContext(node);
@@ -611,6 +617,7 @@ public class NodeAgentImplTest {
.type(NodeType.config)
.wantedDockerImage(dockerImage)
.wantedVespaVersion(vespaVersion)
+ .allowedToBeDown(true)
.build();
NodeAgentContext context = createContext(node);
@@ -689,6 +696,7 @@ public class NodeAgentImplTest {
clock.advance(Duration.ofSeconds(31));
nodeAgent.doConverge(context);
+ inOrder.verify(orchestrator).suspend(hostName);
inOrder.verify(dockerOperations).updateContainer(eq(context), eq(ContainerResources.from(0, 2, 16)));
inOrder.verify(dockerOperations, never()).removeContainer(any(), any());
inOrder.verify(dockerOperations, never()).startContainer(any());
@@ -699,7 +707,7 @@ public class NodeAgentImplTest {
inOrder.verify(orchestrator, never()).suspend(any(String.class));
inOrder.verify(dockerOperations, never()).updateContainer(eq(context), any());
inOrder.verify(dockerOperations, never()).removeContainer(any(), any());
- inOrder.verify(orchestrator).resume(any(String.class));
+ inOrder.verify(orchestrator, never()).resume(any(String.class));
}
@Test
@@ -724,7 +732,7 @@ public class NodeAgentImplTest {
inOrder.verify(orchestrator, never()).suspend(any(String.class));
inOrder.verify(dockerOperations, never()).updateContainer(eq(context), any());
inOrder.verify(dockerOperations, never()).removeContainer(any(), any());
- inOrder.verify(orchestrator).resume(any(String.class));
+ inOrder.verify(orchestrator, never()).resume(any(String.class));
}
private void verifyThatContainerIsStopped(NodeState nodeState, Optional<ApplicationId> owner) {
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/InfrastructureProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/InfrastructureProvisioner.java
index 3392569d1f2..4d04409aaf0 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/InfrastructureProvisioner.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/InfrastructureProvisioner.java
@@ -25,15 +25,20 @@ public class InfrastructureProvisioner extends Maintainer {
this.infraDeployer = infraDeployer;
}
+ public void maintainButThrowOnException() {
+ try {
+ infraDeployer.activateAllSupportedInfraApplications(true);
+ } catch (RuntimeException e) {
+ logger.log(LogLevel.INFO, "Failed to deploy supported infrastructure applications, " +
+ "will sleep 30s before propagating failure, to allow inspection of zk",
+ e.getMessage());
+ try { Thread.sleep(30_000); } catch (InterruptedException ignored) { }
+ throw e;
+ }
+ }
+
@Override
protected void maintain() {
- infraDeployer.getSupportedInfraDeployments().forEach((application, deployment) -> {
- try {
- deployment.activate();
- } catch (RuntimeException e) {
- logger.log(LogLevel.INFO, "Failed to activate " + application, e);
- // loop around to activate the next application
- }
- });
+ infraDeployer.activateAllSupportedInfraApplications(false);
}
}
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java
index f265feac788..ecc550527fc 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java
@@ -94,7 +94,7 @@ public class NodeRepositoryMaintenance extends AbstractComponent {
autoscalingMaintainer = new AutoscalingMaintainer(nodeRepository, provisionServiceProvider.getHostResourcesCalculator(), nodeMetricsDb, deployer, defaults.autoscalingInterval);
// The DuperModel is filled with infrastructure applications by the infrastructure provisioner, so explicitly run that now
- infrastructureProvisioner.maintain();
+ infrastructureProvisioner.maintainButThrowOnException();
}
@Override
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/InfraDeployerImpl.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/InfraDeployerImpl.java
index a56ace07c82..1086a3a7cd9 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/InfraDeployerImpl.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/InfraDeployerImpl.java
@@ -20,7 +20,6 @@ import com.yahoo.vespa.service.monitor.DuperModelInfraApi;
import com.yahoo.vespa.service.monitor.InfraApplicationApi;
import java.util.List;
-import java.util.Map;
import java.util.Optional;
import java.util.logging.Logger;
import java.util.stream.Collectors;
@@ -51,9 +50,22 @@ public class InfraDeployerImpl implements InfraDeployer {
}
@Override
- public Map<ApplicationId, Deployment> getSupportedInfraDeployments() {
- return duperModel.getSupportedInfraApplications().stream()
- .collect(Collectors.toMap(InfraApplicationApi::getApplicationId, InfraDeployment::new));
+ public void activateAllSupportedInfraApplications(boolean propagateException) {
+ duperModel.getSupportedInfraApplications().forEach(api -> {
+ var application = api.getApplicationId();
+ var deployment = new InfraDeployment(api);
+ try {
+ deployment.activate();
+ } catch (RuntimeException e) {
+ logger.log(LogLevel.INFO, "Failed to activate " + application, e);
+ if (propagateException) {
+ throw e;
+ }
+ // loop around to activate the next application
+ }
+ });
+
+ duperModel.infraApplicationsIsNowComplete();
}
private class InfraDeployment implements Deployment {
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockDuperModel.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockDuperModel.java
index ef3d1995df9..e7ebf049e51 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockDuperModel.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockDuperModel.java
@@ -50,4 +50,8 @@ public class MockDuperModel implements DuperModelInfraApi {
public void infraApplicationRemoved(ApplicationId applicationId) {
activeApps.remove(applicationId);
}
+
+ @Override
+ public void infraApplicationsIsNowComplete() {
+ }
}
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockInfraDeployer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockInfraDeployer.java
index e7bf76986ca..742a863fb38 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockInfraDeployer.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockInfraDeployer.java
@@ -5,7 +5,6 @@ import com.yahoo.config.provision.ApplicationId;
import com.yahoo.config.provision.Deployment;
import com.yahoo.config.provision.InfraDeployer;
-import java.util.Map;
import java.util.Optional;
public class MockInfraDeployer implements InfraDeployer {
@@ -15,7 +14,5 @@ public class MockInfraDeployer implements InfraDeployer {
}
@Override
- public Map<ApplicationId, Deployment> getSupportedInfraDeployments() {
- return Map.of();
- }
+ public void activateAllSupportedInfraApplications(boolean propagateException) { }
}
diff --git a/orchestrator/pom.xml b/orchestrator/pom.xml
index 5dd4e7ea87d..fed0e617c79 100644
--- a/orchestrator/pom.xml
+++ b/orchestrator/pom.xml
@@ -81,12 +81,6 @@
<scope>test</scope>
</dependency>
<dependency>
- <groupId>org.easytesting</groupId>
- <artifactId>fest-assert</artifactId>
- <version>1.4</version>
- <scope>test</scope>
- </dependency>
- <dependency>
<groupId>com.yahoo.vespa</groupId>
<artifactId>config-provisioning</artifactId>
<version>${project.version}</version>
diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorContext.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorContext.java
index eb6a4119f8a..af59eb11e9b 100644
--- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorContext.java
+++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorContext.java
@@ -38,9 +38,11 @@ public class OrchestratorContext implements AutoCloseable {
private final HashMap<ApplicationInstanceReference, Runnable> locks = new HashMap<>();
/** Create an OrchestratorContext for operations on multiple applications. */
- public static OrchestratorContext createContextForMultiAppOp(Clock clock, boolean largeLocks) {
+ public static OrchestratorContext createContextForMultiAppOp(Clock clock) {
return new OrchestratorContext(null, clock, TimeBudget.fromNow(clock, DEFAULT_TIMEOUT_FOR_BATCH_OP),
- false, largeLocks, false);
+ false, // probe
+ true, // large locks
+ false); // use permanently down status
}
/** Create an OrchestratorContext for an operation on a single application. */
diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorImpl.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorImpl.java
index fbe6864274c..d0062966a6d 100644
--- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorImpl.java
+++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorImpl.java
@@ -64,7 +64,6 @@ public class OrchestratorImpl implements Orchestrator {
private final ClusterControllerClientFactory clusterControllerClientFactory;
private final Clock clock;
private final ApplicationApiFactory applicationApiFactory;
- private final BooleanFlag enableLargeOrchestratorLocks;
private final BooleanFlag retireWithPermanentlyDownFlag;
@Inject
@@ -101,7 +100,6 @@ public class OrchestratorImpl implements Orchestrator {
this.instanceLookupService = instanceLookupService;
this.clock = clock;
this.applicationApiFactory = applicationApiFactory;
- this.enableLargeOrchestratorLocks = Flags.ENABLE_LARGE_ORCHESTRATOR_LOCKS.bindTo(flagSource);
this.retireWithPermanentlyDownFlag = Flags.RETIRE_WITH_PERMANENTLY_DOWN.bindTo(flagSource);
}
@@ -167,7 +165,7 @@ public class OrchestratorImpl implements Orchestrator {
OrchestratorContext context = OrchestratorContext.createContextForSingleAppOp(clock);
try (MutableStatusRegistry statusRegistry = statusService
.lockApplicationInstance_forCurrentThreadOnly(context, appInstance.reference())) {
- HostStatus currentHostState = statusRegistry.getHostInfo(hostName).status();
+ HostStatus currentHostState = statusRegistry.getHostInfos().getOrNoRemarks(hostName).status();
if (currentHostState == HostStatus.NO_REMARKS) {
return;
}
@@ -258,10 +256,7 @@ public class OrchestratorImpl implements Orchestrator {
@Override
public void suspendAll(HostName parentHostname, List<HostName> hostNames)
throws BatchHostStateChangeDeniedException, BatchHostNameNotFoundException, BatchInternalErrorException {
- boolean largeLocks = enableLargeOrchestratorLocks
- .with(FetchVector.Dimension.HOSTNAME, parentHostname.s())
- .value();
- try (OrchestratorContext context = OrchestratorContext.createContextForMultiAppOp(clock, largeLocks)) {
+ try (OrchestratorContext context = OrchestratorContext.createContextForMultiAppOp(clock)) {
List<NodeGroup> nodeGroupsOrderedByApplication;
try {
nodeGroupsOrderedByApplication = nodeGroupsOrderedForSuspend(hostNames);
diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ClusterApi.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ClusterApi.java
index e9bb4984c2e..65c45c8df76 100644
--- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ClusterApi.java
+++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ClusterApi.java
@@ -26,7 +26,6 @@ public interface ClusterApi {
Optional<StorageNode> storageNodeInGroup();
Optional<StorageNode> upStorageNodeInGroup();
- String servicesDownAndNotInGroupDescription();
- String nodesAllowedToBeDownNotInGroupDescription();
+ String downDescription();
}
diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ClusterApiImpl.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ClusterApiImpl.java
index b747d8c2e22..24f56eac85d 100644
--- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ClusterApiImpl.java
+++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ClusterApiImpl.java
@@ -131,25 +131,56 @@ class ClusterApiImpl implements ClusterApi {
return numberOfServicesDown * 100 / (serviceCluster.serviceInstances().size() + missingServices);
}
+ /**
+ * A description of the hosts outside the group that are allowed to be down,
+ * and a description of the services outside the group and outside of the allowed services
+ * that are down.
+ */
@Override
- public String servicesDownAndNotInGroupDescription() {
- // Sort these for readability and testing stability
- return Stream
- .concat(servicesDownAndNotInGroup.stream().map(ServiceInstance::toString).sorted(),
- missingServices > 0 ? Stream.of(descriptionOfMissingServices) : Stream.of())
- .collect(Collectors.toList())
- .toString();
- }
+ public String downDescription() {
+ StringBuilder description = new StringBuilder();
- @Override
- public String nodesAllowedToBeDownNotInGroupDescription() {
- return servicesNotInGroup.stream()
+ Set<HostName> suspended = servicesNotInGroup.stream()
.map(ServiceInstance::hostName)
.filter(hostName -> hostStatus(hostName).isSuspended())
- .sorted()
- .distinct()
- .collect(Collectors.toList())
- .toString();
+ .collect(Collectors.toSet());
+
+ if (suspended.size() > 0) {
+ description.append(" ");
+
+ final int nodeLimit = 3;
+ description.append("Suspended hosts: ");
+ description.append(suspended.stream().sorted().distinct().limit(nodeLimit).collect(Collectors.toList()).toString());
+ if (suspended.size() > nodeLimit) {
+ description.append(", and " + (suspended.size() - nodeLimit) + " more");
+ }
+ description.append(".");
+ }
+
+ Set<ServiceInstance> downElsewhere = servicesDownAndNotInGroup.stream()
+ .filter(serviceInstance -> !suspended.contains(serviceInstance.hostName()))
+ .collect(Collectors.toSet());
+
+ final int downElsewhereTotal = downElsewhere.size() + missingServices;
+ if (downElsewhereTotal > 0) {
+ description.append(" ");
+
+ final int serviceLimit = 2; // services info is verbose
+ description.append("Services down on resumed hosts: ");
+ description.append(Stream.concat(
+ downElsewhere.stream().map(ServiceInstance::toString).sorted(),
+ missingServices > 0 ? Stream.of(descriptionOfMissingServices) : Stream.of())
+ .limit(serviceLimit)
+ .collect(Collectors.toList())
+ .toString());
+
+ if (downElsewhereTotal > serviceLimit) {
+ description.append(", and " + (downElsewhereTotal - serviceLimit) + " more");
+ }
+ description.append(".");
+ }
+
+ return description.toString();
}
private Optional<StorageNode> storageNodeInGroup(Predicate<ServiceInstance> storageServicePredicate) {
diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/HostStateChangeDeniedException.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/HostStateChangeDeniedException.java
index e13cf17d420..a6b3cbc87dc 100644
--- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/HostStateChangeDeniedException.java
+++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/HostStateChangeDeniedException.java
@@ -2,55 +2,43 @@
package com.yahoo.vespa.orchestrator.policy;
import com.yahoo.vespa.applicationmodel.HostName;
-import com.yahoo.vespa.applicationmodel.ServiceType;
import com.yahoo.vespa.orchestrator.OrchestrationException;
import com.yahoo.vespa.orchestrator.model.NodeGroup;
-import java.util.Optional;
-
/**
* @author bakksjo
*/
public class HostStateChangeDeniedException extends OrchestrationException {
private final String constraintName;
- private final Optional<ServiceType> serviceType;
public HostStateChangeDeniedException(HostName hostName, String constraintName, String message) {
this(hostName, constraintName, message, null);
}
public HostStateChangeDeniedException(HostName hostName, String constraintName, String message, Exception e) {
- this(hostName.s(), constraintName, Optional.empty(), message, e);
+ this(hostName.s(), constraintName, message, e);
}
public HostStateChangeDeniedException(NodeGroup nodeGroup, String constraintName, String message) {
- this(nodeGroup.toCommaSeparatedString(), constraintName, Optional.empty(), message, null);
+ this(nodeGroup.toCommaSeparatedString(), constraintName, message, null);
}
private HostStateChangeDeniedException(String nodes,
String constraintName,
- Optional<ServiceType> serviceType,
String message,
Throwable cause) {
- super(createMessage(nodes, constraintName, serviceType, message), cause);
+ super(createMessage(nodes, constraintName, message), cause);
this.constraintName = constraintName;
- this.serviceType = serviceType;
}
private static String createMessage(String nodes,
String constraintName,
- Optional<ServiceType> serviceType,
String message) {
return "Changing the state of " + nodes + " would violate " + constraintName
- + (serviceType.isPresent() ? " for service type " + serviceType.get() : "")
+ ": " + message;
}
public String getConstraintName() {
return constraintName;
}
-
- public Optional<ServiceType> getServiceType() {
- return serviceType;
- }
}
diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/HostedVespaClusterPolicy.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/HostedVespaClusterPolicy.java
index 1e895d0e757..ccb0bb57186 100644
--- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/HostedVespaClusterPolicy.java
+++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/HostedVespaClusterPolicy.java
@@ -26,13 +26,11 @@ public class HostedVespaClusterPolicy implements ClusterPolicy {
throw new HostStateChangeDeniedException(
clusterApi.getNodeGroup(),
ENOUGH_SERVICES_UP_CONSTRAINT,
- "Suspension percentage for service type " + clusterApi.serviceType()
+ "Suspension for service type " + clusterApi.serviceType()
+ " would increase from " + clusterApi.percentageOfServicesDown()
+ "% to " + clusterApi.percentageOfServicesDownIfGroupIsAllowedToBeDown()
+ "%, over the limit of " + percentageOfServicesAllowedToBeDown + "%."
- + " These instances may be down: " + clusterApi.servicesDownAndNotInGroupDescription()
- + " and these hosts are allowed to be down: "
- + clusterApi.nodesAllowedToBeDownNotInGroupDescription());
+ + clusterApi.downDescription());
}
@Override
@@ -56,9 +54,7 @@ public class HostedVespaClusterPolicy implements ClusterPolicy {
"Down percentage for service type " + clusterApi.serviceType()
+ " would increase to " + clusterApi.percentageOfServicesDownIfGroupIsAllowedToBeDown()
+ "%, over the limit of " + percentageOfServicesAllowedToBeDown + "%."
- + " These instances may be down: " + clusterApi.servicesDownAndNotInGroupDescription()
- + " and these hosts are allowed to be down: "
- + clusterApi.nodesAllowedToBeDownNotInGroupDescription());
+ + clusterApi.downDescription());
}
// Non-private for testing purposes
diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/HostResource.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/HostResource.java
index fc5c5eb5004..c93bb9e1d4a 100644
--- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/HostResource.java
+++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/HostResource.java
@@ -157,8 +157,9 @@ public class HostResource implements HostApi {
private static WebApplicationException webExceptionFromTimeout(String operationDescription,
HostName hostName,
UncheckedTimeoutException e) {
- return createWebException(operationDescription, hostName, e, HostedVespaPolicy.DEADLINE_CONSTRAINT, e.getMessage(),
- Response.Status.GATEWAY_TIMEOUT);
+ // Return timeouts as 409 Conflict instead of 504 Gateway Timeout to reduce noise in 5xx graphs.
+ return createWebException(operationDescription, hostName, e,
+ HostedVespaPolicy.DEADLINE_CONSTRAINT, e.getMessage(), Response.Status.CONFLICT);
}
private static WebApplicationException webExceptionWithDenialReason(
diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/HostSuspensionResource.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/HostSuspensionResource.java
index 79e0fc0f3e9..6e857563f9b 100644
--- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/HostSuspensionResource.java
+++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/HostSuspensionResource.java
@@ -18,7 +18,6 @@ import javax.ws.rs.WebApplicationException;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response;
import java.util.List;
-import java.util.concurrent.TimeoutException;
import java.util.logging.Logger;
import java.util.stream.Collectors;
@@ -48,7 +47,7 @@ public class HostSuspensionResource implements HostSuspensionApi {
throw createWebApplicationException(e.getMessage(), Response.Status.CONFLICT);
} catch (UncheckedTimeoutException e) {
log.log(LogLevel.DEBUG, "Failed to suspend nodes " + hostnames + " with parent host " + parentHostname, e);
- throw createWebApplicationException(e.getMessage(), Response.Status.GATEWAY_TIMEOUT);
+ throw createWebApplicationException(e.getMessage(), Response.Status.CONFLICT);
} catch (BatchHostNameNotFoundException e) {
log.log(LogLevel.DEBUG, "Failed to suspend nodes " + hostnames + " with parent host " + parentHostname, e);
// Note that we're returning BAD_REQUEST instead of NOT_FOUND because the resource identified
diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/MutableStatusRegistry.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/MutableStatusRegistry.java
index d2042dc9fd2..24da83364aa 100644
--- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/MutableStatusRegistry.java
+++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/MutableStatusRegistry.java
@@ -17,9 +17,6 @@ public interface MutableStatusRegistry extends AutoCloseable {
/** Returns the status of this application. */
ApplicationInstanceStatus getStatus();
- /** Returns the host info of the given host. */
- HostInfo getHostInfo(HostName hostName);
-
/** Returns a snapshot of all host infos for this application. */
HostInfos getHostInfos();
diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/ZookeeperStatusService.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/ZookeeperStatusService.java
index b4025167187..2cdce350377 100644
--- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/ZookeeperStatusService.java
+++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/ZookeeperStatusService.java
@@ -371,11 +371,6 @@ public class ZookeeperStatusService implements StatusService {
}
@Override
- public HostInfo getHostInfo(HostName hostName) {
- return ZookeeperStatusService.this.getHostInfo(applicationInstanceReference, hostName);
- }
-
- @Override
public HostInfos getHostInfos() {
return hostInfosCache.getHostInfos(applicationInstanceReference);
}
diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorContextTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorContextTest.java
index 607894ee104..16b66b2804e 100644
--- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorContextTest.java
+++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorContextTest.java
@@ -23,7 +23,7 @@ public class OrchestratorContextTest {
var mutable = new Object() { boolean locked = true; };
Runnable unlock = () -> mutable.locked = false;
- try (OrchestratorContext rootContext = OrchestratorContext.createContextForMultiAppOp(new ManualClock(), true)) {
+ try (OrchestratorContext rootContext = OrchestratorContext.createContextForMultiAppOp(new ManualClock())) {
try (OrchestratorContext probeContext = rootContext.createSubcontextForSingleAppOp(true)) {
assertFalse(probeContext.hasLock(application));
assertTrue(probeContext.registerLockAcquisition(application, unlock));
@@ -41,19 +41,4 @@ public class OrchestratorContextTest {
}
assertFalse(mutable.locked);
}
-
- @Test
- public void testLargeLocksDisabled() {
- var mutable = new Object() { boolean locked = true; };
- Runnable unlock = () -> mutable.locked = false;
-
- try (OrchestratorContext rootContext = OrchestratorContext.createContextForMultiAppOp(new ManualClock(), false)) {
- try (OrchestratorContext probeContext = rootContext.createSubcontextForSingleAppOp(true)) {
- assertFalse(probeContext.hasLock(application));
- assertFalse(probeContext.registerLockAcquisition(application, unlock));
- }
- }
-
- assertTrue(mutable.locked);
- }
} \ No newline at end of file
diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorImplTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorImplTest.java
index 77ec824da54..bfe4b523e4a 100644
--- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorImplTest.java
+++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorImplTest.java
@@ -17,7 +17,6 @@ import com.yahoo.vespa.applicationmodel.ServiceStatus;
import com.yahoo.vespa.applicationmodel.ServiceType;
import com.yahoo.vespa.applicationmodel.TenantId;
import com.yahoo.vespa.curator.mock.MockCurator;
-import com.yahoo.vespa.flags.Flags;
import com.yahoo.vespa.flags.InMemoryFlagSource;
import com.yahoo.vespa.orchestrator.controller.ClusterControllerClientFactory;
import com.yahoo.vespa.orchestrator.controller.ClusterControllerClientFactoryMock;
@@ -325,8 +324,6 @@ public class OrchestratorImplTest {
@Test
public void testLargeLocks() throws Exception {
- flagSource.withBooleanFlag(Flags.ENABLE_LARGE_ORCHESTRATOR_LOCKS.id(), true);
-
var tenantId = new TenantId("tenant");
var applicationInstanceId = new ApplicationInstanceId("app:dev:us-east-1:default");
var applicationInstanceReference = new ApplicationInstanceReference(tenantId, applicationInstanceId);
diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/model/ClusterApiImplTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/model/ClusterApiImplTest.java
index 62925dc003e..a5cb5cfa630 100644
--- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/model/ClusterApiImplTest.java
+++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/model/ClusterApiImplTest.java
@@ -21,7 +21,6 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.EnumSet;
-import java.util.HashMap;
import java.util.List;
import java.util.Optional;
import java.util.Set;
@@ -81,15 +80,10 @@ public class ClusterApiImplTest {
assertEquals("{ clusterId=cluster, serviceType=service-type }", clusterApi.clusterInfo());
assertFalse(clusterApi.isStorageCluster());
- assertEquals("[ServiceInstance{configId=service-2, hostName=host2, serviceStatus=" +
- "ServiceStatusInfo{status=DOWN, since=Optional.empty, lastChecked=Optional.empty}}, "
- + "ServiceInstance{configId=service-3, hostName=host3, serviceStatus=" +
- "ServiceStatusInfo{status=UP, since=Optional.empty, lastChecked=Optional.empty}}, "
- + "ServiceInstance{configId=service-4, hostName=host4, serviceStatus=" +
- "ServiceStatusInfo{status=DOWN, since=Optional.empty, lastChecked=Optional.empty}}]",
- clusterApi.servicesDownAndNotInGroupDescription());
- assertEquals("[host3, host4]",
- clusterApi.nodesAllowedToBeDownNotInGroupDescription());
+ assertEquals(" Suspended hosts: [host3, host4]. Services down on resumed hosts: [" +
+ "ServiceInstance{configId=service-2, hostName=host2, serviceStatus=" +
+ "ServiceStatusInfo{status=DOWN, since=Optional.empty, lastChecked=Optional.empty}}].",
+ clusterApi.downDescription());
assertEquals(60, clusterApi.percentageOfServicesDown());
assertEquals(80, clusterApi.percentageOfServicesDownIfGroupIsAllowedToBeDown());
}
@@ -110,9 +104,9 @@ public class ClusterApiImplTest {
fail();
} catch (HostStateChangeDeniedException e) {
assertThat(e.getMessage(),
- containsString("Changing the state of cfg1 would violate enough-services-up: Suspension percentage " +
- "for service type configserver would increase from 33% to 66%, over the limit of 10%. " +
- "These instances may be down: [1 missing config server] and these hosts are allowed to be down: []"));
+ containsString("Changing the state of cfg1 would violate enough-services-up: " +
+ "Suspension for service type configserver would increase from 33% to 66%, " +
+ "over the limit of 10%. Services down on resumed hosts: [1 missing config server]."));
}
}
diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/policy/HostedVespaClusterPolicyTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/policy/HostedVespaClusterPolicyTest.java
index d834034c9a8..4462e886d1b 100644
--- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/policy/HostedVespaClusterPolicyTest.java
+++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/policy/HostedVespaClusterPolicyTest.java
@@ -101,8 +101,7 @@ public class HostedVespaClusterPolicyTest {
when(clusterApi.serviceType()).thenReturn(new ServiceType("service-type"));
when(clusterApi.percentageOfServicesDown()).thenReturn(5);
when(clusterApi.percentageOfServicesDownIfGroupIsAllowedToBeDown()).thenReturn(percentageOfServicesDownIfGroupIsAllowedToBeDown);
- when(clusterApi.servicesDownAndNotInGroupDescription()).thenReturn("services-down-and-not-in-group");
- when(clusterApi.nodesAllowedToBeDownNotInGroupDescription()).thenReturn("allowed-to-be-down");
+ when(clusterApi.downDescription()).thenReturn(" Down description");
NodeGroup nodeGroup = mock(NodeGroup.class);
when(clusterApi.getNodeGroup()).thenReturn(nodeGroup);
@@ -116,11 +115,9 @@ public class HostedVespaClusterPolicyTest {
}
} catch (HostStateChangeDeniedException e) {
if (!expectSuccess) {
- assertEquals("Changing the state of node-group would violate enough-services-up: "
- + "Suspension percentage for service type service-type would increase from "
- + "5% to 13%, over the limit of 10%. These instances may be down: "
- + "services-down-and-not-in-group and these hosts are allowed to be down: "
- + "allowed-to-be-down", e.getMessage());
+ assertEquals("Changing the state of node-group would violate enough-services-up: " +
+ "Suspension for service type service-type would increase from 5% to 13%, " +
+ "over the limit of 10%. Down description", e.getMessage());
assertEquals("enough-services-up", e.getConstraintName());
}
}
diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/resources/HostResourceTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/resources/HostResourceTest.java
index dc26c1a3770..bfa68145828 100644
--- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/resources/HostResourceTest.java
+++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/resources/HostResourceTest.java
@@ -60,9 +60,9 @@ import java.util.Optional;
import java.util.Set;
import static com.yahoo.vespa.orchestrator.TestUtil.makeServiceClusterSet;
-import static org.fest.assertions.Assertions.assertThat;
-import static org.fest.assertions.Fail.fail;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.doThrow;
import static org.mockito.Mockito.mock;
@@ -167,7 +167,7 @@ public class HostResourceTest {
UpdateHostResponse response = hostResource.suspend(hostName);
- assertThat(response.hostname()).isEqualTo(hostName);
+ assertEquals(hostName, response.hostname());
}
@Test
@@ -175,14 +175,14 @@ public class HostResourceTest {
HostSuspensionResource hostSuspensionResource = new HostSuspensionResource(alwaysAllowOrchestrator);
BatchOperationResult response = hostSuspensionResource.suspendAll("parentHostname",
Arrays.asList("hostname1", "hostname2"));
- assertThat(response.success());
+ assertTrue(response.success());
}
@Test
public void returns_200_empty_batch() {
HostSuspensionResource hostSuspensionResource = new HostSuspensionResource(alwaysAllowOrchestrator);
BatchOperationResult response = hostSuspensionResource.suspendAll("parentHostname", List.of());
- assertThat(response.success());
+ assertTrue(response.success());
}
@Test
@@ -193,7 +193,7 @@ public class HostResourceTest {
hostResource.suspend("hostname");
fail();
} catch (WebApplicationException w) {
- assertThat(w.getResponse().getStatus()).isEqualTo(404);
+ assertEquals(404, w.getResponse().getStatus());
}
}
@@ -207,7 +207,7 @@ public class HostResourceTest {
hostSuspensionResource.suspendAll("parentHostname", Arrays.asList("hostname1", "hostname2"));
fail();
} catch (WebApplicationException w) {
- assertThat(w.getResponse().getStatus()).isEqualTo(400);
+ assertEquals(400, w.getResponse().getStatus());
}
}
@@ -259,7 +259,7 @@ public class HostResourceTest {
hostResource.suspend("hostname");
fail();
} catch (WebApplicationException w) {
- assertThat(w.getResponse().getStatus()).isEqualTo(409);
+ assertEquals(409, w.getResponse().getStatus());
}
}
@@ -280,7 +280,7 @@ public class HostResourceTest {
hostSuspensionResource.suspendAll("parentHostname", Arrays.asList("hostname1", "hostname2"));
fail();
} catch (WebApplicationException w) {
- assertThat(w.getResponse().getStatus()).isEqualTo(409);
+ assertEquals(409, w.getResponse().getStatus());
}
}
@@ -368,7 +368,7 @@ public class HostResourceTest {
}
@Test
- public void throws_504_on_timeout() throws HostNameNotFoundException, HostStateChangeDeniedException {
+ public void throws_409_on_timeout() throws HostNameNotFoundException, HostStateChangeDeniedException {
Orchestrator orchestrator = mock(Orchestrator.class);
doThrow(new UncheckedTimeoutException("Timeout Message")).when(orchestrator).resume(any(HostName.class));
@@ -377,13 +377,13 @@ public class HostResourceTest {
hostResource.resume("hostname");
fail();
} catch (WebApplicationException w) {
- assertThat(w.getResponse().getStatus()).isEqualTo(504);
+ assertEquals(409, w.getResponse().getStatus());
assertEquals("resume failed: Timeout Message [deadline]", w.getMessage());
}
}
@Test
- public void throws_504_on_suspendAll_timeout() throws BatchHostStateChangeDeniedException, BatchHostNameNotFoundException, BatchInternalErrorException {
+ public void throws_409_on_suspendAll_timeout() throws BatchHostStateChangeDeniedException, BatchHostNameNotFoundException, BatchInternalErrorException {
Orchestrator orchestrator = mock(Orchestrator.class);
doThrow(new UncheckedTimeoutException("Timeout Message")).when(orchestrator).suspendAll(any(), any());
@@ -392,7 +392,7 @@ public class HostResourceTest {
resource.suspendAll("parenthost", Arrays.asList("h1", "h2", "h3"));
fail();
} catch (WebApplicationException w) {
- assertThat(w.getResponse().getStatus()).isEqualTo(504);
+ assertEquals(409, w.getResponse().getStatus());
}
}
}
diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/status/ZookeeperStatusServiceTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/status/ZookeeperStatusServiceTest.java
index 12622f22837..687ea951f88 100644
--- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/status/ZookeeperStatusServiceTest.java
+++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/status/ZookeeperStatusServiceTest.java
@@ -117,7 +117,7 @@ public class ZookeeperStatusServiceTest {
TestIds.HOST_NAME1,
hostStatus);
- assertThat(statusRegistry.getHostInfo(TestIds.HOST_NAME1).status(),
+ assertThat(statusRegistry.getHostInfos().getOrNoRemarks(TestIds.HOST_NAME1).status(),
is(hostStatus));
}
}
@@ -182,7 +182,7 @@ public class ZookeeperStatusServiceTest {
killSession(curator.framework(), testingServer);
//Throws SessionFailedException if the SessionFailRetryLoop has not been closed.
- statusRegistry.getHostInfo(TestIds.HOST_NAME1);
+ statusRegistry.getHostInfos().getOrNoRemarks(TestIds.HOST_NAME1);
});
assertThat(resultOfZkOperationAfterLockFailure, notHoldsException());
diff --git a/parent/pom.xml b/parent/pom.xml
index b1ca2539ef5..68ee698e7a5 100644
--- a/parent/pom.xml
+++ b/parent/pom.xml
@@ -735,8 +735,8 @@
<properties>
<antlr.version>3.5.2</antlr.version>
<antlr4.version>4.5</antlr4.version>
- <apache.httpclient.version>4.5.10</apache.httpclient.version>
- <apache.httpcore.version>4.4.12</apache.httpcore.version>
+ <apache.httpclient.version>4.5.11</apache.httpclient.version>
+ <apache.httpcore.version>4.4.13</apache.httpcore.version>
<asm.version>7.0</asm.version>
<!-- Athenz dependencies. Make sure these dependencies match those in Vespa's internal repositories -->
<athenz.version>1.8.44</athenz.version>
diff --git a/searchcommon/src/vespa/searchcommon/attribute/config.cpp b/searchcommon/src/vespa/searchcommon/attribute/config.cpp
index 53e57fd9c66..b4e05875820 100644
--- a/searchcommon/src/vespa/searchcommon/attribute/config.cpp
+++ b/searchcommon/src/vespa/searchcommon/attribute/config.cpp
@@ -17,7 +17,8 @@ Config::Config() :
_growStrategy(),
_compactionStrategy(),
_predicateParams(),
- _tensorType(vespalib::eval::ValueType::error_type())
+ _tensorType(vespalib::eval::ValueType::error_type()),
+ _hnsw_index_params()
{
}
@@ -34,7 +35,8 @@ Config::Config(BasicType bt, CollectionType ct, bool fastSearch_, bool huge_)
_growStrategy(),
_compactionStrategy(),
_predicateParams(),
- _tensorType(vespalib::eval::ValueType::error_type())
+ _tensorType(vespalib::eval::ValueType::error_type()),
+ _hnsw_index_params()
{
}
@@ -60,7 +62,8 @@ Config::operator==(const Config &b) const
_compactionStrategy == b._compactionStrategy &&
_predicateParams == b._predicateParams &&
(_basicType.type() != BasicType::Type::TENSOR ||
- _tensorType == b._tensorType);
+ _tensorType == b._tensorType) &&
+ _hnsw_index_params == b._hnsw_index_params;
}
}
diff --git a/searchcommon/src/vespa/searchcommon/attribute/config.h b/searchcommon/src/vespa/searchcommon/attribute/config.h
index 2f767061f7a..836fcfed84a 100644
--- a/searchcommon/src/vespa/searchcommon/attribute/config.h
+++ b/searchcommon/src/vespa/searchcommon/attribute/config.h
@@ -4,15 +4,21 @@
#include "basictype.h"
#include "collectiontype.h"
+#include "hnsw_index_params.h"
#include "predicate_params.h"
-#include <vespa/searchcommon/common/growstrategy.h>
#include <vespa/searchcommon/common/compaction_strategy.h>
+#include <vespa/searchcommon/common/growstrategy.h>
#include <vespa/eval/eval/value_type.h>
+#include <optional>
namespace search::attribute {
-class Config
-{
+/**
+ * Configuration for an attribute vector.
+ *
+ * Used to determine which implementation to instantiate.
+ */
+class Config {
public:
Config();
Config(BasicType bt, CollectionType ct = CollectionType::SINGLE,
@@ -29,6 +35,7 @@ public:
bool huge() const { return _huge; }
const PredicateParams &predicateParams() const { return _predicateParams; }
vespalib::eval::ValueType tensorType() const { return _tensorType; }
+ const std::optional<HnswIndexParams>& hnsw_index_params() const { return _hnsw_index_params; }
/**
* Check if attribute posting list can consist of a bitvector in
@@ -60,6 +67,10 @@ public:
_tensorType = tensorType_in;
return *this;
}
+ Config& set_hnsw_index_params(const HnswIndexParams& params) {
+ _hnsw_index_params = params;
+ return *this;
+ }
/**
* Enable attribute posting list to consist of a bitvector in
@@ -107,6 +118,7 @@ private:
CompactionStrategy _compactionStrategy;
PredicateParams _predicateParams;
vespalib::eval::ValueType _tensorType;
+ std::optional<HnswIndexParams> _hnsw_index_params;
};
}
diff --git a/searchcommon/src/vespa/searchcommon/attribute/hnsw_index_params.h b/searchcommon/src/vespa/searchcommon/attribute/hnsw_index_params.h
new file mode 100644
index 00000000000..9e98a8c5fb7
--- /dev/null
+++ b/searchcommon/src/vespa/searchcommon/attribute/hnsw_index_params.h
@@ -0,0 +1,32 @@
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+namespace search::attribute {
+
+/**
+ * Configuration parameters for a hnsw index used together with a 1-dimensional indexed tensor
+ * for approximate nearest neighbor search.
+ */
+class HnswIndexParams {
+private:
+ uint32_t _max_links_per_node;
+ uint32_t _neighbors_to_explore_at_insert;
+
+public:
+ HnswIndexParams(uint32_t max_links_per_node_in,
+ uint32_t neighbors_to_explore_at_insert_in)
+ : _max_links_per_node(max_links_per_node_in),
+ _neighbors_to_explore_at_insert(neighbors_to_explore_at_insert_in)
+ {}
+
+ uint32_t max_links_per_node() const { return _max_links_per_node; }
+ uint32_t neighbors_to_explore_at_insert() const { return _neighbors_to_explore_at_insert; }
+
+ bool operator==(const HnswIndexParams& rhs) const {
+ return _max_links_per_node == rhs._max_links_per_node &&
+ _neighbors_to_explore_at_insert == rhs._neighbors_to_explore_at_insert;
+ }
+};
+
+}
diff --git a/searchcore/src/tests/proton/documentdb/configurer/configurer_test.cpp b/searchcore/src/tests/proton/documentdb/configurer/configurer_test.cpp
index 3c3ed33b8ea..14ad82a0aa1 100644
--- a/searchcore/src/tests/proton/documentdb/configurer/configurer_test.cpp
+++ b/searchcore/src/tests/proton/documentdb/configurer/configurer_test.cpp
@@ -208,10 +208,10 @@ Fixture::initViewSet(ViewSet &views)
views._lidReuseDelayer = std::make_unique<documentmetastore::LidReuseDelayer>(views._writeService, metaStore->get());
IndexSearchable::SP indexSearchable;
MatchView::SP matchView(new MatchView(matchers, indexSearchable, attrMgr, sesMgr, metaStore, views._docIdLimit));
- views.searchView.set(make_shared<SearchView>
+ views.searchView.set(SearchView::create
(summaryMgr->createSummarySetup(SummaryConfig(), SummarymapConfig(),
JuniperrcConfig(), views.repo, attrMgr),
- matchView));
+ std::move(matchView)));
views.feedView.set(
make_shared<SearchableFeedView>(StoreOnlyFeedView::Context(summaryAdapter,
schema,
diff --git a/searchcore/src/tests/proton/documentdb/threading_service_config/threading_service_config_test.cpp b/searchcore/src/tests/proton/documentdb/threading_service_config/threading_service_config_test.cpp
index 340619f09bd..37ac5f0d65c 100644
--- a/searchcore/src/tests/proton/documentdb/threading_service_config/threading_service_config_test.cpp
+++ b/searchcore/src/tests/proton/documentdb/threading_service_config/threading_service_config_test.cpp
@@ -1,12 +1,13 @@
// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-#include <vespa/log/log.h>
-LOG_SETUP("threading_service_config_test");
#include <vespa/searchcore/config/config-proton.h>
#include <vespa/searchcore/proton/common/hw_info.h>
#include <vespa/searchcore/proton/server/threading_service_config.h>
#include <vespa/vespalib/testkit/testapp.h>
+#include <vespa/log/log.h>
+LOG_SETUP("threading_service_config_test");
+
using namespace proton;
using ProtonConfig = vespa::config::search::core::ProtonConfig;
using ProtonConfigBuilder = vespa::config::search::core::ProtonConfigBuilder;
@@ -42,6 +43,7 @@ TEST_F("require that indexing threads are set based on cpu cores and feeding con
TEST_DO(f.assertIndexingThreads(3, 18));
TEST_DO(f.assertIndexingThreads(4, 19));
TEST_DO(f.assertIndexingThreads(4, 24));
+ TEST_DO(f.assertIndexingThreads(4, 64)); // Ensure it is capped at 4
}
TEST_F("require that indexing threads is always >= 1", Fixture(0))
diff --git a/searchcore/src/tests/proton/matchengine/matchengine.cpp b/searchcore/src/tests/proton/matchengine/matchengine.cpp
index 9ae938981a4..fd48fffcb5c 100644
--- a/searchcore/src/tests/proton/matchengine/matchengine.cpp
+++ b/searchcore/src/tests/proton/matchengine/matchengine.cpp
@@ -18,18 +18,17 @@ class MySearchHandler : public ISearchHandler {
std::string _reply;
public:
MySearchHandler(size_t numHits = 0) :
- _numHits(numHits), _name("my"), _reply("myreply") {}
- virtual DocsumReply::UP getDocsums(const DocsumRequest &) override {
- return DocsumReply::UP(new DocsumReply);
+ _numHits(numHits), _name("my"), _reply("myreply")
+ {}
+ DocsumReply::UP getDocsums(const DocsumRequest &) override {
+ return std::make_unique<DocsumReply>();
}
- virtual search::engine::SearchReply::UP match(
- const ISearchHandler::SP &,
- const search::engine::SearchRequest &,
- vespalib::ThreadBundle &) const override {
- SearchReply::UP retval(new SearchReply);
+ SearchReply::UP match(const SearchRequest &, vespalib::ThreadBundle &) const override
+ {
+ auto retval = std::make_unique<SearchReply>();
for (size_t i = 0; i < _numHits; ++i) {
- retval->hits.push_back(SearchReply::Hit());
+ retval->hits.emplace_back();
}
return retval;
}
@@ -43,7 +42,7 @@ private:
public:
LocalSearchClient();
- ~LocalSearchClient();
+ ~LocalSearchClient() override;
void searchDone(SearchReply::UP reply) override {
std::lock_guard<std::mutex> guard(_lock);
_reply = std::move(reply);
@@ -62,8 +61,8 @@ public:
}
};
-LocalSearchClient::LocalSearchClient() {}
-LocalSearchClient::~LocalSearchClient() {}
+LocalSearchClient::LocalSearchClient() = default;
+LocalSearchClient::~LocalSearchClient() = default;
TEST("requireThatSearchesExecute")
{
@@ -71,23 +70,23 @@ TEST("requireThatSearchesExecute")
MatchEngine engine(numMatcherThreads, 1, 7);
engine.setNodeUp(true);
- MySearchHandler::SP handler(new MySearchHandler);
+ auto handler = std::make_shared<MySearchHandler>();
DocTypeName dtnvfoo("foo");
engine.putSearchHandler(dtnvfoo, handler);
LocalSearchClient client;
SearchRequest::Source request(new SearchRequest());
SearchReply::UP reply = engine.search(std::move(request), client);
- EXPECT_TRUE(reply.get() == NULL);
+ EXPECT_FALSE(reply);
reply = client.getReply(10000);
- EXPECT_TRUE(reply.get() != NULL);
+ EXPECT_TRUE(reply);
}
bool
assertSearchReply(MatchEngine & engine, const std::string & searchDocType, size_t expHits)
{
- SearchRequest *request = new SearchRequest();
+ auto *request = new SearchRequest();
request->propertiesMap.lookupCreate(search::MapNames::MATCH).add("documentdb.searchdoctype", searchDocType);
LocalSearchClient client;
engine.search(SearchRequest::Source(request), client);
@@ -99,9 +98,9 @@ TEST("requireThatCorrectHandlerIsUsed")
{
MatchEngine engine(1, 1, 7);
engine.setNodeUp(true);
- ISearchHandler::SP h1(new MySearchHandler(2));
- ISearchHandler::SP h2(new MySearchHandler(4));
- ISearchHandler::SP h3(new MySearchHandler(6));
+ auto h1 = std::make_shared<MySearchHandler>(2);
+ auto h2 = std::make_shared<MySearchHandler>(4);
+ auto h3 = std::make_shared<MySearchHandler>(6);
DocTypeName dtnvfoo("foo");
DocTypeName dtnvbar("bar");
DocTypeName dtnvbaz("baz");
@@ -120,13 +119,12 @@ struct ObserveBundleMatchHandler : MySearchHandler {
mutable size_t bundleSize;
ObserveBundleMatchHandler() : bundleSize(0) {}
- virtual search::engine::SearchReply::UP match(
- const ISearchHandler::SP &,
+ search::engine::SearchReply::UP match(
const search::engine::SearchRequest &,
vespalib::ThreadBundle &threadBundle) const override
{
bundleSize = threadBundle.size();
- return SearchReply::UP(new SearchReply);
+ return std::make_unique<SearchReply>();
}
};
@@ -135,7 +133,7 @@ TEST("requireThatBundlesAreUsed")
MatchEngine engine(15, 5, 7);
engine.setNodeUp(true);
- ObserveBundleMatchHandler::SP handler(new ObserveBundleMatchHandler());
+ auto handler = std::make_shared<ObserveBundleMatchHandler>();
DocTypeName dtnvfoo("foo");
engine.putSearchHandler(dtnvfoo, handler);
@@ -151,20 +149,20 @@ TEST("requireThatHandlersCanBeRemoved")
{
MatchEngine engine(1, 1, 7);
engine.setNodeUp(true);
- ISearchHandler::SP h(new MySearchHandler(1));
+ auto h = std::make_shared<MySearchHandler>(1);
DocTypeName docType("foo");
engine.putSearchHandler(docType, h);
ISearchHandler::SP r = engine.getSearchHandler(docType);
- EXPECT_TRUE(r.get() != NULL);
+ EXPECT_TRUE(r);
EXPECT_TRUE(h.get() == r.get());
r = engine.removeSearchHandler(docType);
- EXPECT_TRUE(r.get() != NULL);
+ EXPECT_TRUE(r);
EXPECT_TRUE(h.get() == r.get());
r = engine.getSearchHandler(docType);
- EXPECT_TRUE(r.get() == NULL);
+ EXPECT_FALSE(r);
}
TEST("requireThatEmptySearchReplyIsReturnedWhenEngineIsClosed")
@@ -175,7 +173,7 @@ TEST("requireThatEmptySearchReplyIsReturnedWhenEngineIsClosed")
LocalSearchClient client;
SearchRequest::Source request(new SearchRequest());
SearchReply::UP reply = engine.search(std::move(request), client);
- EXPECT_TRUE(reply.get() != NULL);
+ EXPECT_TRUE(reply );
EXPECT_EQUAL(0u, reply->hits.size());
EXPECT_EQUAL(7u, reply->getDistributionKey());
}
diff --git a/searchcore/src/tests/proton/matching/matching_test.cpp b/searchcore/src/tests/proton/matching/matching_test.cpp
index 95ab43dbcba..00a319db394 100644
--- a/searchcore/src/tests/proton/matching/matching_test.cpp
+++ b/searchcore/src/tests/proton/matching/matching_test.cpp
@@ -289,7 +289,7 @@ struct MyWorld {
DocsumReply::UP getDocsums(const DocsumRequest &) override {
return DocsumReply::UP();
}
- SearchReply::UP match(const ISearchHandler::SP &, const SearchRequest &, vespalib::ThreadBundle &) const override {
+ SearchReply::UP match(const SearchRequest &, vespalib::ThreadBundle &) const override {
return SearchReply::UP();
}
};
diff --git a/searchcore/src/tests/proton/summaryengine/summaryengine.cpp b/searchcore/src/tests/proton/summaryengine/summaryengine.cpp
index a9cae7d8ab7..7cdd8d767c6 100644
--- a/searchcore/src/tests/proton/summaryengine/summaryengine.cpp
+++ b/searchcore/src/tests/proton/summaryengine/summaryengine.cpp
@@ -42,7 +42,7 @@ public:
: _name(name), _reply(reply)
{}
- virtual DocsumReply::UP getDocsums(const DocsumRequest &request) override {
+ DocsumReply::UP getDocsums(const DocsumRequest &request) override {
return (request.useRootSlime())
? std::make_unique<DocsumReply>(createSlimeReply(request.hits.size()))
: createOldDocSum(request);
@@ -62,7 +62,7 @@ public:
}
DocsumReply::UP createOldDocSum(const DocsumRequest &request) {
- DocsumReply::UP retval(new DocsumReply());
+ auto retval = std::make_unique<DocsumReply>();
for (size_t i = 0; i < request.hits.size(); i++) {
const DocsumRequest::Hit &h = request.hits[i];
DocsumReply::Docsum docsum;
@@ -74,11 +74,8 @@ public:
return retval;
}
- virtual search::engine::SearchReply::UP match(
- const ISearchHandler::SP &,
- const search::engine::SearchRequest &,
- vespalib::ThreadBundle &) const override {
- return SearchReply::UP(new SearchReply);
+ SearchReply::UP match(const SearchRequest &, vespalib::ThreadBundle &) const override {
+ return std::make_unique<SearchReply>();
}
};
@@ -90,8 +87,7 @@ private:
public:
MyDocsumClient();
-
- ~MyDocsumClient();
+ ~MyDocsumClient() override;
void getDocsumsDone(DocsumReply::UP reply) override {
std::lock_guard<std::mutex> guard(_lock);
@@ -111,19 +107,19 @@ public:
}
};
-MyDocsumClient::MyDocsumClient() {}
+MyDocsumClient::MyDocsumClient() = default;
-MyDocsumClient::~MyDocsumClient() {}
+MyDocsumClient::~MyDocsumClient() = default;
DocsumRequest::UP
createRequest(size_t num = 1) {
- DocsumRequest::UP r(new DocsumRequest());
+ auto r = std::make_unique<DocsumRequest>();
if (num == 1) {
r->hits.emplace_back(GlobalId("aaaaaaaaaaaa"));
} else {
for (size_t i = 0; i < num; i++) {
vespalib::string s = vespalib::make_string("aaaaaaaaaaa%c", char('a' + i % 26));
- r->hits.push_back(GlobalId(s.c_str()));
+ r->hits.emplace_back(GlobalId(s.c_str()));
}
}
return r;
@@ -132,7 +128,7 @@ createRequest(size_t num = 1) {
TEST("requireThatGetDocsumsExecute") {
int numSummaryThreads = 2;
SummaryEngine engine(numSummaryThreads);
- ISearchHandler::SP handler(new MySearchHandler);
+ auto handler = std::make_shared<MySearchHandler>();
DocTypeName dtnvfoo("foo");
engine.putSearchHandler(dtnvfoo, handler);
@@ -140,9 +136,9 @@ TEST("requireThatGetDocsumsExecute") {
{ // async call when engine running
DocsumRequest::Source request(createRequest());
DocsumReply::UP reply = engine.getDocsums(std::move(request), client);
- EXPECT_TRUE(reply.get() == NULL);
+ EXPECT_FALSE(reply);
reply = client.getReply(10000);
- EXPECT_TRUE(reply.get() != NULL);
+ EXPECT_TRUE(reply);
EXPECT_EQUAL(1u, reply->docsums.size());
EXPECT_EQUAL(10u, reply->docsums[0].docid);
EXPECT_EQUAL(GlobalId("aaaaaaaaaaaa"), reply->docsums[0].gid);
@@ -152,7 +148,7 @@ TEST("requireThatGetDocsumsExecute") {
{ // sync call when engine closed
DocsumRequest::Source request(createRequest());
DocsumReply::UP reply = engine.getDocsums(std::move(request), client);
- EXPECT_TRUE(reply.get() != NULL);
+ EXPECT_TRUE(reply);
}
}
@@ -161,23 +157,23 @@ TEST("requireThatHandlersAreStored") {
DocTypeName dtnvbar("bar");
int numSummaryThreads = 2;
SummaryEngine engine(numSummaryThreads);
- ISearchHandler::SP h1(new MySearchHandler("foo"));
- ISearchHandler::SP h2(new MySearchHandler("bar"));
- ISearchHandler::SP h3(new MySearchHandler("baz"));
+ auto h1 = std::make_shared<MySearchHandler>("foo");
+ auto h2 = std::make_shared<MySearchHandler>("bar");
+ auto h3 = std::make_shared<MySearchHandler>("baz");
// not found
- EXPECT_TRUE(engine.getSearchHandler(dtnvfoo).get() == NULL);
- EXPECT_TRUE(engine.removeSearchHandler(dtnvfoo).get() == NULL);
+ EXPECT_FALSE(engine.getSearchHandler(dtnvfoo));
+ EXPECT_FALSE(engine.removeSearchHandler(dtnvfoo));
// put & get
- EXPECT_TRUE(engine.putSearchHandler(dtnvfoo, h1).get() == NULL);
+ EXPECT_FALSE(engine.putSearchHandler(dtnvfoo, h1));
EXPECT_EQUAL(engine.getSearchHandler(dtnvfoo).get(), h1.get());
- EXPECT_TRUE(engine.putSearchHandler(dtnvbar, h2).get() == NULL);
+ EXPECT_FALSE(engine.putSearchHandler(dtnvbar, h2));
EXPECT_EQUAL(engine.getSearchHandler(dtnvbar).get(), h2.get());
// replace
EXPECT_TRUE(engine.putSearchHandler(dtnvfoo, h3).get() == h1.get());
EXPECT_EQUAL(engine.getSearchHandler(dtnvfoo).get(), h3.get());
// remove
EXPECT_EQUAL(engine.removeSearchHandler(dtnvfoo).get(), h3.get());
- EXPECT_TRUE(engine.getSearchHandler(dtnvfoo).get() == NULL);
+ EXPECT_FALSE(engine.getSearchHandler(dtnvfoo));
}
bool
@@ -196,9 +192,9 @@ TEST("requireThatCorrectHandlerIsUsed") {
DocTypeName dtnvbar("bar");
DocTypeName dtnvbaz("baz");
SummaryEngine engine(1);
- ISearchHandler::SP h1(new MySearchHandler("foo", "foo reply"));
- ISearchHandler::SP h2(new MySearchHandler("bar", "bar reply"));
- ISearchHandler::SP h3(new MySearchHandler("baz", "baz reply"));
+ auto h1 = std::make_shared<MySearchHandler>("foo", "foo reply");
+ auto h2 = std::make_shared<MySearchHandler>("bar", "bar reply");
+ auto h3 = std::make_shared<MySearchHandler>("baz", "baz reply");
engine.putSearchHandler(dtnvfoo, h1);
engine.putSearchHandler(dtnvbar, h2);
engine.putSearchHandler(dtnvbaz, h3);
@@ -369,7 +365,7 @@ public:
Server::Server()
: BaseServer(),
engine(2),
- handler(new MySearchHandler("slime", stringref(buf.GetDrainPos(), buf.GetUsedLen()))),
+ handler(std::make_shared<MySearchHandler>("slime", stringref(buf.GetDrainPos(), buf.GetUsedLen()))),
docsumBySlime(engine),
docsumByRPC(docsumBySlime)
{
@@ -377,7 +373,7 @@ Server::Server()
engine.putSearchHandler(dtnvfoo, handler);
}
-Server::~Server() {}
+Server::~Server() = default;
vespalib::string
getAnswer(size_t num) {
diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.cpp
index 74e3e903540..f1035776a8f 100644
--- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.cpp
+++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.cpp
@@ -361,7 +361,7 @@ public:
_immediateCommit(immediateCommit),
_onWriteDone(onWriteDone)
{}
- ~BatchRemoveTask() override {}
+ ~BatchRemoveTask() override = default;
void run() override {
for (auto field : _writeCtx.getFields()) {
auto &attr = field.getAttribute();
@@ -469,9 +469,9 @@ AttributeWriter::internalRemove(SerialNum serialNum, DocumentIdT lid, bool immed
}
}
-AttributeWriter::AttributeWriter(const proton::IAttributeManager::SP &mgr)
- : _mgr(mgr),
- _attributeFieldWriter(mgr->getAttributeFieldWriter()),
+AttributeWriter::AttributeWriter(proton::IAttributeManager::SP mgr)
+ : _mgr(std::move(mgr)),
+ _attributeFieldWriter(_mgr->getAttributeFieldWriter()),
_writeContexts(),
_dataType(nullptr),
_hasStructFieldAttribute(false),
diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.h b/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.h
index 9e5d8f4ce5d..8c9b756cd89 100644
--- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.h
+++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.h
@@ -74,7 +74,7 @@ private:
bool immediateCommit, OnWriteDoneType onWriteDone);
public:
- AttributeWriter(const proton::IAttributeManager::SP &mgr);
+ AttributeWriter(proton::IAttributeManager::SP mgr);
~AttributeWriter();
/* Only for in tests that add attributes after AttributeWriter construction. */
diff --git a/searchcore/src/vespa/searchcore/proton/common/doctypename.cpp b/searchcore/src/vespa/searchcore/proton/common/doctypename.cpp
index 70daeeebeca..21b27ef6ffd 100644
--- a/searchcore/src/vespa/searchcore/proton/common/doctypename.cpp
+++ b/searchcore/src/vespa/searchcore/proton/common/doctypename.cpp
@@ -4,20 +4,16 @@
#include <vespa/searchlib/engine/request.h>
#include <vespa/document/datatype/documenttype.h>
-namespace proton
-{
+namespace proton {
DocTypeName::DocTypeName(const search::engine::Request &request)
: _name(request.propertiesMap.matchProperties().lookup("documentdb", "searchdoctype").get(""))
-{
-}
+{}
DocTypeName::DocTypeName(const document::DocumentType &docType)
: _name(docType.getName())
-{
-}
+{}
-
-} // namespace proton
+}
diff --git a/searchcore/src/vespa/searchcore/proton/common/handlermap.hpp b/searchcore/src/vespa/searchcore/proton/common/handlermap.hpp
index 3fb32a9d1e0..e762ac527f0 100644
--- a/searchcore/src/vespa/searchcore/proton/common/handlermap.hpp
+++ b/searchcore/src/vespa/searchcore/proton/common/handlermap.hpp
@@ -53,7 +53,6 @@ public:
bool valid() const override { return (_offset < _handlers.size()); }
T *get() const override { return _handlers[_offset].get(); }
- HandlerSP getSP() const { return _handlers[_offset]; }
void next() override { ++_offset; }
};
diff --git a/searchcore/src/vespa/searchcore/proton/matchengine/imatchhandler.h b/searchcore/src/vespa/searchcore/proton/matchengine/imatchhandler.h
deleted file mode 100644
index 63283de0a4a..00000000000
--- a/searchcore/src/vespa/searchcore/proton/matchengine/imatchhandler.h
+++ /dev/null
@@ -1,46 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-#pragma once
-
-#include <vespa/searchcore/proton/summaryengine/isearchhandler.h>
-#include <vespa/vespalib/util/thread_bundle.h>
-
-namespace searche {
-namespace engine {
- class SearchRequest;
- class SearchReply;
-}
-}
-
-namespace proton {
-
-/**
- * This interface describes a sync match operation handler. It is implemented by
- * the DocumentDB class, and used by the MatchEngine class to delegate
- * operations to the appropriate db.
- */
-class IMatchHandler {
-protected:
- using SearchReply = search::engine::SearchReply;
- using SearchRequest = search::engine::SearchRequest;
- using ThreadBundle = vespalib::ThreadBundle;
- IMatchHandler() = default;
-public:
- IMatchHandler(const IMatchHandler &) = delete;
- IMatchHandler & operator = (const IMatchHandler &) = delete;
- /**
- * Convenience typedefs.
- */
- typedef std::unique_ptr<IMatchHandler> UP;
- typedef std::shared_ptr<IMatchHandler> SP;
-
- virtual ~IMatchHandler() { }
-
- /**
- * @return Use the request and produce the matching result.
- */
- virtual std::unique_ptr<SearchReply>
- match(const ISearchHandler::SP &searchHandler, const SearchRequest &req, ThreadBundle &threadBundle) const = 0;
-};
-
-} // namespace proton
-
diff --git a/searchcore/src/vespa/searchcore/proton/matchengine/matchengine.cpp b/searchcore/src/vespa/searchcore/proton/matchengine/matchengine.cpp
index b170f60d71f..ad47fb98232 100644
--- a/searchcore/src/vespa/searchcore/proton/matchengine/matchengine.cpp
+++ b/searchcore/src/vespa/searchcore/proton/matchengine/matchengine.cpp
@@ -121,12 +121,12 @@ MatchEngine::performSearch(search::engine::SearchRequest::Source req,
ISearchHandler::SP searchHandler;
vespalib::SimpleThreadBundle::UP threadBundle = _threadBundlePool.obtain();
{ // try to find the match handler corresponding to the specified search doc type
- std::lock_guard<std::mutex> guard(_lock);
DocTypeName docTypeName(*searchRequest);
+ std::lock_guard<std::mutex> guard(_lock);
searchHandler = _handlers.getHandler(docTypeName);
}
if (searchHandler) {
- ret = searchHandler->match(searchHandler, *searchRequest, *threadBundle);
+ ret = searchHandler->match(*searchRequest, *threadBundle);
} else {
HandlerMap<ISearchHandler>::Snapshot snapshot;
{
@@ -134,8 +134,7 @@ MatchEngine::performSearch(search::engine::SearchRequest::Source req,
snapshot = _handlers.snapshot();
}
if (snapshot.valid()) {
- ISearchHandler::SP handler = snapshot.getSP();
- ret = handler->match(handler, *searchRequest, *threadBundle); // use the first handler
+ ret = snapshot.get()->match(*searchRequest, *threadBundle); // use the first handler
}
}
_threadBundlePool.release(std::move(threadBundle));
diff --git a/searchcore/src/vespa/searchcore/proton/matchengine/matchengine.h b/searchcore/src/vespa/searchcore/proton/matchengine/matchengine.h
index 896ac83403a..34aacdafcad 100644
--- a/searchcore/src/vespa/searchcore/proton/matchengine/matchengine.h
+++ b/searchcore/src/vespa/searchcore/proton/matchengine/matchengine.h
@@ -1,7 +1,7 @@
// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
#pragma once
-#include "imatchhandler.h"
+#include <vespa/searchcore/proton/summaryengine/isearchhandler.h>
#include <vespa/searchcore/proton/common/doctypename.h>
#include <vespa/searchcore/proton/common/handlermap.hpp>
#include <vespa/searchcore/proton/common/statusreport.h>
@@ -50,7 +50,7 @@ public:
* Frees any allocated resources. this will also stop all internal threads
* and wait for them to finish. All pending search requests are deleted.
*/
- ~MatchEngine();
+ ~MatchEngine() override;
/**
* Observe and reset internal executor stats
@@ -123,13 +123,11 @@ public:
StatusReport::UP reportStatus() const;
- // Implements SearchServer.
search::engine::SearchReply::UP search(
search::engine::SearchRequest::Source request,
search::engine::SearchClient &client) override;
- // Implements vespalib::StateExplorer
- virtual void get_state(const vespalib::slime::Inserter &inserter, bool full) const override;
+ void get_state(const vespalib::slime::Inserter &inserter, bool full) const override;
};
} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/matching/match_thread.cpp b/searchcore/src/vespa/searchcore/proton/matching/match_thread.cpp
index 7c5e7584eed..0e80d31a063 100644
--- a/searchcore/src/vespa/searchcore/proton/matching/match_thread.cpp
+++ b/searchcore/src/vespa/searchcore/proton/matching/match_thread.cpp
@@ -35,7 +35,7 @@ namespace {
struct WaitTimer {
double &wait_time_s;
vespalib::Timer wait_time;
- WaitTimer(double &wait_time_s_in)
+ explicit WaitTimer(double &wait_time_s_in)
: wait_time_s(wait_time_s_in), wait_time()
{ }
void done() {
diff --git a/searchcore/src/vespa/searchcore/proton/matching/search_session.h b/searchcore/src/vespa/searchcore/proton/matching/search_session.h
index 0aec02e9d31..13d24624597 100644
--- a/searchcore/src/vespa/searchcore/proton/matching/search_session.h
+++ b/searchcore/src/vespa/searchcore/proton/matching/search_session.h
@@ -26,7 +26,7 @@ public:
OwnershipBundle(OwnershipBundle &&) = default;
OwnershipBundle & operator = (OwnershipBundle &&) = default;
~OwnershipBundle();
- ISearchHandler::SP search_handler;
+ std::shared_ptr<const ISearchHandler> search_handler;
std::unique_ptr<search::fef::Properties> feature_overrides;
std::unique_ptr<MatchContext> context;
IDocumentMetaStoreContext::IReadGuard::UP readGuard;
diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp b/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp
index 1b182c3e618..302ffc93f6a 100644
--- a/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp
@@ -756,11 +756,11 @@ DocumentDB::getNewestFlushedSerial()
}
std::unique_ptr<SearchReply>
-DocumentDB::match(const ISearchHandler::SP &, const SearchRequest &req, vespalib::ThreadBundle &threadBundle) const
+DocumentDB::match(const SearchRequest &req, vespalib::ThreadBundle &threadBundle) const
{
// Ignore input searchhandler. Use readysubdb's searchhandler instead.
ISearchHandler::SP view(_subDBs.getReadySubDB()->getSearchView());
- return view->match(view, req, threadBundle);
+ return view->match(req, threadBundle);
}
std::unique_ptr<DocsumReply>
diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdb.h b/searchcore/src/vespa/searchcore/proton/server/documentdb.h
index 02ad144af68..f296e264903 100644
--- a/searchcore/src/vespa/searchcore/proton/server/documentdb.h
+++ b/searchcore/src/vespa/searchcore/proton/server/documentdb.h
@@ -368,8 +368,7 @@ public:
virtual SerialNum getNewestFlushedSerial();
std::unique_ptr<search::engine::SearchReply>
- match(const ISearchHandler::SP &searchHandler,
- const search::engine::SearchRequest &req,
+ match(const search::engine::SearchRequest &req,
vespalib::ThreadBundle &threadBundle) const;
std::unique_ptr<search::engine::DocsumReply>
diff --git a/searchcore/src/vespa/searchcore/proton/server/emptysearchview.cpp b/searchcore/src/vespa/searchcore/proton/server/emptysearchview.cpp
index 90cf0011685..9f2ec26ad4b 100644
--- a/searchcore/src/vespa/searchcore/proton/server/emptysearchview.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/emptysearchview.cpp
@@ -14,10 +14,7 @@ using search::engine::SearchRequest;
namespace proton {
-EmptySearchView::EmptySearchView()
- : ISearchHandler()
-{
-}
+EmptySearchView::EmptySearchView() = default;
DocsumReply::UP
@@ -25,20 +22,16 @@ EmptySearchView::getDocsums(const DocsumRequest &req)
{
LOG(debug, "getDocsums(): resultClass(%s), numHits(%zu)",
req.resultClassName.c_str(), req.hits.size());
- DocsumReply::UP reply(new DocsumReply());
- for (size_t i = 0; i < req.hits.size(); ++i) {
- reply->docsums.push_back(DocsumReply::Docsum());
- reply->docsums.back().gid = req.hits[i].gid;
+ auto reply = std::make_unique<DocsumReply>();
+ for (const auto & hit : req.hits) {
+ reply->docsums.emplace_back(hit.gid);
}
return reply;
}
SearchReply::UP
-EmptySearchView::match(const ISearchHandler::SP &,
- const SearchRequest &,
- vespalib::ThreadBundle &) const {
- SearchReply::UP reply(new SearchReply);
- return reply;
+EmptySearchView::match(const SearchRequest &, vespalib::ThreadBundle &) const {
+ return std::make_unique<SearchReply>();
}
diff --git a/searchcore/src/vespa/searchcore/proton/server/emptysearchview.h b/searchcore/src/vespa/searchcore/proton/server/emptysearchview.h
index a3e2a2b176c..92fb97f6177 100644
--- a/searchcore/src/vespa/searchcore/proton/server/emptysearchview.h
+++ b/searchcore/src/vespa/searchcore/proton/server/emptysearchview.h
@@ -13,15 +13,10 @@ public:
EmptySearchView();
- /**
- * Implements ISearchHandler
- */
- virtual std::unique_ptr<DocsumReply> getDocsums(const DocsumRequest & req) override;
-
- virtual std::unique_ptr<SearchReply>
- match(const ISearchHandler::SP &searchHandler,
- const SearchRequest &req,
- vespalib::ThreadBundle &threadBundle) const override;
+ std::unique_ptr<DocsumReply> getDocsums(const DocsumRequest & req) override;
+
+ std::unique_ptr<SearchReply>
+ match(const SearchRequest &req, vespalib::ThreadBundle &threadBundle) const override;
};
} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/server/matchview.cpp b/searchcore/src/vespa/searchcore/proton/server/matchview.cpp
index 6f32f886637..7ba9b971715 100644
--- a/searchcore/src/vespa/searchcore/proton/server/matchview.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/matchview.cpp
@@ -31,17 +31,17 @@ using matching::ISearchContext;
using matching::Matcher;
using matching::SessionManager;
-MatchView::MatchView(const Matchers::SP &matchers,
- const IndexSearchable::SP &indexSearchable,
- const IAttributeManager::SP &attrMgr,
- const SessionManagerSP &sessionMgr,
- const IDocumentMetaStoreContext::SP &metaStore,
+MatchView::MatchView(Matchers::SP matchers,
+ IndexSearchable::SP indexSearchable,
+ IAttributeManager::SP attrMgr,
+ SessionManagerSP sessionMgr,
+ IDocumentMetaStoreContext::SP metaStore,
DocIdLimit &docIdLimit)
- : _matchers(matchers),
- _indexSearchable(indexSearchable),
- _attrMgr(attrMgr),
- _sessionMgr(sessionMgr),
- _metaStore(metaStore),
+ : _matchers(std::move(matchers)),
+ _indexSearchable(std::move(indexSearchable)),
+ _attrMgr(std::move(attrMgr)),
+ _sessionMgr(std::move(sessionMgr)),
+ _metaStore(std::move(metaStore)),
_docIdLimit(docIdLimit)
{ }
@@ -68,12 +68,12 @@ MatchView::createContext() const {
std::unique_ptr<SearchReply>
-MatchView::match(const ISearchHandler::SP &searchHandler, const SearchRequest &req,
+MatchView::match(std::shared_ptr<const ISearchHandler> searchHandler, const SearchRequest &req,
vespalib::ThreadBundle &threadBundle) const
{
Matcher::SP matcher = getMatcher(req.ranking);
SearchSession::OwnershipBundle owned_objects;
- owned_objects.search_handler = searchHandler;
+ owned_objects.search_handler = std::move(searchHandler);
owned_objects.context = createContext();
owned_objects.readGuard = _metaStore->getReadGuard();;
MatchContext *ctx = owned_objects.context.get();
diff --git a/searchcore/src/vespa/searchcore/proton/server/matchview.h b/searchcore/src/vespa/searchcore/proton/server/matchview.h
index 983e06d3414..721cb439508 100644
--- a/searchcore/src/vespa/searchcore/proton/server/matchview.h
+++ b/searchcore/src/vespa/searchcore/proton/server/matchview.h
@@ -36,11 +36,11 @@ public:
MatchView(const MatchView &) = delete;
MatchView & operator = (const MatchView &) = delete;
- MatchView(const Matchers::SP &matchers,
- const searchcorespi::IndexSearchable::SP &indexSearchable,
- const IAttributeManager::SP &attrMgr,
- const SessionManagerSP &sessionMgr,
- const IDocumentMetaStoreContext::SP &metaStore,
+ MatchView(Matchers::SP matchers,
+ searchcorespi::IndexSearchable::SP indexSearchable,
+ IAttributeManager::SP attrMgr,
+ SessionManagerSP sessionMgr,
+ IDocumentMetaStoreContext::SP metaStore,
DocIdLimit &docIdLimit);
~MatchView();
@@ -62,7 +62,7 @@ public:
matching::MatchContext::UP createContext() const;
std::unique_ptr<search::engine::SearchReply>
- match(const std::shared_ptr<ISearchHandler> &searchHandler,
+ match(std::shared_ptr<const ISearchHandler> searchHandler,
const search::engine::SearchRequest &req,
vespalib::ThreadBundle &threadBundle) const;
};
diff --git a/searchcore/src/vespa/searchcore/proton/server/proton.cpp b/searchcore/src/vespa/searchcore/proton/server/proton.cpp
index 28de4dff917..20de5bb07c1 100644
--- a/searchcore/src/vespa/searchcore/proton/server/proton.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/proton.cpp
@@ -628,8 +628,7 @@ Proton::removeDocumentDB(const DocTypeName &docTypeName)
{
// Not allowed to get to service layer to call pause().
std::unique_lock<std::shared_timed_mutex> persistenceWguard(_persistenceEngine->getWLock());
- IPersistenceHandler::SP oldHandler;
- oldHandler = _persistenceEngine->removeHandler(persistenceWguard, old->getBucketSpace(), docTypeName);
+ IPersistenceHandler::SP oldHandler = _persistenceEngine->removeHandler(persistenceWguard, old->getBucketSpace(), docTypeName);
if (_initComplete && oldHandler) {
// TODO: Fix race with bucket db modifying ops.
_persistenceEngine->grabExtraModifiedBuckets(old->getBucketSpace(), *oldHandler);
diff --git a/searchcore/src/vespa/searchcore/proton/server/searchable_doc_subdb_configurer.cpp b/searchcore/src/vespa/searchcore/proton/server/searchable_doc_subdb_configurer.cpp
index 069d3727850..9ef038b7325 100644
--- a/searchcore/src/vespa/searchcore/proton/server/searchable_doc_subdb_configurer.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/searchable_doc_subdb_configurer.cpp
@@ -84,17 +84,17 @@ reconfigureMatchView(const Matchers::SP &matchers,
}
void
-SearchableDocSubDBConfigurer::reconfigureSearchView(const MatchView::SP &matchView)
+SearchableDocSubDBConfigurer::reconfigureSearchView(MatchView::SP matchView)
{
SearchView::SP curr = _searchView.get();
- _searchView.set(SearchView::SP(new SearchView(curr->getSummarySetup(), matchView)));
+ _searchView.set(SearchView::create(curr->getSummarySetup(), std::move(matchView)));
}
void
-SearchableDocSubDBConfigurer::reconfigureSearchView(const ISummaryManager::ISummarySetup::SP &summarySetup,
- const MatchView::SP &matchView)
+SearchableDocSubDBConfigurer::reconfigureSearchView(ISummaryManager::ISummarySetup::SP summarySetup,
+ MatchView::SP matchView)
{
- _searchView.set(SearchView::SP(new SearchView(summarySetup, matchView)));
+ _searchView.set(SearchView::create(std::move(summarySetup), std::move(matchView)));
}
SearchableDocSubDBConfigurer::
diff --git a/searchcore/src/vespa/searchcore/proton/server/searchable_doc_subdb_configurer.h b/searchcore/src/vespa/searchcore/proton/server/searchable_doc_subdb_configurer.h
index 5aac069853b..459c4651e67 100644
--- a/searchcore/src/vespa/searchcore/proton/server/searchable_doc_subdb_configurer.h
+++ b/searchcore/src/vespa/searchcore/proton/server/searchable_doc_subdb_configurer.h
@@ -65,11 +65,10 @@ private:
const IAttributeManager::SP &attrMgr);
void
- reconfigureSearchView(const MatchView::SP &matchView);
+ reconfigureSearchView(MatchView::SP matchView);
void
- reconfigureSearchView(const ISummaryManager::ISummarySetup::SP &summarySetup,
- const MatchView::SP &matchView);
+ reconfigureSearchView(ISummaryManager::ISummarySetup::SP summarySetup, MatchView::SP matchView);
public:
SearchableDocSubDBConfigurer(const SearchableDocSubDBConfigurer &) = delete;
diff --git a/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.cpp b/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.cpp
index 77852dcc918..3ca8a4cff49 100644
--- a/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.cpp
@@ -203,16 +203,16 @@ SearchableDocSubDB::initViews(const DocumentDBConfig &configSnapshot, const Sess
const IIndexManager::SP &indexMgr = getIndexManager();
_constantValueRepo.reconfigure(configSnapshot.getRankingConstants());
Matchers::SP matchers(_configurer.createMatchers(schema, configSnapshot.getRankProfilesConfig()).release());
- auto matchView = std::make_shared<MatchView>(matchers, indexMgr->getSearchable(), attrMgr,
+ auto matchView = std::make_shared<MatchView>(std::move(matchers), indexMgr->getSearchable(), attrMgr,
sessionManager, _metaStoreCtx, _docIdLimit);
- _rSearchView.set(std::make_shared<SearchView>(
+ _rSearchView.set(SearchView::create(
getSummaryManager()->createSummarySetup(
configSnapshot.getSummaryConfig(),
configSnapshot.getSummarymapConfig(),
configSnapshot.getJuniperrcConfig(),
configSnapshot.getDocumentTypeRepoSP(),
- matchView->getAttributeManager()),
- matchView));
+ attrMgr),
+ std::move(matchView)));
auto attrWriter = std::make_shared<AttributeWriter>(attrMgr);
{
diff --git a/searchcore/src/vespa/searchcore/proton/server/searchhandlerproxy.cpp b/searchcore/src/vespa/searchcore/proton/server/searchhandlerproxy.cpp
index f611b4e1f4c..6da6c09cdba 100644
--- a/searchcore/src/vespa/searchcore/proton/server/searchhandlerproxy.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/searchhandlerproxy.cpp
@@ -7,8 +7,8 @@
namespace proton {
-SearchHandlerProxy::SearchHandlerProxy(const DocumentDB::SP &documentDB)
- : _documentDB(documentDB)
+SearchHandlerProxy::SearchHandlerProxy(DocumentDB::SP documentDB)
+ : _documentDB(std::move(documentDB))
{
_documentDB->retain();
}
@@ -25,11 +25,9 @@ SearchHandlerProxy::getDocsums(const DocsumRequest & request)
}
std::unique_ptr<search::engine::SearchReply>
-SearchHandlerProxy::match(const ISearchHandler::SP &searchHandler,
- const SearchRequest &req,
- vespalib::ThreadBundle &threadBundle) const
+SearchHandlerProxy::match(const SearchRequest &req, vespalib::ThreadBundle &threadBundle) const
{
- return _documentDB->match(searchHandler, req, threadBundle);
+ return _documentDB->match(req, threadBundle);
}
} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/server/searchhandlerproxy.h b/searchcore/src/vespa/searchcore/proton/server/searchhandlerproxy.h
index 4e9f5bdab8a..fc4f517fb36 100644
--- a/searchcore/src/vespa/searchcore/proton/server/searchhandlerproxy.h
+++ b/searchcore/src/vespa/searchcore/proton/server/searchhandlerproxy.h
@@ -13,11 +13,11 @@ class SearchHandlerProxy : public ISearchHandler
private:
std::shared_ptr<DocumentDB> _documentDB;
public:
- SearchHandlerProxy(const std::shared_ptr<DocumentDB> &documentDB);
+ SearchHandlerProxy(std::shared_ptr<DocumentDB> documentDB);
- virtual~SearchHandlerProxy();
+ ~SearchHandlerProxy() override;
std::unique_ptr<DocsumReply> getDocsums(const DocsumRequest & request) override;
- std::unique_ptr<SearchReply> match(const ISearchHandler::SP &searchHandler, const SearchRequest &req, ThreadBundle &threadBundle) const override;
+ std::unique_ptr<SearchReply> match(const SearchRequest &req, ThreadBundle &threadBundle) const override;
};
} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/server/searchview.cpp b/searchcore/src/vespa/searchcore/proton/server/searchview.cpp
index 9830048cdd8..36d873d9948 100644
--- a/searchcore/src/vespa/searchcore/proton/server/searchview.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/searchview.cpp
@@ -105,11 +105,14 @@ createEmptyReply(const DocsumRequest & request)
}
-SearchView::SearchView(const ISummaryManager::ISummarySetup::SP & summarySetup,
- const MatchView::SP & matchView)
+std::shared_ptr<SearchView>
+SearchView::create(ISummaryManager::ISummarySetup::SP summarySetup, MatchView::SP matchView) {
+ return std::shared_ptr<SearchView>( new SearchView(std::move(summarySetup), std::move(matchView)));
+}
+SearchView::SearchView(ISummaryManager::ISummarySetup::SP summarySetup, MatchView::SP matchView)
: ISearchHandler(),
- _summarySetup(summarySetup),
- _matchView(matchView)
+ _summarySetup(std::move(summarySetup)),
+ _matchView(std::move(matchView))
{ }
SearchView::~SearchView() = default;
@@ -161,8 +164,8 @@ SearchView::getDocsumsInternal(const DocsumRequest & req)
}
std::unique_ptr<SearchReply>
-SearchView::match(const ISearchHandler::SP &self, const SearchRequest &req, ThreadBundle &threadBundle) const {
- return _matchView->match(self, req, threadBundle);
+SearchView::match(const SearchRequest &req, ThreadBundle &threadBundle) const {
+ return _matchView->match(shared_from_this(), req, threadBundle);
}
} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/server/searchview.h b/searchcore/src/vespa/searchcore/proton/server/searchview.h
index 28c7ffd2b36..630d29c32fd 100644
--- a/searchcore/src/vespa/searchcore/proton/server/searchview.h
+++ b/searchcore/src/vespa/searchcore/proton/server/searchview.h
@@ -8,7 +8,7 @@
namespace proton {
-class SearchView : public ISearchHandler
+class SearchView : public ISearchHandler, public std::enable_shared_from_this<SearchView>
{
public:
using SessionManagerSP = std::shared_ptr<matching::SessionManager>;
@@ -16,12 +16,12 @@ public:
using InternalDocsumReply = std::pair<std::unique_ptr<DocsumReply>, bool>;
typedef std::shared_ptr<SearchView> SP;
- SearchView(const ISummaryManager::ISummarySetup::SP &summarySetup, const MatchView::SP &matchView);
+ static std::shared_ptr<SearchView> create(ISummaryManager::ISummarySetup::SP summarySetup, MatchView::SP matchView);
SearchView(const SearchView &) = delete;
SearchView(SearchView &&) = delete;
SearchView &operator=(const SearchView &) = delete;
SearchView &operator=(SearchView &&) = delete;
- ~SearchView();
+ ~SearchView() override;
const ISummaryManager::ISummarySetup::SP & getSummarySetup() const { return _summarySetup; }
const MatchView::SP & getMatchView() const { return _matchView; }
@@ -34,8 +34,9 @@ public:
matching::MatchingStats getMatcherStats(const vespalib::string &rankProfile) const { return _matchView->getMatcherStats(rankProfile); }
std::unique_ptr<DocsumReply> getDocsums(const DocsumRequest & req) override;
- std::unique_ptr<SearchReply> match(const ISearchHandler::SP &self, const SearchRequest &req, vespalib::ThreadBundle &threadBundle) const override;
+ std::unique_ptr<SearchReply> match(const SearchRequest &req, vespalib::ThreadBundle &threadBundle) const override;
private:
+ SearchView(ISummaryManager::ISummarySetup::SP summarySetup, MatchView::SP matchView);
InternalDocsumReply getDocsumsInternal(const DocsumRequest & req);
ISummaryManager::ISummarySetup::SP _summarySetup;
MatchView::SP _matchView;
diff --git a/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.h b/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.h
index 9d5b8c18d01..14f62513c34 100644
--- a/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.h
+++ b/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.h
@@ -13,7 +13,6 @@
#include <vespa/searchcore/proton/documentmetastore/documentmetastorecontext.h>
#include <vespa/searchcore/proton/documentmetastore/documentmetastoreflushtarget.h>
#include <vespa/searchcore/proton/documentmetastore/ilidreusedelayer.h>
-#include <vespa/searchcore/proton/matchengine/imatchhandler.h>
#include <vespa/searchcore/proton/summaryengine/isearchhandler.h>
#include <vespa/searchcore/proton/common/commit_time_tracker.h>
#include <vespa/searchcore/proton/persistenceengine/i_document_retriever.h>
diff --git a/searchcore/src/vespa/searchcore/proton/server/threading_service_config.cpp b/searchcore/src/vespa/searchcore/proton/server/threading_service_config.cpp
index 8f1c3560e9b..55aa1a20ef6 100644
--- a/searchcore/src/vespa/searchcore/proton/server/threading_service_config.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/threading_service_config.cpp
@@ -22,7 +22,10 @@ namespace {
uint32_t
calculateIndexingThreads(uint32_t cfgIndexingThreads, double concurrency, const HwInfo::Cpu &cpuInfo)
{
- double scaledCores = cpuInfo.cores() * concurrency;
+ // We are capping at 12 threads to reduce cost of waking up threads
+ // to achieve a better throughput.
+ // TODO: Fix this in a simpler/better way.
+ double scaledCores = std::min(12.0, cpuInfo.cores() * concurrency);
uint32_t indexingThreads = std::max((uint32_t)std::ceil(scaledCores / 3), cfgIndexingThreads);
return std::max(indexingThreads, 1u);
}
diff --git a/searchcore/src/vespa/searchcore/proton/summaryengine/isearchhandler.h b/searchcore/src/vespa/searchcore/proton/summaryengine/isearchhandler.h
index ffcdfbaf365..b7dd438ae29 100644
--- a/searchcore/src/vespa/searchcore/proton/summaryengine/isearchhandler.h
+++ b/searchcore/src/vespa/searchcore/proton/summaryengine/isearchhandler.h
@@ -26,12 +26,11 @@ protected:
using DocsumRequest = search::engine::DocsumRequest;
using ThreadBundle = vespalib::ThreadBundle;
public:
- typedef std::unique_ptr<ISearchHandler> UP;
typedef std::shared_ptr<ISearchHandler> SP;
ISearchHandler(const ISearchHandler &) = delete;
ISearchHandler & operator = (const ISearchHandler &) = delete;
- virtual ~ISearchHandler() { }
+ virtual ~ISearchHandler() = default;
/**
* @return Use the request and produce the document summary result.
@@ -39,7 +38,7 @@ public:
virtual std::unique_ptr<DocsumReply> getDocsums(const DocsumRequest & request) = 0;
virtual std::unique_ptr<SearchReply>
- match(const ISearchHandler::SP &self, const SearchRequest &req, ThreadBundle &threadBundle) const = 0;
+ match(const SearchRequest &req, ThreadBundle &threadBundle) const = 0;
};
} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/summaryengine/summaryengine.cpp b/searchcore/src/vespa/searchcore/proton/summaryengine/summaryengine.cpp
index 692b05899f4..1189e8a550c 100644
--- a/searchcore/src/vespa/searchcore/proton/summaryengine/summaryengine.cpp
+++ b/searchcore/src/vespa/searchcore/proton/summaryengine/summaryengine.cpp
@@ -107,13 +107,13 @@ SummaryEngine::getDocsums(DocsumRequest::Source request, DocsumClient & client)
{
if (_closed) {
LOG(warning, "Receiving docsumrequest after engine has been shutdown");
- DocsumReply::UP ret(new DocsumReply());
+ auto ret = std::make_unique<DocsumReply>();
// TODO: Notify closed.
return ret;
}
- vespalib::Executor::Task::UP task(new DocsumTask(*this, std::move(request), client));
+ auto task =std::make_unique<DocsumTask>(*this, std::move(request), client);
_executor.execute(std::move(task));
return DocsumReply::UP();
}
diff --git a/searchlib/src/apps/tests/memoryindexstress_test.cpp b/searchlib/src/apps/tests/memoryindexstress_test.cpp
index a7689cd6b9f..60f3a6b7664 100644
--- a/searchlib/src/apps/tests/memoryindexstress_test.cpp
+++ b/searchlib/src/apps/tests/memoryindexstress_test.cpp
@@ -16,7 +16,6 @@
#include <vespa/searchlib/index/i_field_length_inspector.h>
#include <vespa/searchlib/memoryindex/memory_index.h>
#include <vespa/searchlib/query/tree/simplequery.h>
-#include <vespa/searchlib/queryeval/booleanmatchiteratorwrapper.h>
#include <vespa/searchlib/queryeval/fake_requestcontext.h>
#include <vespa/searchlib/queryeval/fake_search.h>
#include <vespa/searchlib/queryeval/fake_searchable.h>
@@ -24,7 +23,6 @@
#include <vespa/searchlib/test/index/mock_field_length_inspector.h>
#include <vespa/searchlib/util/rand48.h>
#include <vespa/vespalib/testkit/testapp.h>
-#include <vespa/vespalib/util/stringfmt.h>
#include <vespa/vespalib/util/threadstackexecutor.h>
#include <vespa/log/log.h>
diff --git a/searchlib/src/tests/attribute/attributemanager/attributemanager_test.cpp b/searchlib/src/tests/attribute/attributemanager/attributemanager_test.cpp
index 7d09b2aa0b8..850a967ed3d 100644
--- a/searchlib/src/tests/attribute/attributemanager/attributemanager_test.cpp
+++ b/searchlib/src/tests/attribute/attributemanager/attributemanager_test.cpp
@@ -278,6 +278,22 @@ AttributeManagerTest::testConfigConvert()
AttributeVector::Config out = ConfigConverter::convert(a);
EXPECT_EQUAL("tensor(x[5])", out.tensorType().to_spec());
}
+ { // hnsw index params (enabled)
+ CACA a;
+ a.index.hnsw.enabled = true;
+ a.index.hnsw.maxlinkspernode = 32;
+ a.index.hnsw.neighborstoexploreatinsert = 300;
+ auto out = ConfigConverter::convert(a);
+ EXPECT_TRUE(out.hnsw_index_params().has_value());
+ EXPECT_EQUAL(32u, out.hnsw_index_params().value().max_links_per_node());
+ EXPECT_EQUAL(300u, out.hnsw_index_params().value().neighbors_to_explore_at_insert());
+ }
+ { // hnsw index params (disabled)
+ CACA a;
+ a.index.hnsw.enabled = false;
+ auto out = ConfigConverter::convert(a);
+ EXPECT_FALSE(out.hnsw_index_params().has_value());
+ }
}
bool gt_attribute(const attribute::IAttributeVector * a, const attribute::IAttributeVector * b) {
diff --git a/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp b/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp
index 7e0fcdc0ccc..5089743a54a 100644
--- a/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp
+++ b/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp
@@ -1,34 +1,48 @@
// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-#include <vespa/vespalib/testkit/test_kit.h>
#include <vespa/document/base/exceptions.h>
-#include <vespa/searchlib/tensor/tensor_attribute.h>
-#include <vespa/searchlib/tensor/generic_tensor_attribute.h>
-#include <vespa/searchlib/tensor/dense_tensor_attribute.h>
-#include <vespa/searchlib/attribute/attributeguard.h>
-#include <vespa/eval/tensor/tensor.h>
-#include <vespa/eval/tensor/dense/dense_tensor.h>
#include <vespa/eval/tensor/default_tensor_engine.h>
-#include <vespa/vespalib/io/fileutil.h>
-#include <vespa/vespalib/data/fileheader.h>
+#include <vespa/eval/tensor/dense/dense_tensor.h>
+#include <vespa/eval/tensor/tensor.h>
#include <vespa/fastos/file.h>
+#include <vespa/searchlib/attribute/attributeguard.h>
+#include <vespa/searchlib/tensor/default_nearest_neighbor_index_factory.h>
+#include <vespa/searchlib/tensor/dense_tensor_attribute.h>
+#include <vespa/searchlib/tensor/doc_vector_access.h>
+#include <vespa/searchlib/tensor/generic_tensor_attribute.h>
+#include <vespa/searchlib/tensor/hnsw_index.h>
+#include <vespa/searchlib/tensor/nearest_neighbor_index.h>
+#include <vespa/searchlib/tensor/nearest_neighbor_index_factory.h>
+#include <vespa/searchlib/tensor/tensor_attribute.h>
+#include <vespa/vespalib/data/fileheader.h>
+#include <vespa/vespalib/io/fileutil.h>
+#include <vespa/vespalib/test/insertion_operators.h>
+#include <vespa/vespalib/testkit/test_kit.h>
+
#include <vespa/log/log.h>
LOG_SETUP("tensorattribute_test");
using document::WrongTensorTypeException;
-using search::tensor::TensorAttribute;
-using search::tensor::DenseTensorAttribute;
-using search::tensor::GenericTensorAttribute;
using search::AttributeGuard;
using search::AttributeVector;
-using vespalib::eval::ValueType;
+using search::attribute::HnswIndexParams;
+using search::tensor::DefaultNearestNeighborIndexFactory;
+using search::tensor::DenseTensorAttribute;
+using search::tensor::DocVectorAccess;
+using search::tensor::GenericTensorAttribute;
+using search::tensor::HnswIndex;
+using search::tensor::NearestNeighborIndex;
+using search::tensor::NearestNeighborIndexFactory;
+using search::tensor::TensorAttribute;
using vespalib::eval::TensorSpec;
-using vespalib::tensor::Tensor;
-using vespalib::tensor::DenseTensor;
+using vespalib::eval::ValueType;
using vespalib::tensor::DefaultTensorEngine;
+using vespalib::tensor::DenseTensor;
+using vespalib::tensor::Tensor;
+
+using DoubleVector = std::vector<double>;
-namespace vespalib {
-namespace tensor {
+namespace vespalib::tensor {
static bool operator==(const Tensor &lhs, const Tensor &rhs)
{
@@ -36,10 +50,10 @@ static bool operator==(const Tensor &lhs, const Tensor &rhs)
}
}
-}
vespalib::string sparseSpec("tensor(x{},y{})");
vespalib::string denseSpec("tensor(x[2],y[3])");
+vespalib::string vec_2d_spec("tensor(x[2])");
Tensor::UP createTensor(const TensorSpec &spec) {
auto value = DefaultTensorEngine::ref().from_spec(spec);
@@ -52,6 +66,78 @@ Tensor::UP createTensor(const TensorSpec &spec) {
return Tensor::UP(tensor);
}
+TensorSpec
+vec_2d(double x0, double x1)
+{
+ return TensorSpec(vec_2d_spec).add({{"x", 0}}, x0).add({{"x", 1}}, x1);
+}
+
+class MockNearestNeighborIndex : public NearestNeighborIndex {
+private:
+ using Entry = std::pair<uint32_t, DoubleVector>;
+ using EntryVector = std::vector<Entry>;
+
+ const DocVectorAccess& _vectors;
+ EntryVector _adds;
+ EntryVector _removes;
+
+public:
+ MockNearestNeighborIndex(const DocVectorAccess& vectors)
+ : _vectors(vectors),
+ _adds(),
+ _removes()
+ {
+ }
+ void clear() {
+ _adds.clear();
+ _removes.clear();
+ }
+ void expect_empty_add() const {
+ EXPECT_TRUE(_adds.empty());
+ }
+ void expect_add(uint32_t exp_docid, const DoubleVector& exp_vector) const {
+ EXPECT_EQUAL(1u, _adds.size());
+ EXPECT_EQUAL(exp_docid, _adds.back().first);
+ EXPECT_EQUAL(exp_vector, _adds.back().second);
+ }
+ void expect_adds(const EntryVector &exp_adds) const {
+ EXPECT_EQUAL(exp_adds, _adds);
+ }
+ void expect_empty_remove() const {
+ EXPECT_TRUE(_removes.empty());
+ }
+ void expect_remove(uint32_t exp_docid, const DoubleVector& exp_vector) const {
+ EXPECT_EQUAL(1u, _removes.size());
+ EXPECT_EQUAL(exp_docid, _removes.back().first);
+ EXPECT_EQUAL(exp_vector, _removes.back().second);
+ }
+ void add_document(uint32_t docid) override {
+ auto vector = _vectors.get_vector(docid).typify<double>();
+ _adds.emplace_back(docid, DoubleVector(vector.begin(), vector.end()));
+ }
+ void remove_document(uint32_t docid) override {
+ auto vector = _vectors.get_vector(docid).typify<double>();
+ _removes.emplace_back(docid, DoubleVector(vector.begin(), vector.end()));
+ }
+ std::vector<Neighbor> find_top_k(uint32_t k, vespalib::tensor::TypedCells vector, uint32_t explore_k) const override {
+ (void) k;
+ (void) vector;
+ (void) explore_k;
+ return std::vector<Neighbor>();
+ }
+};
+
+class MockNearestNeighborIndexFactory : public NearestNeighborIndexFactory {
+
+ std::unique_ptr<NearestNeighborIndex> make(const DocVectorAccess& vectors,
+ ValueType::CellType cell_type,
+ const search::attribute::HnswIndexParams& params) const override {
+ (void) params;
+ assert(cell_type == ValueType::CellType::DOUBLE);
+ return std::make_unique<MockNearestNeighborIndex>(vectors);
+ }
+};
+
struct Fixture
{
using BasicType = search::attribute::BasicType;
@@ -61,16 +147,20 @@ struct Fixture
Config _cfg;
vespalib::string _name;
vespalib::string _typeSpec;
+ std::unique_ptr<NearestNeighborIndexFactory> _index_factory;
std::shared_ptr<TensorAttribute> _tensorAttr;
std::shared_ptr<AttributeVector> _attr;
bool _denseTensors;
bool _useDenseTensorAttribute;
Fixture(const vespalib::string &typeSpec,
- bool useDenseTensorAttribute = false)
+ bool useDenseTensorAttribute = false,
+ bool enable_hnsw_index = false,
+ bool use_mock_index = false)
: _cfg(BasicType::TENSOR, CollectionType::SINGLE),
_name("test"),
_typeSpec(typeSpec),
+ _index_factory(std::make_unique<DefaultNearestNeighborIndexFactory>()),
_tensorAttr(),
_attr(),
_denseTensors(false),
@@ -80,20 +170,40 @@ struct Fixture
if (_cfg.tensorType().is_dense()) {
_denseTensors = true;
}
+ if (enable_hnsw_index) {
+ _cfg.set_hnsw_index_params(HnswIndexParams(4, 20));
+ if (use_mock_index) {
+ _index_factory = std::make_unique<MockNearestNeighborIndexFactory>();
+ }
+ }
_tensorAttr = makeAttr();
_attr = _tensorAttr;
_attr->addReservedDoc();
}
+ ~Fixture() {}
std::shared_ptr<TensorAttribute> makeAttr() {
if (_useDenseTensorAttribute) {
assert(_denseTensors);
- return std::make_shared<DenseTensorAttribute>(_name, _cfg);
+ return std::make_shared<DenseTensorAttribute>(_name, _cfg, *_index_factory);
} else {
return std::make_shared<GenericTensorAttribute>(_name, _cfg);
}
}
+ const DenseTensorAttribute& as_dense_tensor() const {
+ auto result = dynamic_cast<const DenseTensorAttribute*>(_tensorAttr.get());
+ assert(result != nullptr);
+ return *result;
+ }
+
+ MockNearestNeighborIndex& mock_index() {
+ assert(as_dense_tensor().nearest_neighbor_index() != nullptr);
+ auto mock_index = dynamic_cast<const MockNearestNeighborIndex*>(as_dense_tensor().nearest_neighbor_index());
+ assert(mock_index != nullptr);
+ return *const_cast<MockNearestNeighborIndex*>(mock_index);
+ }
+
void ensureSpace(uint32_t docId) {
while (_attr->getNumDocs() <= docId) {
uint32_t newDocId = 0u;
@@ -108,7 +218,15 @@ struct Fixture
_attr->commit();
}
- void setTensor(uint32_t docId, const Tensor &tensor) {
+ void set_tensor(uint32_t docid, const TensorSpec &spec) {
+ set_tensor_internal(docid, *createTensor(spec));
+ }
+
+ void set_empty_tensor(uint32_t docid) {
+ set_tensor_internal(docid, *_tensorAttr->getEmptyTensor());
+ }
+
+ void set_tensor_internal(uint32_t docId, const Tensor &tensor) {
ensureSpace(docId);
_tensorAttr->setTensor(docId, tensor);
_attr->commit();
@@ -119,27 +237,18 @@ struct Fixture
return _attr->getStatus();
}
- void
- assertGetNoTensor(uint32_t docId) {
+ void assertGetNoTensor(uint32_t docId) {
AttributeGuard guard(_attr);
Tensor::UP actTensor = _tensorAttr->getTensor(docId);
EXPECT_FALSE(actTensor);
}
- void
- assertGetTensor(const Tensor &expTensor, uint32_t docId)
- {
+ void assertGetTensor(const TensorSpec &expSpec, uint32_t docId) {
+ Tensor::UP expTensor = createTensor(expSpec);
AttributeGuard guard(_attr);
Tensor::UP actTensor = _tensorAttr->getTensor(docId);
EXPECT_TRUE(static_cast<bool>(actTensor));
- EXPECT_EQUAL(expTensor, *actTensor);
- }
-
- void
- assertGetTensor(const TensorSpec &expSpec, uint32_t docId)
- {
- Tensor::UP expTensor = createTensor(expSpec);
- assertGetTensor(*expTensor, docId);
+ EXPECT_EQUAL(*expTensor, *actTensor);
}
void save() {
@@ -154,23 +263,20 @@ struct Fixture
EXPECT_TRUE(loadok);
}
- Tensor::UP expDenseTensor3() const
- {
- return createTensor(TensorSpec(denseSpec)
- .add({{"x", 0}, {"y", 1}}, 11)
- .add({{"x", 1}, {"y", 2}}, 0));
+ TensorSpec expDenseTensor3() const {
+ return TensorSpec(denseSpec)
+ .add({{"x", 0}, {"y", 1}}, 11)
+ .add({{"x", 1}, {"y", 2}}, 0);
}
- Tensor::UP expDenseFillTensor() const
- {
- return createTensor(TensorSpec(denseSpec)
- .add({{"x", 0}, {"y", 0}}, 5)
- .add({{"x", 1}, {"y", 2}}, 0));
+ TensorSpec expDenseFillTensor() const {
+ return TensorSpec(denseSpec)
+ .add({{"x", 0}, {"y", 0}}, 5)
+ .add({{"x", 1}, {"y", 2}}, 0);
}
- Tensor::UP expEmptyDenseTensor() const
- {
- return createTensor(TensorSpec(denseSpec));
+ TensorSpec expEmptyDenseTensor() const {
+ return TensorSpec(denseSpec);
}
vespalib::string expEmptyDenseTensorSpec() const {
@@ -200,21 +306,21 @@ Fixture::testSetTensorValue()
EXPECT_EQUAL(5u, _attr->getNumDocs());
EXPECT_EQUAL(5u, _attr->getCommittedDocIdLimit());
TEST_DO(assertGetNoTensor(4));
- EXPECT_EXCEPTION(setTensor(4, *createTensor(TensorSpec("double"))),
+ EXPECT_EXCEPTION(set_tensor(4, TensorSpec("double")),
WrongTensorTypeException,
"but other tensor type is 'double'");
TEST_DO(assertGetNoTensor(4));
- setTensor(4, *_tensorAttr->getEmptyTensor());
+ set_empty_tensor(4);
if (_denseTensors) {
- TEST_DO(assertGetTensor(*expEmptyDenseTensor(), 4));
- setTensor(3, *expDenseTensor3());
- TEST_DO(assertGetTensor(*expDenseTensor3(), 3));
+ TEST_DO(assertGetTensor(expEmptyDenseTensor(), 4));
+ set_tensor(3, expDenseTensor3());
+ TEST_DO(assertGetTensor(expDenseTensor3(), 3));
} else {
TEST_DO(assertGetTensor(TensorSpec(sparseSpec), 4));
- setTensor(3, *createTensor(TensorSpec(sparseSpec)
- .add({{"x", ""}, {"y", ""}}, 11)));
+ set_tensor(3, TensorSpec(sparseSpec)
+ .add({{"x", ""}, {"y", ""}}, 11));
TEST_DO(assertGetTensor(TensorSpec(sparseSpec)
- .add({{"x", ""}, {"y", ""}}, 11), 3));
+ .add({{"x", ""}, {"y", ""}}, 11), 3));
}
TEST_DO(assertGetNoTensor(2));
TEST_DO(clearTensor(3));
@@ -225,23 +331,23 @@ void
Fixture::testSaveLoad()
{
ensureSpace(4);
- setTensor(4, *_tensorAttr->getEmptyTensor());
+ set_empty_tensor(4);
if (_denseTensors) {
- setTensor(3, *expDenseTensor3());
+ set_tensor(3, expDenseTensor3());
} else {
- setTensor(3, *createTensor(TensorSpec(sparseSpec)
- .add({{"x", ""}, {"y", "1"}}, 11)));
+ set_tensor(3, TensorSpec(sparseSpec)
+ .add({{"x", ""}, {"y", "1"}}, 11));
}
TEST_DO(save());
TEST_DO(load());
EXPECT_EQUAL(5u, _attr->getNumDocs());
EXPECT_EQUAL(5u, _attr->getCommittedDocIdLimit());
if (_denseTensors) {
- TEST_DO(assertGetTensor(*expDenseTensor3(), 3));
- TEST_DO(assertGetTensor(*expEmptyDenseTensor(), 4));
+ TEST_DO(assertGetTensor(expDenseTensor3(), 3));
+ TEST_DO(assertGetTensor(expEmptyDenseTensor(), 4));
} else {
TEST_DO(assertGetTensor(TensorSpec(sparseSpec)
- .add({{"x", ""}, {"y", "1"}}, 11), 3));
+ .add({{"x", ""}, {"y", "1"}}, 11), 3));
TEST_DO(assertGetTensor(TensorSpec(sparseSpec), 4));
}
TEST_DO(assertGetNoTensor(2));
@@ -256,29 +362,28 @@ Fixture::testCompaction()
return;
}
ensureSpace(4);
- Tensor::UP emptytensor = _tensorAttr->getEmptyTensor();
- Tensor::UP emptyxytensor = createTensor(TensorSpec(sparseSpec));
- Tensor::UP simpletensor = createTensor(TensorSpec(sparseSpec)
- .add({{"x", ""}, {"y", "1"}}, 11));
- Tensor::UP filltensor = createTensor(TensorSpec(sparseSpec)
- .add({{"x", ""}, {"y", ""}}, 5));
+ TensorSpec empty_xy_tensor(sparseSpec);
+ TensorSpec simple_tensor = TensorSpec(sparseSpec)
+ .add({{"x", ""}, {"y", "1"}}, 11);
+ TensorSpec fill_tensor = TensorSpec(sparseSpec)
+ .add({{"x", ""}, {"y", ""}}, 5);
if (_denseTensors) {
- emptyxytensor = expEmptyDenseTensor();
- simpletensor = expDenseTensor3();
- filltensor = expDenseFillTensor();
+ empty_xy_tensor = expEmptyDenseTensor();
+ simple_tensor = expDenseTensor3();
+ fill_tensor = expDenseFillTensor();
}
- setTensor(4, *emptytensor);
- setTensor(3, *simpletensor);
- setTensor(2, *filltensor);
+ set_empty_tensor(4);
+ set_tensor(3, simple_tensor);
+ set_tensor(2, fill_tensor);
clearTensor(2);
- setTensor(2, *filltensor);
+ set_tensor(2, fill_tensor);
search::attribute::Status oldStatus = getStatus();
search::attribute::Status newStatus = oldStatus;
uint64_t iter = 0;
uint64_t iterLimit = 100000;
for (; iter < iterLimit; ++iter) {
clearTensor(2);
- setTensor(2, *filltensor);
+ set_tensor(2, fill_tensor);
newStatus = getStatus();
if (newStatus.getUsed() < oldStatus.getUsed()) {
break;
@@ -290,9 +395,9 @@ Fixture::testCompaction()
"iter = %" PRIu64 ", memory usage %" PRIu64 ", -> %" PRIu64,
iter, oldStatus.getUsed(), newStatus.getUsed());
TEST_DO(assertGetNoTensor(1));
- TEST_DO(assertGetTensor(*filltensor, 2));
- TEST_DO(assertGetTensor(*simpletensor, 3));
- TEST_DO(assertGetTensor(*emptyxytensor, 4));
+ TEST_DO(assertGetTensor(fill_tensor, 2));
+ TEST_DO(assertGetTensor(simple_tensor, 3));
+ TEST_DO(assertGetTensor(empty_xy_tensor, 4));
}
void
@@ -357,4 +462,73 @@ TEST("Test dense tensors with dense tensor attribute")
testAll([]() { return std::make_shared<Fixture>(denseSpec, true); });
}
+TEST_F("Hnsw index is NOT instantiated in dense tensor attribute by default",
+ Fixture(vec_2d_spec, true, false))
+{
+ const auto& tensor = f.as_dense_tensor();
+ EXPECT_TRUE(tensor.nearest_neighbor_index() == nullptr);
+}
+
+TEST_F("Hnsw index is instantiated in dense tensor attribute when specified in config",
+ Fixture(vec_2d_spec, true, true))
+{
+ const auto& tensor = f.as_dense_tensor();
+ ASSERT_TRUE(tensor.nearest_neighbor_index() != nullptr);
+ auto hnsw_index = dynamic_cast<const HnswIndex*>(tensor.nearest_neighbor_index());
+ ASSERT_TRUE(hnsw_index != nullptr);
+
+ const auto& cfg = hnsw_index->config();
+ EXPECT_EQUAL(8u, cfg.max_links_at_level_0());
+ EXPECT_EQUAL(4u, cfg.max_links_at_hierarchic_levels());
+ EXPECT_EQUAL(20u, cfg.neighbors_to_explore_at_construction());
+ EXPECT_TRUE(cfg.heuristic_select_neighbors());
+}
+
+class DenseTensorAttributeMockIndex : public Fixture {
+public:
+ DenseTensorAttributeMockIndex() : Fixture(vec_2d_spec, true, true, true) {}
+};
+
+TEST_F("setTensor() updates nearest neighbor index", DenseTensorAttributeMockIndex)
+{
+ auto& index = f.mock_index();
+
+ f.set_tensor(1, vec_2d(3, 5));
+ index.expect_add(1, {3, 5});
+ index.expect_empty_remove();
+ index.clear();
+
+ // Replaces previous value.
+ f.set_tensor(1, vec_2d(7, 9));
+ index.expect_remove(1, {3, 5});
+ index.expect_add(1, {7, 9});
+}
+
+TEST_F("clearDoc() updates nearest neighbor index", DenseTensorAttributeMockIndex)
+{
+ auto& index = f.mock_index();
+
+ // Nothing to clear.
+ f.clearTensor(1);
+ index.expect_empty_remove();
+ index.expect_empty_add();
+
+ // Clears previous value.
+ f.set_tensor(1, vec_2d(3, 5));
+ index.clear();
+ f.clearTensor(1);
+ index.expect_remove(1, {3, 5});
+ index.expect_empty_add();
+}
+
+TEST_F("onLoad() updates nearest neighbor index", DenseTensorAttributeMockIndex)
+{
+ f.set_tensor(1, vec_2d(3, 5));
+ f.set_tensor(2, vec_2d(7, 9));
+ f.save();
+ f.load();
+ auto& index = f.mock_index();
+ index.expect_adds({{1, {3, 5}}, {2, {7, 9}}});
+}
+
TEST_MAIN() { TEST_RUN_ALL(); vespalib::unlink("test.dat"); }
diff --git a/searchlib/src/tests/common/sequencedtaskexecutor/.gitignore b/searchlib/src/tests/common/sequencedtaskexecutor/.gitignore
index 35d038b0b7c..4bd94f124fb 100644
--- a/searchlib/src/tests/common/sequencedtaskexecutor/.gitignore
+++ b/searchlib/src/tests/common/sequencedtaskexecutor/.gitignore
@@ -1 +1,2 @@
searchlib_sequencedtaskexecutor_test_app
+searchlib_sequencedtaskexecutor_benchmark_app
diff --git a/searchlib/src/tests/common/sequencedtaskexecutor/CMakeLists.txt b/searchlib/src/tests/common/sequencedtaskexecutor/CMakeLists.txt
index 6ba30a1647f..6c593d20683 100644
--- a/searchlib/src/tests/common/sequencedtaskexecutor/CMakeLists.txt
+++ b/searchlib/src/tests/common/sequencedtaskexecutor/CMakeLists.txt
@@ -1,4 +1,11 @@
# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_executable(searchlib_sequencedtaskexecutor_benchmark_app TEST
+ SOURCES
+ sequencedtaskexecutor_benchmark.cpp
+ DEPENDS
+ searchlib
+)
+
vespa_add_executable(searchlib_sequencedtaskexecutor_test_app TEST
SOURCES
sequencedtaskexecutor_test.cpp
diff --git a/searchlib/src/tests/common/sequencedtaskexecutor/sequencedtaskexecutor_benchmark.cpp b/searchlib/src/tests/common/sequencedtaskexecutor/sequencedtaskexecutor_benchmark.cpp
new file mode 100644
index 00000000000..a51becfbf13
--- /dev/null
+++ b/searchlib/src/tests/common/sequencedtaskexecutor/sequencedtaskexecutor_benchmark.cpp
@@ -0,0 +1,24 @@
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/searchlib/common/sequencedtaskexecutor.h>
+#include <vespa/vespalib/util/lambdatask.h>
+#include <atomic>
+
+using search::SequencedTaskExecutor;
+using ExecutorId = search::ISequencedTaskExecutor::ExecutorId;
+
+int main(int argc, char *argv[]) {
+ unsigned long numTasks = 1000000;
+ unsigned numThreads = 4;
+ std::atomic<long> counter(0);
+ if (argc > 1)
+ numTasks = atol(argv[1]);
+ if (argc > 2)
+ numThreads = atoi(argv[2]);
+
+ SequencedTaskExecutor executor(numThreads);
+ for (unsigned long tid(0); tid < numTasks; tid++) {
+ executor.executeTask(ExecutorId(tid%numThreads), vespalib::makeLambdaTask([&counter] { counter++; }));
+ }
+ return 0;
+}
diff --git a/searchlib/src/tests/features/prod_features.cpp b/searchlib/src/tests/features/prod_features.cpp
index 56aaf2dcbc9..de436dffff1 100644
--- a/searchlib/src/tests/features/prod_features.cpp
+++ b/searchlib/src/tests/features/prod_features.cpp
@@ -9,16 +9,15 @@
#include <vespa/searchlib/attribute/floatbase.h>
#include <vespa/searchlib/attribute/integerbase.h>
#include <vespa/searchlib/attribute/stringbase.h>
+#include <vespa/searchlib/attribute/singleboolattribute.h>
#include <vespa/searchlib/features/agefeature.h>
#include <vespa/searchlib/features/array_parser.hpp>
#include <vespa/searchlib/features/attributefeature.h>
-#include <vespa/searchlib/features/attributematchfeature.h>
#include <vespa/searchlib/features/closenessfeature.h>
#include <vespa/searchlib/features/distancefeature.h>
#include <vespa/searchlib/features/dotproductfeature.h>
#include <vespa/searchlib/features/fieldlengthfeature.h>
#include <vespa/searchlib/features/fieldmatchfeature.h>
-#include <vespa/searchlib/features/fieldtermmatchfeature.h>
#include <vespa/searchlib/features/firstphasefeature.h>
#include <vespa/searchlib/features/foreachfeature.h>
#include <vespa/searchlib/features/freshnessfeature.h>
@@ -35,7 +34,6 @@
#include <vespa/searchlib/features/setup.h>
#include <vespa/searchlib/features/termfeature.h>
#include <vespa/searchlib/features/utils.h>
-#include <vespa/searchlib/features/valuefeature.h>
#include <vespa/searchlib/features/weighted_set_parser.hpp>
#include <vespa/searchlib/fef/featurenamebuilder.h>
#include <vespa/searchlib/fef/indexproperties.h>
@@ -60,6 +58,7 @@ using search::AttributeFactory;
using search::IntegerAttribute;
using search::FloatingPointAttribute;
using search::StringAttribute;
+using search::SingleBoolAttribute;
using search::WeightedSetStringExtAttribute;
using search::attribute::WeightedEnumContent;
@@ -212,7 +211,7 @@ Test::setupForAgeTest(FtFeatureTest & ft, uint64_t docTime)
doctime->addReservedDoc();
doctime->addDocs(1);
ft.getIndexEnv().getAttributeMap().add(doctime);
- (static_cast<IntegerAttribute *>(doctime.get()))->update(1, docTime);
+ (dynamic_cast<IntegerAttribute *>(doctime.get()))->update(1, docTime);
doctime->commit();
}
@@ -240,7 +239,12 @@ Test::testAttribute()
RankResult exp;
exp.addScore("attribute(sint)", 10).
addScore("attribute(sint,0)", 10).
+ addScore("attribute(slong)", 20).
+ addScore("attribute(sbyte)", 37).
+ addScore("attribute(sbool)", 1).
+ addScore("attribute(sebool)", 0).
addScore("attribute(sfloat)", 60.5f).
+ addScore("attribute(sdouble)", 67.5f).
addScore("attribute(sstr)", (feature_t)vespalib::hash_code("foo")).
addScore("attribute(sint).count", 1).
addScore("attribute(sfloat).count", 1).
@@ -250,12 +254,18 @@ Test::testAttribute()
addScore("attribute(udefstr)", (feature_t)vespalib::hash_code(""));
FtFeatureTest ft(_factory, exp.getKeys());
- ft.getIndexEnv().getBuilder().addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "sint").
- addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "sfloat").
- addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "sstr").
- addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "udefint").
- addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "udeffloat").
- addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "udefstr");
+ ft.getIndexEnv().getBuilder()
+ .addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "sint")
+ .addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "slong")
+ .addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "sbyte")
+ .addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "sbool")
+ .addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "sebool")
+ .addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "sfloat")
+ .addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "sdouble")
+ .addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "sstr")
+ .addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "udefint")
+ .addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "udeffloat")
+ .addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "udefstr");
setupForAttributeTest(ft);
ASSERT_TRUE(ft.setup());
ASSERT_TRUE(ft.execute(exp));
@@ -370,6 +380,11 @@ Test::setupForAttributeTest(FtFeatureTest &ft, bool setup_env)
avs.push_back(AttributeFactory::createAttribute("udefint", AVC(AVBT::INT32, AVCT::SINGLE))); // 9
avs.push_back(AttributeFactory::createAttribute("udeffloat", AVC(AVBT::FLOAT, AVCT::SINGLE))); // 10
avs.push_back(AttributeFactory::createAttribute("udefstr", AVC(AVBT::STRING, AVCT::SINGLE))); // 11
+ avs.push_back(AttributeFactory::createAttribute("sbyte", AVC(AVBT::INT64, AVCT::SINGLE))); // 12
+ avs.push_back(AttributeFactory::createAttribute("slong", AVC(AVBT::INT64, AVCT::SINGLE))); // 13
+ avs.push_back(AttributeFactory::createAttribute("sbool", AVC(AVBT::BOOL, AVCT::SINGLE))); // 14
+ avs.push_back(AttributeFactory::createAttribute("sebool", AVC(AVBT::BOOL, AVCT::SINGLE))); // 15
+ avs.push_back(AttributeFactory::createAttribute("sdouble", AVC(AVBT::DOUBLE, AVCT::SINGLE))); // 16
// simulate a unique only attribute as specified in sd
AVC cfg(AVBT::INT32, AVCT::SINGLE);
@@ -391,36 +406,46 @@ Test::setupForAttributeTest(FtFeatureTest &ft, bool setup_env)
.addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "udefint")
.addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "udeffloat")
.addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "udefstr")
- .addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "unique");
+ .addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "unique")
+ .addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "slong")
+ .addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "sdouble")
+ .addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "sbyte")
+ .addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "sbool")
+ .addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "sebool");
}
- for (uint32_t i = 0; i < avs.size(); ++i) {
- avs[i]->addReservedDoc();
- avs[i]->addDocs(1);
- ft.getIndexEnv().getAttributeMap().add(avs[i]);
+ for (const auto & attr : avs) {
+ attr->addReservedDoc();
+ attr->addDocs(1);
+ ft.getIndexEnv().getAttributeMap().add(attr);
}
// integer attributes
- (static_cast<IntegerAttribute *>(avs[0].get()))->update(1, 10);
- (static_cast<IntegerAttribute *>(avs[1].get()))->append(1, 20, 0);
- (static_cast<IntegerAttribute *>(avs[1].get()))->append(1, 30, 0);
- (static_cast<IntegerAttribute *>(avs[2].get()))->append(1, 40, 10);
- (static_cast<IntegerAttribute *>(avs[2].get()))->append(1, 50, 20);
- (static_cast<IntegerAttribute *>(avs[9].get()))->update(1, search::attribute::getUndefined<int32_t>());
+ (dynamic_cast<IntegerAttribute *>(avs[0].get()))->update(1, 10);
+ (dynamic_cast<IntegerAttribute *>(avs[12].get()))->update(1, 37);
+ (dynamic_cast<IntegerAttribute *>(avs[13].get()))->update(1, 20);
+ (dynamic_cast<SingleBoolAttribute *>(avs[14].get()))->update(1, 1);
+ (dynamic_cast<SingleBoolAttribute *>(avs[15].get()))->update(1, 0);
+ (dynamic_cast<IntegerAttribute *>(avs[1].get()))->append(1, 20, 0);
+ (dynamic_cast<IntegerAttribute *>(avs[1].get()))->append(1, 30, 0);
+ (dynamic_cast<IntegerAttribute *>(avs[2].get()))->append(1, 40, 10);
+ (dynamic_cast<IntegerAttribute *>(avs[2].get()))->append(1, 50, 20);
+ (dynamic_cast<IntegerAttribute *>(avs[9].get()))->update(1, search::attribute::getUndefined<int32_t>());
// feature_t attributes
- (static_cast<FloatingPointAttribute *>(avs[3].get()))->update(1, 60.5f);
- (static_cast<FloatingPointAttribute *>(avs[4].get()))->append(1, 70.5f, 0);
- (static_cast<FloatingPointAttribute *>(avs[4].get()))->append(1, 80.5f, 0);
- (static_cast<FloatingPointAttribute *>(avs[5].get()))->append(1, 90.5f, -30);
- (static_cast<FloatingPointAttribute *>(avs[5].get()))->append(1, 100.5f, -40);
- (static_cast<FloatingPointAttribute *>(avs[10].get()))->update(1, search::attribute::getUndefined<float>());
+ (dynamic_cast<FloatingPointAttribute *>(avs[3].get()))->update(1, 60.5f);
+ (dynamic_cast<FloatingPointAttribute *>(avs[4].get()))->append(1, 70.5f, 0);
+ (dynamic_cast<FloatingPointAttribute *>(avs[4].get()))->append(1, 80.5f, 0);
+ (dynamic_cast<FloatingPointAttribute *>(avs[5].get()))->append(1, 90.5f, -30);
+ (dynamic_cast<FloatingPointAttribute *>(avs[5].get()))->append(1, 100.5f, -40);
+ (dynamic_cast<FloatingPointAttribute *>(avs[10].get()))->update(1, search::attribute::getUndefined<float>());
+ (dynamic_cast<FloatingPointAttribute *>(avs[16].get()))->update(1, 67.5);
// string attributes
- (static_cast<StringAttribute *>(avs[6].get()))->update(1, "foo");
- (static_cast<StringAttribute *>(avs[7].get()))->append(1, "bar", 0);
- (static_cast<StringAttribute *>(avs[7].get()))->append(1, "baz", 0);
- (static_cast<StringAttribute *>(avs[8].get()))->append(1, "qux", 11);
- (static_cast<StringAttribute *>(avs[8].get()))->append(1, "quux", 12);
- (static_cast<StringAttribute *>(avs[11].get()))->update(1, "");
+ (dynamic_cast<StringAttribute *>(avs[6].get()))->update(1, "foo");
+ (dynamic_cast<StringAttribute *>(avs[7].get()))->append(1, "bar", 0);
+ (dynamic_cast<StringAttribute *>(avs[7].get()))->append(1, "baz", 0);
+ (dynamic_cast<StringAttribute *>(avs[8].get()))->append(1, "qux", 11);
+ (dynamic_cast<StringAttribute *>(avs[8].get()))->append(1, "quux", 12);
+ (dynamic_cast<StringAttribute *>(avs[11].get()))->update(1, "");
for (uint32_t i = 0; i < avs.size() - 1; ++i) { // do not commit the noupdate attribute
avs[i]->commit();
@@ -475,7 +500,7 @@ Test::assertCloseness(feature_t exp, const vespalib::string & attr, double dista
FtFeatureTest ft(_factory, feature);
std::vector<std::pair<int32_t, int32_t> > positions;
int32_t x = 0;
- positions.push_back(std::make_pair(x, x));
+ positions.emplace_back(x, x);
setupForDistanceTest(ft, "pos", positions, false);
ft.getQueryEnv().getLocation().setXPosition((int)distance);
ft.getQueryEnv().getLocation().setValid(true);
@@ -572,7 +597,7 @@ Test::assertFieldMatch(const vespalib::string & spec,
const vespalib::string & field,
uint32_t totalTermWeight)
{
- assertFieldMatch(spec, query, field, NULL, totalTermWeight);
+ assertFieldMatch(spec, query, field, nullptr, totalTermWeight);
}
void
@@ -581,7 +606,7 @@ Test::assertFieldMatchTS(const vespalib::string & spec,
const vespalib::string & field,
feature_t totalSignificance)
{
- assertFieldMatch(spec, query, field, NULL, 0, totalSignificance);
+ assertFieldMatch(spec, query, field, nullptr, 0, totalSignificance);
}
@@ -871,12 +896,12 @@ Test::setupForDistanceTest(FtFeatureTest &ft, const vespalib::string & attrName,
pos->addDocs(1);
ft.getIndexEnv().getAttributeMap().add(pos);
- IntegerAttribute * ia = static_cast<IntegerAttribute *>(pos.get());
- for (uint32_t i = 0; i < positions.size(); ++i) {
+ auto ia = dynamic_cast<IntegerAttribute *>(pos.get());
+ for (const auto & p : positions) {
if (zcurve) {
- ia->append(1, vespalib::geo::ZCurve::encode(positions[i].first, positions[i].second), 0);
+ ia->append(1, vespalib::geo::ZCurve::encode(p.first, p.second), 0);
} else {
- ia->append(1, positions[i].first, 0);
+ ia->append(1, p.first, 0);
}
}
@@ -891,11 +916,11 @@ Test::assert2DZDistance(feature_t exp, const vespalib::string & positions,
FtFeatureTest ft(_factory, "distance(pos)");
std::vector<vespalib::string> ta = FtUtil::tokenize(positions, ",");
std::vector<std::pair<int32_t, int32_t> > pos;
- for (uint32_t i = 0; i < ta.size(); ++i) {
- std::vector<vespalib::string> tb = FtUtil::tokenize(ta[i], ":");
- int32_t x = util::strToNum<int32_t>(tb[0]);
- int32_t y = util::strToNum<int32_t>(tb[1]);
- pos.push_back(std::make_pair(x, y));
+ for (const auto & s : ta) {
+ std::vector<vespalib::string> tb = FtUtil::tokenize(s, ":");
+ auto x = util::strToNum<int32_t>(tb[0]);
+ auto y = util::strToNum<int32_t>(tb[1]);
+ pos.emplace_back(x, y);
}
setupForDistanceTest(ft, "pos", pos, true);
ft.getQueryEnv().getLocation().setXPosition(xquery);
@@ -927,7 +952,7 @@ Test::testDistanceToPath()
{
// Test executor.
std::vector<std::pair<int32_t, int32_t> > pos;
- pos.push_back(std::make_pair(0, 0));
+ pos.emplace_back(0, 0);
// invalid path
assertDistanceToPath(pos, "a");
@@ -965,7 +990,7 @@ Test::testDistanceToPath()
assertDistanceToPath(pos, "(-3,2,2,2,2,-1,0,-1)", 1, 1, 2);
// multiple document locations
- pos.push_back(std::make_pair(0, 1));
+ pos.emplace_back(0, 1);
assertDistanceToPath(pos, "(-1,1,1,1)", 0, 0.5);
assertDistanceToPath(pos, "(-2,-1,-1,1)", 1, 1, 2);
assertDistanceToPath(pos, "(-1,0.25,1,0.25)", 0.25, 0.5, 0.5);
@@ -1017,7 +1042,7 @@ Test::testDistanceToPath()
}
void
-Test::assertDistanceToPath(const std::vector<std::pair<int32_t, int32_t> > pos,
+Test::assertDistanceToPath(const std::vector<std::pair<int32_t, int32_t> > & pos,
const vespalib::string &path, feature_t distance, feature_t traveled, feature_t product)
{
LOG(info, "Testing distance to path '%s' with %zd document locations.", path.c_str(), pos.size());
@@ -1033,20 +1058,6 @@ Test::assertDistanceToPath(const std::vector<std::pair<int32_t, int32_t> > pos,
.addScore("distanceToPath(pos).product", product)));
}
-void
-Test::setupForDocumentTest(FtFeatureTest &ft, const vespalib::string & attrName, const vespalib::string & docType)
-{
- AttributePtr type = AttributeFactory::createAttribute(attrName, AVC(AVBT::STRING, AVCT::SINGLE));
-
- type->addReservedDoc();
- type->addDocs(1);
- ft.getIndexEnv().getAttributeMap().add(type);
-
- (static_cast<StringAttribute *>(type.get()))->update(1, docType);
- type->commit();
-}
-
-
namespace {
void
@@ -1264,6 +1275,8 @@ void
Test::setupForDotProductTest(FtFeatureTest & ft)
{
struct Config {
+ Config() : name(nullptr), dataType(AVBT::BOOL), collectionType(AVCT::SINGLE), fastSearch(false) {}
+ Config(const char *n, AVBT dt, AVCT ct, bool fs) : name(n), dataType(dt), collectionType(ct), fastSearch(fs) {}
const char * name;
AVBT dataType;
AVCT collectionType;
@@ -1296,11 +1309,11 @@ Test::setupForDotProductTest(FtFeatureTest & ft)
baf->addDocs(2);
ft.getIndexEnv().getAttributeMap().add(baf);
for (size_t i(1); i < 6; i++) {
- IntegerAttribute * ia = dynamic_cast<IntegerAttribute *>(baf.get());
+ auto ia = dynamic_cast<IntegerAttribute *>(baf.get());
if (ia) {
ia->append(1, i, i);
} else {
- FloatingPointAttribute * fa = dynamic_cast<FloatingPointAttribute *>(baf.get());
+ auto fa = dynamic_cast<FloatingPointAttribute *>(baf.get());
fa->append(1, i, i);
}
}
@@ -1315,14 +1328,14 @@ Test::setupForDotProductTest(FtFeatureTest & ft)
ft.getIndexEnv().getAttributeMap().add(c);
ft.getIndexEnv().getAttributeMap().add(d);
- StringAttribute * sa = static_cast<StringAttribute *>(a.get());
+ auto sa = dynamic_cast<StringAttribute *>(a.get());
sa->append(1, "a", 1);
sa->append(1, "b", 2);
sa->append(1, "c", 3);
sa->append(1, "d", 4);
sa->append(1, "e", 5);
- WeightedSetStringExtAttribute * ea = static_cast<WeightedSetStringExtAttribute *>(d.get());
+ auto ea = dynamic_cast<WeightedSetStringExtAttribute *>(d.get());
EXPECT_TRUE(!ea->hasEnum());
uint32_t docId;
ea->addDoc(docId); // reserved doc
@@ -1517,9 +1530,9 @@ Test::testMatchCount()
ft.getIndexEnv().getBuilder().addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "foo");
ft.getIndexEnv().getBuilder().addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "bar");
ft.getIndexEnv().getBuilder().addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "baz");
- ASSERT_TRUE(ft.getQueryEnv().getBuilder().addAttributeNode("foo") != NULL); // query term 0, hit in foo
- ASSERT_TRUE(ft.getQueryEnv().getBuilder().addAttributeNode("bar") != NULL); // query term 1, hit in bar
- ASSERT_TRUE(ft.getQueryEnv().getBuilder().addAttributeNode("foo") != NULL); // query term 2, hit in foo
+ ASSERT_TRUE(ft.getQueryEnv().getBuilder().addAttributeNode("foo") != nullptr); // query term 0, hit in foo
+ ASSERT_TRUE(ft.getQueryEnv().getBuilder().addAttributeNode("bar") != nullptr); // query term 1, hit in bar
+ ASSERT_TRUE(ft.getQueryEnv().getBuilder().addAttributeNode("foo") != nullptr); // query term 2, hit in foo
ASSERT_TRUE(ft.setup());
MatchDataBuilder::UP mdb = ft.createMatchDataBuilder();
@@ -1577,9 +1590,9 @@ Test::testMatches()
ft.getIndexEnv().getBuilder().addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "foo");
ft.getIndexEnv().getBuilder().addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "bar");
ft.getIndexEnv().getBuilder().addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "baz");
- ASSERT_TRUE(ft.getQueryEnv().getBuilder().addAttributeNode("foo") != NULL); // query term 0, hit in foo
- ASSERT_TRUE(ft.getQueryEnv().getBuilder().addAttributeNode("bar") != NULL); // query term 1, hit in bar
- ASSERT_TRUE(ft.getQueryEnv().getBuilder().addAttributeNode("foo") != NULL); // query term 2, hit in foo
+ ASSERT_TRUE(ft.getQueryEnv().getBuilder().addAttributeNode("foo") != nullptr); // query term 0, hit in foo
+ ASSERT_TRUE(ft.getQueryEnv().getBuilder().addAttributeNode("bar") != nullptr); // query term 1, hit in bar
+ ASSERT_TRUE(ft.getQueryEnv().getBuilder().addAttributeNode("foo") != nullptr); // query term 2, hit in foo
ASSERT_TRUE(ft.setup());
MatchDataBuilder::UP mdb = ft.createMatchDataBuilder();
@@ -1613,11 +1626,9 @@ Test::assertMatches(uint32_t output,
ASSERT_TRUE(ft.execute(output, EPS, docId));
// Execute and compare results.
- if (!EXPECT_TRUE(ft.execute(output, EPS, docId))) return false;
- return true;
+ return EXPECT_TRUE(ft.execute(output, EPS, docId));
}
-
void
Test::testQuery()
{
@@ -1721,7 +1732,7 @@ Test::testRandom()
search::Rand48 rnd;
rnd.srand48(100);
for (uint32_t i = 0; i < 5; ++i) {
- feature_t exp = rnd.lrand48() / (feature_t)0x80000000u;
+ feature_t exp = static_cast<feature_t>(rnd.lrand48()) / static_cast<feature_t>(0x80000000u);
ASSERT_TRUE(ft.execute(exp, EPS, i + 1));
}
}
@@ -1744,7 +1755,7 @@ Test::testRandom()
search::Rand48 rnd;
for (uint32_t i = 1; i <= 5; ++i) {
rnd.srand48(100 + i); // seed + lid
- feature_t exp = rnd.lrand48() / (feature_t)0x80000000u;
+ feature_t exp = static_cast<feature_t>(rnd.lrand48()) / static_cast<feature_t>(0x80000000u);
ASSERT_TRUE(ft.execute(exp, EPS, i));
}
}
@@ -2067,10 +2078,7 @@ Test::assertTermDistance(const TermDistanceCalculator::Result & exp,
rr.addScore(feature + ".forwardTermPosition", exp.forwardTermPos);
rr.addScore(feature + ".reverse", exp.reverseDist);
rr.addScore(feature + ".reverseTermPosition", exp.reverseTermPos);
- if (!EXPECT_TRUE(ft.execute(rr, docId))) {
- return false;
- }
- return true;
+ return EXPECT_TRUE(ft.execute(rr, docId));
}
void
diff --git a/searchlib/src/tests/features/prod_features.h b/searchlib/src/tests/features/prod_features.h
index 8e6578f34bd..6d30dd3fcd8 100644
--- a/searchlib/src/tests/features/prod_features.h
+++ b/searchlib/src/tests/features/prod_features.h
@@ -10,10 +10,10 @@ class Test : public FtTestApp
{
public:
Test();
- ~Test();
+ ~Test() override;
int Main() override;
void testFramework();
- void testFtLib();
+ static void testFtLib();
void testAge();
void testAttribute();
void testAttributeMatch();
@@ -39,10 +39,9 @@ public:
void testRankingExpression();
void testTerm();
void testTermDistance();
- void testUtils();
+ static void testUtils();
static void setupForDotProductTest(FtFeatureTest & ft);
- static void setupForDocumentTest(FtFeatureTest &ft, const vespalib::string & attrName, const vespalib::string & docType);
private:
void testFieldMatchBluePrint();
@@ -81,21 +80,21 @@ private:
void testFieldMatchExecutorRemaining();
void assertAge(feature_t expAge, const vespalib::string & attr, uint64_t now, uint64_t docTime);
- void setupForAgeTest(FtFeatureTest & ft, uint64_t docTime);
- void setupForAttributeTest(FtFeatureTest &ft, bool setup_env = true);
+ static void setupForAgeTest(FtFeatureTest & ft, uint64_t docTime);
+ static void setupForAttributeTest(FtFeatureTest &ft, bool setup_env = true);
void assertCloseness(feature_t exp, const vespalib::string & attr, double distance, double maxDistance = 0, double halfResponse = 0);
- void setupForDistanceTest(FtFeatureTest & ft, const vespalib::string & attrName,
- const std::vector<std::pair<int32_t, int32_t> > & positions, bool zcurve);
+ static void setupForDistanceTest(FtFeatureTest & ft, const vespalib::string & attrName,
+ const std::vector<std::pair<int32_t, int32_t> > & positions, bool zcurve);
void assert2DZDistance(feature_t exp, const vespalib::string & positions,
int32_t xquery, int32_t yquery, uint32_t xAspect = 0);
- void assertDistanceToPath(const std::vector<std::pair<int32_t, int32_t> > pos, const vespalib::string &path,
+ void assertDistanceToPath(const std::vector<std::pair<int32_t, int32_t> > & pos, const vespalib::string &path,
feature_t distance = search::features::DistanceToPathExecutor::DEFAULT_DISTANCE,
feature_t traveled = 1, feature_t product = 0);
void assertDotProduct(feature_t exp, const vespalib::string & vector, uint32_t docId = 1,
const vespalib::string & attribute = "wsstr", const vespalib::string & attributeOverride="");
void assertFieldMatch(const vespalib::string & spec, const vespalib::string & query, const vespalib::string & field,
- const search::features::fieldmatch::Params * params = NULL, uint32_t totalTermWeight = 0, feature_t totalSignificance = 0.0f);
+ const search::features::fieldmatch::Params * params = nullptr, uint32_t totalTermWeight = 0, feature_t totalSignificance = 0.0f);
void assertFieldMatch(const vespalib::string & spec, const vespalib::string & query, const vespalib::string & field,
uint32_t totalTermWeight);
void assertFieldMatchTS(const vespalib::string & spec, const vespalib::string & query, const vespalib::string & field,
diff --git a/searchlib/src/tests/fef/resolver/resolver_test.cpp b/searchlib/src/tests/fef/resolver/resolver_test.cpp
index 955c072810f..7591d028620 100644
--- a/searchlib/src/tests/fef/resolver/resolver_test.cpp
+++ b/searchlib/src/tests/fef/resolver/resolver_test.cpp
@@ -4,10 +4,13 @@
#include <vespa/searchlib/fef/fef.h>
#include <vespa/searchlib/fef/test/indexenvironment.h>
#include <vespa/searchlib/features/valuefeature.h>
+#include <vespa/searchlib/features/rankingexpressionfeature.h>
#include <vespa/log/log.h>
LOG_SETUP("resolver_test");
+using search::features::RankingExpressionBlueprint;
+
namespace search {
namespace fef {
@@ -58,6 +61,7 @@ class Test : public vespalib::TestApp {
private:
BlueprintFactory _factory;
void requireThatWeGetUniqueBlueprints();
+ void require_that_bad_input_is_handled();
public:
Test();
~Test();
@@ -69,6 +73,7 @@ Test::Test() :
{
_factory.addPrototype(Blueprint::SP(new BaseBlueprint()));
_factory.addPrototype(Blueprint::SP(new CombineBlueprint()));
+ _factory.addPrototype(std::make_shared<RankingExpressionBlueprint>());
}
Test::~Test() {}
@@ -85,12 +90,28 @@ Test::requireThatWeGetUniqueBlueprints()
EXPECT_TRUE(dynamic_cast<CombineBlueprint *>(spec[1].blueprint.get()) != NULL);
}
+void
+Test::require_that_bad_input_is_handled()
+{
+ test::IndexEnvironment ienv;
+ ienv.getProperties().add(indexproperties::eval::LazyExpressions::NAME, "false");
+ ienv.getProperties().add("rankingExpression(badinput).rankingScript", "base.foobad + base.bar");
+ BlueprintResolver::SP res(new BlueprintResolver(_factory, ienv));
+ res->addSeed("rankingExpression(badinput)");
+ EXPECT_FALSE(res->compile());
+ const BlueprintResolver::ExecutorSpecList & spec = res->getExecutorSpecs();
+ EXPECT_EQUAL(2u, spec.size());
+ EXPECT_TRUE(dynamic_cast<BaseBlueprint *>(spec[0].blueprint.get()) != nullptr);
+ EXPECT_TRUE(dynamic_cast<RankingExpressionBlueprint *>(spec[1].blueprint.get()) != nullptr);
+}
+
int
Test::Main()
{
TEST_INIT("resolver_test");
requireThatWeGetUniqueBlueprints();
+ require_that_bad_input_is_handled();
TEST_DONE();
}
diff --git a/searchlib/src/tests/queryeval/nearest_neighbor/nearest_neighbor_test.cpp b/searchlib/src/tests/queryeval/nearest_neighbor/nearest_neighbor_test.cpp
index 7bc582ab442..ee8d4b787bd 100644
--- a/searchlib/src/tests/queryeval/nearest_neighbor/nearest_neighbor_test.cpp
+++ b/searchlib/src/tests/queryeval/nearest_neighbor/nearest_neighbor_test.cpp
@@ -12,6 +12,7 @@
#include <vespa/searchlib/queryeval/simpleresult.h>
#include <vespa/searchlib/tensor/dense_tensor_attribute.h>
#include <vespa/vespalib/test/insertion_operators.h>
+#include <vespa/searchlib/queryeval/nns_index_iterator.h>
#include <vespa/log/log.h>
LOG_SETUP("nearest_neighbor_test");
@@ -190,4 +191,70 @@ TEST("require that NearestNeighborIterator sets expected rawscore") {
TEST_DO(verify_iterator_sets_expected_rawscore(denseSpecFloat, denseSpecDouble));
}
+TEST("require that NnsIndexIterator works as expected") {
+ std::vector<NnsIndexIterator::Hit> hits{{2,4.0}, {3,9.0}, {5,1.0}, {8,16.0}, {9,36.0}};
+ auto md = MatchData::makeTestInstance(2, 2);
+ auto &tfmd = *(md->resolveTermField(0));
+ auto search = NnsIndexIterator::create(tfmd, hits);
+ uint32_t docid = 1;
+ search->initFullRange();
+ bool match = search->seek(docid);
+ EXPECT_FALSE(match);
+ EXPECT_FALSE(search->isAtEnd());
+ EXPECT_EQUAL(2u, search->getDocId());
+ docid = 2;
+ match = search->seek(docid);
+ EXPECT_TRUE(match);
+ EXPECT_FALSE(search->isAtEnd());
+ EXPECT_EQUAL(docid, search->getDocId());
+ search->unpack(docid);
+ EXPECT_EQUAL(2.0, tfmd.getRawScore());
+
+ docid = 3;
+ match = search->seek(docid);
+ EXPECT_TRUE(match);
+ EXPECT_FALSE(search->isAtEnd());
+ EXPECT_EQUAL(docid, search->getDocId());
+ search->unpack(docid);
+ EXPECT_EQUAL(3.0, tfmd.getRawScore());
+
+ docid = 4;
+ match = search->seek(docid);
+ EXPECT_FALSE(match);
+ EXPECT_FALSE(search->isAtEnd());
+ EXPECT_EQUAL(5u, search->getDocId());
+
+ docid = 6;
+ match = search->seek(docid);
+ EXPECT_FALSE(match);
+ EXPECT_FALSE(search->isAtEnd());
+ EXPECT_EQUAL(8u, search->getDocId());
+ docid = 8;
+ search->unpack(docid);
+ EXPECT_EQUAL(4.0, tfmd.getRawScore());
+ docid = 9;
+ match = search->seek(docid);
+ EXPECT_TRUE(match);
+ EXPECT_FALSE(search->isAtEnd());
+ docid = 10;
+ match = search->seek(docid);
+ EXPECT_FALSE(match);
+ EXPECT_TRUE(search->isAtEnd());
+
+ docid = 4;
+ search->initRange(docid, 7);
+ match = search->seek(docid);
+ EXPECT_FALSE(match);
+ EXPECT_FALSE(search->isAtEnd());
+ EXPECT_EQUAL(5u, search->getDocId());
+ docid = 5;
+ search->unpack(docid);
+ EXPECT_EQUAL(1.0, tfmd.getRawScore());
+ EXPECT_FALSE(search->isAtEnd());
+ docid = 6;
+ match = search->seek(docid);
+ EXPECT_FALSE(match);
+ EXPECT_TRUE(search->isAtEnd());
+}
+
TEST_MAIN() { TEST_RUN_ALL(); }
diff --git a/searchlib/src/tests/rankingexpression/intrinsic_blueprint_adapter/intrinsic_blueprint_adapter_test.cpp b/searchlib/src/tests/rankingexpression/intrinsic_blueprint_adapter/intrinsic_blueprint_adapter_test.cpp
index b10da86dd8c..861af3527ca 100644
--- a/searchlib/src/tests/rankingexpression/intrinsic_blueprint_adapter/intrinsic_blueprint_adapter_test.cpp
+++ b/searchlib/src/tests/rankingexpression/intrinsic_blueprint_adapter/intrinsic_blueprint_adapter_test.cpp
@@ -5,6 +5,7 @@
#include <vespa/searchlib/features/rankingexpression/intrinsic_blueprint_adapter.h>
#include <vespa/searchlib/fef/test/indexenvironment.h>
#include <vespa/searchlib/fef/test/queryenvironment.h>
+#include <vespa/vespalib/util/stash.h>
#include <set>
using namespace search::features::rankingexpression;
diff --git a/searchlib/src/tests/tensor/hnsw_index/hnsw_index_test.cpp b/searchlib/src/tests/tensor/hnsw_index/hnsw_index_test.cpp
index c6246bb8434..52f45860c1e 100644
--- a/searchlib/src/tests/tensor/hnsw_index/hnsw_index_test.cpp
+++ b/searchlib/src/tests/tensor/hnsw_index/hnsw_index_test.cpp
@@ -48,8 +48,7 @@ using HnswIndexUP = std::unique_ptr<HnswIndex>;
class HnswIndexTest : public ::testing::Test {
public:
FloatVectors vectors;
- FloatSqEuclideanDistance distance_func;
- LevelGenerator level_generator;
+ LevelGenerator* level_generator;
HnswIndexUP index;
HnswIndexTest()
@@ -62,11 +61,14 @@ public:
.set(7, {3, 5}).set(8, {0, 3}).set(9, {4, 5});
}
void init(bool heuristic_select_neighbors) {
- index = std::make_unique<HnswIndex>(vectors, distance_func, level_generator,
+ auto generator = std::make_unique<LevelGenerator>();
+ level_generator = generator.get();
+ index = std::make_unique<HnswIndex>(vectors, std::make_unique<FloatSqEuclideanDistance>(),
+ std::move(generator),
HnswIndex::Config(2, 1, 10, heuristic_select_neighbors));
}
void add_document(uint32_t docid, uint32_t max_level = 0) {
- level_generator.level = max_level;
+ level_generator->level = max_level;
index->add_document(docid);
}
void remove_document(uint32_t docid) {
@@ -100,8 +102,10 @@ public:
if (exp_hits.size() == k) {
std::vector<uint32_t> expected_by_docid = exp_hits;
std::sort(expected_by_docid.begin(), expected_by_docid.end());
- std::vector<uint32_t> got_by_docid = index->find_top_k(k, qv, k);
- EXPECT_EQ(expected_by_docid, got_by_docid);
+ auto got_by_docid = index->find_top_k(k, qv, k);
+ for (idx = 0; idx < k; ++idx) {
+ EXPECT_EQ(expected_by_docid[idx], got_by_docid[idx].docid);
+ }
}
}
};
@@ -262,5 +266,38 @@ TEST_F(HnswIndexTest, 2d_vectors_inserted_in_hierarchic_graph_with_heuristic_sel
expect_level_0(7, {3, 6});
}
+TEST_F(HnswIndexTest, manual_insert)
+{
+ init(false);
+
+ std::vector<uint32_t> nbl;
+ HnswNode empty{nbl};
+ index->set_node(1, empty);
+ index->set_node(2, empty);
+
+ HnswNode three{{1,2}};
+ index->set_node(3, three);
+ expect_level_0(1, {3});
+ expect_level_0(2, {3});
+ expect_level_0(3, {1,2});
+
+ expect_entry_point(1, 0);
+
+ HnswNode twolevels{{{1},nbl}};
+ index->set_node(4, twolevels);
+
+ expect_entry_point(4, 1);
+ expect_level_0(1, {3,4});
+
+ HnswNode five{{{1,2}, {4}}};
+ index->set_node(5, five);
+
+ expect_levels(1, {{3,4,5}});
+ expect_levels(2, {{3,5}});
+ expect_levels(3, {{1,2}});
+ expect_levels(4, {{1}, {5}});
+ expect_levels(5, {{1,2}, {4}});
+}
+
GTEST_MAIN_RUN_ALL_TESTS()
diff --git a/searchlib/src/vespa/searchlib/attribute/configconverter.cpp b/searchlib/src/vespa/searchlib/attribute/configconverter.cpp
index 535e81fc032..10e1a1edb52 100644
--- a/searchlib/src/vespa/searchlib/attribute/configconverter.cpp
+++ b/searchlib/src/vespa/searchlib/attribute/configconverter.cpp
@@ -73,6 +73,10 @@ ConfigConverter::convert(const AttributesConfig::Attribute & cfg)
predicateParams.setBounds(cfg.lowerbound, cfg.upperbound);
predicateParams.setDensePostingListThreshold(cfg.densepostinglistthreshold);
retval.setPredicateParams(predicateParams);
+ if (cfg.index.hnsw.enabled) {
+ retval.set_hnsw_index_params(HnswIndexParams(cfg.index.hnsw.maxlinkspernode,
+ cfg.index.hnsw.neighborstoexploreatinsert));
+ }
if (retval.basicType().type() == BasicType::Type::TENSOR) {
if (!cfg.tensortype.empty()) {
retval.setTensorType(ValueType::from_spec(cfg.tensortype));
diff --git a/searchlib/src/vespa/searchlib/common/sequencedtaskexecutor.cpp b/searchlib/src/vespa/searchlib/common/sequencedtaskexecutor.cpp
index bb779b659ab..617e4bf5c85 100644
--- a/searchlib/src/vespa/searchlib/common/sequencedtaskexecutor.cpp
+++ b/searchlib/src/vespa/searchlib/common/sequencedtaskexecutor.cpp
@@ -2,7 +2,6 @@
#include "sequencedtaskexecutor.h"
#include <vespa/vespalib/util/blockingthreadstackexecutor.h>
-#include <vespa/vespalib/stllike/hash_map.hpp>
using vespalib::BlockingThreadStackExecutor;
diff --git a/searchlib/src/vespa/searchlib/common/sequencedtaskexecutor.h b/searchlib/src/vespa/searchlib/common/sequencedtaskexecutor.h
index 2b7e70d69c7..9337f393150 100644
--- a/searchlib/src/vespa/searchlib/common/sequencedtaskexecutor.h
+++ b/searchlib/src/vespa/searchlib/common/sequencedtaskexecutor.h
@@ -2,7 +2,6 @@
#pragma once
#include "isequencedtaskexecutor.h"
-#include <vespa/vespalib/stllike/hash_map.h>
#include <vector>
namespace vespalib {
@@ -16,7 +15,7 @@ namespace search {
* Class to run multiple tasks in parallel, but tasks with same
* id has to be run in sequence.
*/
-class SequencedTaskExecutor : public ISequencedTaskExecutor
+class SequencedTaskExecutor final : public ISequencedTaskExecutor
{
using Stats = vespalib::ExecutorStats;
std::vector<std::shared_ptr<vespalib::BlockingThreadStackExecutor>> _executors;
diff --git a/searchlib/src/vespa/searchlib/features/agefeature.cpp b/searchlib/src/vespa/searchlib/features/agefeature.cpp
index 258648408f8..e93691e7241 100644
--- a/searchlib/src/vespa/searchlib/features/agefeature.cpp
+++ b/searchlib/src/vespa/searchlib/features/agefeature.cpp
@@ -4,6 +4,7 @@
#include "valuefeature.h"
#include <vespa/searchlib/fef/featurenamebuilder.h>
#include <vespa/searchlib/fef/matchdata.h>
+#include <vespa/vespalib/util/stash.h>
using search::attribute::IAttributeVector;
@@ -18,20 +19,18 @@ AgeExecutor::AgeExecutor(const IAttributeVector *attribute) :
_attribute(attribute),
_buf()
{
- if (_attribute != NULL) {
+ if (_attribute != nullptr) {
_buf.allocate(attribute->getMaxValueCount());
}
}
-AgeBlueprint::~AgeBlueprint()
-{
-}
+AgeBlueprint::~AgeBlueprint() = default;
void
AgeExecutor::execute(uint32_t docId)
{
feature_t age = 10000000000.0;
- if (_attribute != NULL) {
+ if (_attribute != nullptr) {
_buf.fill(*_attribute, docId);
int64_t docTime = _buf[0];
feature_t currTime = inputs().get_number(0);
@@ -65,7 +64,7 @@ AgeBlueprint::setup(const search::fef::IIndexEnvironment &env,
search::fef::Blueprint::UP
AgeBlueprint::createInstance() const
{
- return search::fef::Blueprint::UP(new AgeBlueprint());
+ return std::make_unique<AgeBlueprint>();
}
search::fef::FeatureExecutor &
diff --git a/searchlib/src/vespa/searchlib/features/attributematchfeature.cpp b/searchlib/src/vespa/searchlib/features/attributematchfeature.cpp
index 4776437a14b..c26d18eb11c 100644
--- a/searchlib/src/vespa/searchlib/features/attributematchfeature.cpp
+++ b/searchlib/src/vespa/searchlib/features/attributematchfeature.cpp
@@ -8,6 +8,7 @@
#include <vespa/searchlib/fef/properties.h>
#include <vespa/searchlib/fef/parameterdescriptions.h>
#include <vespa/searchcommon/attribute/attributecontent.h>
+#include <vespa/vespalib/util/stash.h>
#include <vespa/log/log.h>
LOG_SETUP(".features.attributematchfeature");
diff --git a/searchlib/src/vespa/searchlib/features/bm25_feature.cpp b/searchlib/src/vespa/searchlib/features/bm25_feature.cpp
index e16b4bba996..64e365b25ac 100644
--- a/searchlib/src/vespa/searchlib/features/bm25_feature.cpp
+++ b/searchlib/src/vespa/searchlib/features/bm25_feature.cpp
@@ -6,8 +6,8 @@
#include <vespa/searchlib/fef/itermfielddata.h>
#include <vespa/searchlib/fef/objectstore.h>
#include <vespa/searchlib/fef/properties.h>
+#include <vespa/vespalib/util/stash.h>
#include <cmath>
-#include <memory>
#include <stdexcept>
#include <vespa/log/log.h>
diff --git a/searchlib/src/vespa/searchlib/features/closenessfeature.cpp b/searchlib/src/vespa/searchlib/features/closenessfeature.cpp
index 2358e54e9f5..ccd76f5435f 100644
--- a/searchlib/src/vespa/searchlib/features/closenessfeature.cpp
+++ b/searchlib/src/vespa/searchlib/features/closenessfeature.cpp
@@ -3,14 +3,14 @@
#include "closenessfeature.h"
#include "utils.h"
#include <vespa/searchlib/fef/properties.h>
+#include <vespa/vespalib/util/stash.h>
#include <vespa/log/log.h>
LOG_SETUP(".features.closenessfeature");
using namespace search::fef;
-namespace search {
-namespace features {
+namespace search::features {
ClosenessExecutor::ClosenessExecutor(feature_t maxDistance, feature_t scaleDistance) :
FeatureExecutor(),
@@ -96,7 +96,7 @@ ClosenessBlueprint::setup(const IIndexEnvironment & env,
Blueprint::UP
ClosenessBlueprint::createInstance() const
{
- return Blueprint::UP(new ClosenessBlueprint());
+ return std::make_unique<ClosenessBlueprint>();
}
FeatureExecutor &
@@ -105,6 +105,4 @@ ClosenessBlueprint::createExecutor(const IQueryEnvironment &, vespalib::Stash &s
return stash.create<ClosenessExecutor>(_maxDistance, _scaleDistance);
}
-
-} // namespace features
-} // namespace search
+}
diff --git a/searchlib/src/vespa/searchlib/features/constant_feature.cpp b/searchlib/src/vespa/searchlib/features/constant_feature.cpp
index ced9d95fb33..7fc0c5c05fc 100644
--- a/searchlib/src/vespa/searchlib/features/constant_feature.cpp
+++ b/searchlib/src/vespa/searchlib/features/constant_feature.cpp
@@ -4,14 +4,14 @@
#include "valuefeature.h"
#include <vespa/searchlib/fef/featureexecutor.h>
#include <vespa/eval/eval/value_cache/constant_value.h>
+#include <vespa/vespalib/util/stash.h>
#include <vespa/log/log.h>
LOG_SETUP(".features.constant_feature");
using namespace search::fef;
-namespace search {
-namespace features {
+namespace search::features {
/**
* Feature executor that returns a constant value.
@@ -25,8 +25,8 @@ public:
ConstantFeatureExecutor(const vespalib::eval::Value &value)
: _value(value)
{}
- virtual bool isPure() override { return true; }
- virtual void execute(uint32_t) override {
+ bool isPure() override { return true; }
+ void execute(uint32_t) override {
outputs().set_object(0, _value);
}
static FeatureExecutor &create(const vespalib::eval::Value &value, vespalib::Stash &stash) {
@@ -41,9 +41,7 @@ ConstantBlueprint::ConstantBlueprint()
{
}
-ConstantBlueprint::~ConstantBlueprint()
-{
-}
+ConstantBlueprint::~ConstantBlueprint() = default;
void
ConstantBlueprint::visitDumpFeatures(const IIndexEnvironment &,
@@ -54,7 +52,7 @@ ConstantBlueprint::visitDumpFeatures(const IIndexEnvironment &,
Blueprint::UP
ConstantBlueprint::createInstance() const
{
- return Blueprint::UP(new ConstantBlueprint());
+ return std::make_unique<ConstantBlueprint>();
}
bool
@@ -88,5 +86,4 @@ ConstantBlueprint::createExecutor(const IQueryEnvironment &env, vespalib::Stash
}
}
-} // namespace features
-} // namespace search
+}
diff --git a/searchlib/src/vespa/searchlib/features/constant_tensor_executor.h b/searchlib/src/vespa/searchlib/features/constant_tensor_executor.h
index 2ab7b98fabe..785eb357795 100644
--- a/searchlib/src/vespa/searchlib/features/constant_tensor_executor.h
+++ b/searchlib/src/vespa/searchlib/features/constant_tensor_executor.h
@@ -7,7 +7,7 @@
#include <vespa/eval/eval/value.h>
#include <vespa/eval/eval/value_type.h>
#include <vespa/eval/tensor/default_tensor_engine.h>
-#include <memory>
+#include <vespa/vespalib/util/stash.h>
namespace search::features {
diff --git a/searchlib/src/vespa/searchlib/features/debug_attribute_wait.cpp b/searchlib/src/vespa/searchlib/features/debug_attribute_wait.cpp
index 86a71184f43..8cc75a9a424 100644
--- a/searchlib/src/vespa/searchlib/features/debug_attribute_wait.cpp
+++ b/searchlib/src/vespa/searchlib/features/debug_attribute_wait.cpp
@@ -2,6 +2,8 @@
#include "debug_attribute_wait.h"
#include <vespa/vespalib/util/time.h>
+#include <vespa/vespalib/util/stash.h>
+
using search::attribute::IAttributeVector;
using namespace search::fef;
diff --git a/searchlib/src/vespa/searchlib/features/debug_wait.cpp b/searchlib/src/vespa/searchlib/features/debug_wait.cpp
index fb002564572..57d19618ba4 100644
--- a/searchlib/src/vespa/searchlib/features/debug_wait.cpp
+++ b/searchlib/src/vespa/searchlib/features/debug_wait.cpp
@@ -2,6 +2,7 @@
#include "debug_wait.h"
#include <vespa/vespalib/util/time.h>
+#include <vespa/vespalib/util/stash.h>
using namespace search::fef;
diff --git a/searchlib/src/vespa/searchlib/features/distancefeature.cpp b/searchlib/src/vespa/searchlib/features/distancefeature.cpp
index 501bfd7cd14..4d7d77fe315 100644
--- a/searchlib/src/vespa/searchlib/features/distancefeature.cpp
+++ b/searchlib/src/vespa/searchlib/features/distancefeature.cpp
@@ -5,6 +5,7 @@
#include <vespa/searchlib/fef/matchdata.h>
#include <vespa/document/datatype/positiondatatype.h>
#include <vespa/vespalib/geo/zcurve.h>
+#include <vespa/vespalib/util/stash.h>
#include <cmath>
#include <limits>
@@ -13,13 +14,12 @@ LOG_SETUP(".features.distancefeature");
using namespace search::fef;
-namespace search {
-namespace features {
+namespace search::features {
feature_t
DistanceExecutor::calculateDistance(uint32_t docId)
{
- if (_location.isValid() && _pos != NULL) {
+ if (_location.isValid() && _pos != nullptr) {
return calculate2DZDistance(docId);
}
return DEFAULT_DISTANCE;
@@ -66,7 +66,7 @@ DistanceExecutor::DistanceExecutor(const Location & location,
_pos(pos),
_intBuf()
{
- if (_pos != NULL) {
+ if (_pos != nullptr) {
_intBuf.allocate(_pos->getMaxValueCount());
}
}
@@ -86,9 +86,7 @@ DistanceBlueprint::DistanceBlueprint() :
{
}
-DistanceBlueprint::~DistanceBlueprint()
-{
-}
+DistanceBlueprint::~DistanceBlueprint() = default;
void
DistanceBlueprint::visitDumpFeatures(const IIndexEnvironment &,
@@ -99,7 +97,7 @@ DistanceBlueprint::visitDumpFeatures(const IIndexEnvironment &,
Blueprint::UP
DistanceBlueprint::createInstance() const
{
- return Blueprint::UP(new DistanceBlueprint());
+ return std::make_unique<DistanceBlueprint>();
}
bool
@@ -116,26 +114,26 @@ DistanceBlueprint::setup(const IIndexEnvironment & env,
FeatureExecutor &
DistanceBlueprint::createExecutor(const IQueryEnvironment &env, vespalib::Stash &stash) const
{
- const search::attribute::IAttributeVector * pos = NULL;
+ const search::attribute::IAttributeVector * pos = nullptr;
const Location & location = env.getLocation();
LOG(debug, "DistanceBlueprint::createExecutor location.valid='%s', '%s', alternatively '%s'",
location.isValid() ? "true" : "false", _posAttr.c_str(), document::PositionDataType::getZCurveFieldName(_posAttr).c_str());
if (location.isValid()) {
pos = env.getAttributeContext().getAttribute(_posAttr);
- if (pos == NULL) {
+ if (pos == nullptr) {
LOG(debug, "Failed to find attribute '%s', resorting too '%s'",
_posAttr.c_str(), document::PositionDataType::getZCurveFieldName(_posAttr).c_str());
pos = env.getAttributeContext().getAttribute(document::PositionDataType::getZCurveFieldName(_posAttr));
}
- if (pos != NULL) {
+ if (pos != nullptr) {
if (!pos->isIntegerType()) {
LOG(warning, "The position attribute '%s' is not an integer attribute. Will use default distance.",
pos->getName().c_str());
- pos = NULL;
+ pos = nullptr;
} else if (pos->getCollectionType() == attribute::CollectionType::WSET) {
LOG(warning, "The position attribute '%s' is a weighted set attribute. Will use default distance.",
pos->getName().c_str());
- pos = NULL;
+ pos = nullptr;
}
} else {
LOG(warning, "The position attribute '%s' was not found. Will use default distance.", _posAttr.c_str());
@@ -145,7 +143,4 @@ DistanceBlueprint::createExecutor(const IQueryEnvironment &env, vespalib::Stash
return stash.create<DistanceExecutor>(location, pos);
}
-
-
-} // namespace features
-} // namespace search
+}
diff --git a/searchlib/src/vespa/searchlib/features/distancetopathfeature.cpp b/searchlib/src/vespa/searchlib/features/distancetopathfeature.cpp
index 834f5913af9..a2f9225d3c4 100644
--- a/searchlib/src/vespa/searchlib/features/distancetopathfeature.cpp
+++ b/searchlib/src/vespa/searchlib/features/distancetopathfeature.cpp
@@ -6,10 +6,10 @@
#include <vespa/searchlib/fef/properties.h>
#include <vespa/document/datatype/positiondatatype.h>
#include <vespa/vespalib/geo/zcurve.h>
+#include <vespa/vespalib/util/stash.h>
#include <boost/algorithm/string/split.hpp>
#include <boost/algorithm/string/classification.hpp>
#include <cmath>
-#include <sstream>
#include <vespa/log/log.h>
LOG_SETUP(".features.distancetopathfeature");
@@ -25,7 +25,7 @@ DistanceToPathExecutor::DistanceToPathExecutor(std::vector<Vector2> &path,
_path(),
_pos(pos)
{
- if (_pos != NULL) {
+ if (_pos != nullptr) {
_intBuf.allocate(_pos->getMaxValueCount());
}
_path.swap(path); // avoid copy
@@ -34,7 +34,7 @@ DistanceToPathExecutor::DistanceToPathExecutor(std::vector<Vector2> &path,
void
DistanceToPathExecutor::execute(uint32_t docId)
{
- if (_path.size() > 1 && _pos != NULL) {
+ if (_path.size() > 1 && _pos != nullptr) {
double pos = -1, trip = 0, product = 0;
double minSqDist = std::numeric_limits<double>::max();
_intBuf.fill(*_pos, docId);
@@ -145,21 +145,21 @@ DistanceToPathBlueprint::createExecutor(const search::fef::IQueryEnvironment &en
}
// Lookup the attribute vector that holds document positions.
- const search::attribute::IAttributeVector *pos = NULL;
+ const search::attribute::IAttributeVector *pos = nullptr;
if (path.size() > 1) {
pos = env.getAttributeContext().getAttribute(_posAttr);
- if (pos == NULL) {
+ if (pos == nullptr) {
pos = env.getAttributeContext().getAttribute(document::PositionDataType::getZCurveFieldName(_posAttr));
}
- if (pos != NULL) {
+ if (pos != nullptr) {
if (!pos->isIntegerType()) {
LOG(warning, "The position attribute '%s' is not an integer attribute. Will use default distance.",
pos->getName().c_str());
- pos = NULL;
+ pos = nullptr;
} else if (pos->getCollectionType() == attribute::CollectionType::WSET) {
LOG(warning, "The position attribute '%s' is a weighted set attribute. Will use default distance.",
pos->getName().c_str());
- pos = NULL;
+ pos = nullptr;
}
} else {
LOG(warning, "The position attribute '%s' was not found. Will use default distance.", _posAttr.c_str());
diff --git a/searchlib/src/vespa/searchlib/features/dotproductfeature.cpp b/searchlib/src/vespa/searchlib/features/dotproductfeature.cpp
index 8998f01b59e..1072607aa8a 100644
--- a/searchlib/src/vespa/searchlib/features/dotproductfeature.cpp
+++ b/searchlib/src/vespa/searchlib/features/dotproductfeature.cpp
@@ -10,12 +10,11 @@
#include <vespa/searchlib/attribute/floatbase.h>
#include <vespa/searchlib/attribute/multinumericattribute.h>
#include <vespa/searchlib/attribute/multienumattribute.h>
-#include <type_traits>
-
-#include <vespa/log/log.h>
#include <vespa/eval/tensor/serialization/typed_binary_format.h>
#include <vespa/vespalib/objects/nbostream.h>
+#include <vespa/vespalib/util/stash.h>
+#include <vespa/log/log.h>
LOG_SETUP(".features.dotproduct");
using namespace search::attribute;
diff --git a/searchlib/src/vespa/searchlib/features/element_completeness_feature.cpp b/searchlib/src/vespa/searchlib/features/element_completeness_feature.cpp
index 1622a87e733..1014fe4679a 100644
--- a/searchlib/src/vespa/searchlib/features/element_completeness_feature.cpp
+++ b/searchlib/src/vespa/searchlib/features/element_completeness_feature.cpp
@@ -5,6 +5,7 @@
#include <vespa/searchlib/fef/featurenamebuilder.h>
#include <vespa/searchlib/fef/properties.h>
#include <vespa/vespalib/locale/c.h>
+#include <vespa/vespalib/util/stash.h>
#include <cassert>
namespace search::features {
@@ -116,7 +117,7 @@ ElementCompletenessBlueprint::visitDumpFeatures(const fef::IIndexEnvironment &en
fef::Blueprint::UP
ElementCompletenessBlueprint::createInstance() const
{
- return Blueprint::UP(new ElementCompletenessBlueprint());
+ return std::make_unique<ElementCompletenessBlueprint>();
}
bool
diff --git a/searchlib/src/vespa/searchlib/features/element_similarity_feature.cpp b/searchlib/src/vespa/searchlib/features/element_similarity_feature.cpp
index 0676b0a46c4..45fcd013fbf 100644
--- a/searchlib/src/vespa/searchlib/features/element_similarity_feature.cpp
+++ b/searchlib/src/vespa/searchlib/features/element_similarity_feature.cpp
@@ -6,6 +6,7 @@
#include <vespa/searchlib/fef/properties.h>
#include <vespa/eval/eval/llvm/compiled_function.h>
#include <vespa/eval/eval/llvm/compile_cache.h>
+#include <vespa/vespalib/util/stash.h>
#include <vespa/log/log.h>
LOG_SETUP(".features.elementsimilarity");
diff --git a/searchlib/src/vespa/searchlib/features/euclidean_distance_feature.cpp b/searchlib/src/vespa/searchlib/features/euclidean_distance_feature.cpp
index cd007f1396f..9d71d358b46 100644
--- a/searchlib/src/vespa/searchlib/features/euclidean_distance_feature.cpp
+++ b/searchlib/src/vespa/searchlib/features/euclidean_distance_feature.cpp
@@ -4,9 +4,9 @@
#include "euclidean_distance_feature.h"
#include "array_parser.hpp"
#include <vespa/searchlib/attribute/integerbase.h>
-#include <vespa/searchlib/attribute/floatbase.h>
#include <vespa/searchlib/fef/properties.h>
#include <vespa/searchcommon/attribute/attributecontent.h>
+#include <vespa/vespalib/util/stash.h>
#include <cmath>
#include <vespa/log/log.h>
@@ -15,8 +15,7 @@ LOG_SETUP(".features.euclidean_distance_feature");
using namespace search::attribute;
using namespace search::fef;
-namespace search {
-namespace features {
+namespace search::features {
template <typename DataType>
@@ -57,7 +56,7 @@ EuclideanDistanceBlueprint::EuclideanDistanceBlueprint() :
{
}
-EuclideanDistanceBlueprint::~EuclideanDistanceBlueprint() {}
+EuclideanDistanceBlueprint::~EuclideanDistanceBlueprint() = default;
void
EuclideanDistanceBlueprint::visitDumpFeatures(const IIndexEnvironment &, IDumpFeatureVisitor &) const
@@ -78,7 +77,7 @@ EuclideanDistanceBlueprint::setup(const IIndexEnvironment &env, const ParameterL
Blueprint::UP
EuclideanDistanceBlueprint::createInstance() const
{
- return Blueprint::UP(new EuclideanDistanceBlueprint());
+ return std::make_unique<EuclideanDistanceBlueprint>();
}
namespace {
@@ -97,7 +96,7 @@ FeatureExecutor &
EuclideanDistanceBlueprint::createExecutor(const IQueryEnvironment &env, vespalib::Stash &stash) const
{
const IAttributeVector * attribute = env.getAttributeContext().getAttribute(_attributeName);
- if (attribute == NULL) {
+ if (attribute == nullptr) {
LOG(warning, "The attribute vector '%s' was not found in the attribute manager, returning executor with default value.",
_attributeName.c_str());
return stash.create<SingleZeroValueExecutor>();
@@ -118,6 +117,5 @@ EuclideanDistanceBlueprint::createExecutor(const IQueryEnvironment &env, vespali
}
-} // namespace features
-} // namespace search
+}
diff --git a/searchlib/src/vespa/searchlib/features/fieldinfofeature.cpp b/searchlib/src/vespa/searchlib/features/fieldinfofeature.cpp
index d7e17187ff4..18a15ccf541 100644
--- a/searchlib/src/vespa/searchlib/features/fieldinfofeature.cpp
+++ b/searchlib/src/vespa/searchlib/features/fieldinfofeature.cpp
@@ -7,12 +7,11 @@
#include <vespa/searchlib/fef/fieldinfo.h>
#include <vespa/searchlib/fef/fieldtype.h>
#include <vespa/searchlib/fef/featurenamebuilder.h>
-#include <vespa/searchlib/fef/itermdata.h>
#include <vespa/searchlib/fef/handle.h>
+#include <vespa/vespalib/util/stash.h>
#include <sstream>
-namespace search {
-namespace features {
+namespace search::features {
IndexFieldInfoExecutor::IndexFieldInfoExecutor(feature_t type, feature_t isFilter,
[[maybe_unused]] uint32_t field, uint32_t fieldHandle)
@@ -238,5 +237,4 @@ FieldInfoBlueprint::createExecutor(const fef::IQueryEnvironment &queryEnv, vespa
return stash.create<ValueExecutor>(values);
}
-} // namespace features
-} // namespace search
+}
diff --git a/searchlib/src/vespa/searchlib/features/fieldlengthfeature.cpp b/searchlib/src/vespa/searchlib/features/fieldlengthfeature.cpp
index d0680e8fc19..74bfa156b5f 100644
--- a/searchlib/src/vespa/searchlib/features/fieldlengthfeature.cpp
+++ b/searchlib/src/vespa/searchlib/features/fieldlengthfeature.cpp
@@ -6,13 +6,12 @@
#include <vespa/searchlib/fef/itermdata.h>
#include <vespa/searchlib/fef/featurenamebuilder.h>
#include <vespa/searchlib/fef/fieldinfo.h>
-#include <vespa/searchlib/fef/fieldtype.h>
-#include <vespa/vespalib/util/stringfmt.h>
+#include <vespa/vespalib/util/stash.h>
+
using namespace search::fef;
-namespace search {
-namespace features {
+namespace search::features {
FieldLengthExecutor::
FieldLengthExecutor(const IQueryEnvironment &env,
@@ -63,7 +62,7 @@ FieldLengthExecutor::handle_bind_match_data(const MatchData &md)
FieldLengthBlueprint::FieldLengthBlueprint()
: Blueprint("fieldLength"),
- _field(NULL)
+ _field(nullptr)
{
}
@@ -86,7 +85,7 @@ FieldLengthBlueprint::setup(const IIndexEnvironment &env,
Blueprint::UP
FieldLengthBlueprint::createInstance() const
{
- return Blueprint::UP(new FieldLengthBlueprint());
+ return std::make_unique<FieldLengthBlueprint>();
}
FeatureExecutor &
@@ -100,4 +99,4 @@ FieldLengthBlueprint::createExecutor(const IQueryEnvironment &env, vespalib::Sta
return stash.create<FieldLengthExecutor>(env, _field->id());
}
-}}
+}
diff --git a/searchlib/src/vespa/searchlib/features/fieldmatchfeature.cpp b/searchlib/src/vespa/searchlib/features/fieldmatchfeature.cpp
index 583afa6e698..7ad9eef0506 100644
--- a/searchlib/src/vespa/searchlib/features/fieldmatchfeature.cpp
+++ b/searchlib/src/vespa/searchlib/features/fieldmatchfeature.cpp
@@ -7,6 +7,7 @@
#include <vespa/searchlib/fef/properties.h>
#include <vespa/vespalib/util/stringfmt.h>
#include <vespa/vespalib/locale/c.h>
+#include <vespa/vespalib/util/stash.h>
using namespace search::fef;
using CollectionType = FieldInfo::CollectionType;
@@ -92,14 +93,12 @@ FieldMatchExecutor::handle_bind_match_data(const fef::MatchData &md)
FieldMatchBlueprint::FieldMatchBlueprint() :
Blueprint("fieldMatch"),
- _field(NULL),
+ _field(nullptr),
_params()
{
}
-FieldMatchBlueprint::~FieldMatchBlueprint()
-{
-}
+FieldMatchBlueprint::~FieldMatchBlueprint() = default;
void
FieldMatchBlueprint::visitDumpFeatures(const IIndexEnvironment & env,
@@ -158,7 +157,7 @@ FieldMatchBlueprint::visitDumpFeatures(const IIndexEnvironment & env,
Blueprint::UP
FieldMatchBlueprint::createInstance() const
{
- return Blueprint::UP(new FieldMatchBlueprint());
+ return std::make_unique<FieldMatchBlueprint>();
}
bool
@@ -306,5 +305,4 @@ FieldMatchBlueprint::createExecutor(const IQueryEnvironment & env, vespalib::Sta
return stash.create<FieldMatchExecutor>(env, *_field, _params);
}
-
}
diff --git a/searchlib/src/vespa/searchlib/features/fieldtermmatchfeature.cpp b/searchlib/src/vespa/searchlib/features/fieldtermmatchfeature.cpp
index a7a00bee956..2f065fee289 100644
--- a/searchlib/src/vespa/searchlib/features/fieldtermmatchfeature.cpp
+++ b/searchlib/src/vespa/searchlib/features/fieldtermmatchfeature.cpp
@@ -8,9 +8,10 @@
#include <vespa/searchlib/fef/properties.h>
#include <vespa/searchlib/fef/itermdata.h>
#include <vespa/vespalib/util/stringfmt.h>
+#include <vespa/vespalib/util/stash.h>
-namespace search {
-namespace features {
+
+namespace search::features {
FieldTermMatchExecutor::FieldTermMatchExecutor(const search::fef::IQueryEnvironment &env,
uint32_t fieldId, uint32_t termId) :
@@ -122,7 +123,7 @@ FieldTermMatchBlueprint::setup(const search::fef::IIndexEnvironment &env,
search::fef::Blueprint::UP
FieldTermMatchBlueprint::createInstance() const
{
- return search::fef::Blueprint::UP(new FieldTermMatchBlueprint());
+ return std::make_unique<FieldTermMatchBlueprint>();
}
search::fef::FeatureExecutor &
@@ -131,4 +132,4 @@ FieldTermMatchBlueprint::createExecutor(const search::fef::IQueryEnvironment &en
return stash.create<FieldTermMatchExecutor>(env, _fieldId, _termId);
}
-}}
+}
diff --git a/searchlib/src/vespa/searchlib/features/firstphasefeature.cpp b/searchlib/src/vespa/searchlib/features/firstphasefeature.cpp
index 2e6bae14a44..9d1831c6102 100644
--- a/searchlib/src/vespa/searchlib/features/firstphasefeature.cpp
+++ b/searchlib/src/vespa/searchlib/features/firstphasefeature.cpp
@@ -4,11 +4,12 @@
#include <vespa/searchlib/fef/featureexecutor.h>
#include <vespa/searchlib/fef/indexproperties.h>
#include <vespa/searchlib/fef/properties.h>
+#include <vespa/vespalib/util/stash.h>
+
using namespace search::fef;
-namespace search {
-namespace features {
+namespace search::features {
void
FirstPhaseExecutor::execute(uint32_t)
@@ -34,7 +35,7 @@ FirstPhaseBlueprint::visitDumpFeatures(const IIndexEnvironment &,
Blueprint::UP
FirstPhaseBlueprint::createInstance() const
{
- return Blueprint::UP(new FirstPhaseBlueprint());
+ return std::make_unique<FirstPhaseBlueprint>();
}
bool
@@ -54,5 +55,4 @@ FirstPhaseBlueprint::createExecutor(const IQueryEnvironment &, vespalib::Stash &
}
-} // namespace features
-} // namespace search
+}
diff --git a/searchlib/src/vespa/searchlib/features/flow_completeness_feature.cpp b/searchlib/src/vespa/searchlib/features/flow_completeness_feature.cpp
index eda83b991bf..e43faaec4e1 100644
--- a/searchlib/src/vespa/searchlib/features/flow_completeness_feature.cpp
+++ b/searchlib/src/vespa/searchlib/features/flow_completeness_feature.cpp
@@ -7,6 +7,8 @@
#include <vespa/searchlib/fef/indexproperties.h>
#include <vespa/vespalib/stllike/hash_map.h>
#include <vespa/vespalib/locale/c.h>
+#include <vespa/vespalib/util/stash.h>
+
#include <cassert>
#include <vespa/log/log.h>
@@ -285,7 +287,7 @@ FlowCompletenessBlueprint::visitDumpFeatures(const fef::IIndexEnvironment &env,
fef::Blueprint::UP
FlowCompletenessBlueprint::createInstance() const
{
- return Blueprint::UP(new FlowCompletenessBlueprint());
+ return std::make_unique<FlowCompletenessBlueprint>();
}
bool
@@ -318,6 +320,4 @@ FlowCompletenessBlueprint::createExecutor(const fef::IQueryEnvironment &env, ves
return stash.create<FlowCompletenessExecutor>(env, _params);
}
-//-----------------------------------------------------------------------------
-
}
diff --git a/searchlib/src/vespa/searchlib/features/foreachfeature.cpp b/searchlib/src/vespa/searchlib/features/foreachfeature.cpp
index a67d8001a36..21167dd23d4 100644
--- a/searchlib/src/vespa/searchlib/features/foreachfeature.cpp
+++ b/searchlib/src/vespa/searchlib/features/foreachfeature.cpp
@@ -6,6 +6,7 @@
#include <vespa/searchlib/fef/properties.h>
#include <vespa/vespalib/util/stringfmt.h>
+#include <vespa/vespalib/util/stash.h>
#include <boost/algorithm/string/replace.hpp>
#include <vespa/log/log.h>
@@ -120,9 +121,7 @@ ForeachBlueprint::ForeachBlueprint() :
{
}
-ForeachBlueprint::~ForeachBlueprint()
-{
-}
+ForeachBlueprint::~ForeachBlueprint() = default;
void
ForeachBlueprint::visitDumpFeatures(const IIndexEnvironment &,
@@ -171,13 +170,13 @@ ForeachBlueprint::setup(const IIndexEnvironment & env,
Blueprint::UP
ForeachBlueprint::createInstance() const
{
- return Blueprint::UP(new ForeachBlueprint());
+ return std::make_unique<ForeachBlueprint>();
}
FeatureExecutor &
ForeachBlueprint::createExecutor(const IQueryEnvironment &, vespalib::Stash &stash) const
{
- if (_executorCreator.get() != NULL) {
+ if (_executorCreator) {
return _executorCreator->create(_num_inputs, stash);
}
return stash.create<SingleZeroValueExecutor>();
diff --git a/searchlib/src/vespa/searchlib/features/freshnessfeature.cpp b/searchlib/src/vespa/searchlib/features/freshnessfeature.cpp
index 11ae8305e16..6e621f61034 100644
--- a/searchlib/src/vespa/searchlib/features/freshnessfeature.cpp
+++ b/searchlib/src/vespa/searchlib/features/freshnessfeature.cpp
@@ -3,14 +3,14 @@
#include "freshnessfeature.h"
#include "utils.h"
#include <vespa/searchlib/fef/properties.h>
+#include <vespa/vespalib/util/stash.h>
#include <vespa/log/log.h>
LOG_SETUP(".features.freshnessfeature");
using namespace search::fef;
-namespace search {
-namespace features {
+namespace search::features {
FreshnessExecutor::FreshnessExecutor(feature_t maxAge, feature_t scaleAge) :
FeatureExecutor(),
@@ -86,7 +86,7 @@ FreshnessBlueprint::setup(const IIndexEnvironment & env,
Blueprint::UP
FreshnessBlueprint::createInstance() const
{
- return Blueprint::UP(new FreshnessBlueprint());
+ return std::make_unique<FreshnessBlueprint>();
}
fef::ParameterDescriptions
@@ -101,7 +101,4 @@ FreshnessBlueprint::createExecutor(const IQueryEnvironment &, vespalib::Stash &s
return stash.create<FreshnessExecutor>(_maxAge, _scaleAge);
}
-
-} // namespace features
-} // namespace search
-
+}
diff --git a/searchlib/src/vespa/searchlib/features/internal_max_reduce_prod_join_feature.cpp b/searchlib/src/vespa/searchlib/features/internal_max_reduce_prod_join_feature.cpp
index fd1faeae5ea..e2e8a206099 100644
--- a/searchlib/src/vespa/searchlib/features/internal_max_reduce_prod_join_feature.cpp
+++ b/searchlib/src/vespa/searchlib/features/internal_max_reduce_prod_join_feature.cpp
@@ -5,12 +5,13 @@
#include "weighted_set_parser.h"
#include "dotproductfeature.h"
-#include <vespa/searchlib/attribute/attribute.h>
#include <vespa/searchlib/attribute/imported_attribute_vector_read_guard.h>
#include <vespa/searchlib/attribute/multinumericattribute.h>
#include <vespa/searchlib/fef/properties.h>
#include <vespa/searchlib/fef/featureexecutor.h>
#include <vespa/searchcommon/common/datatype.h>
+#include <vespa/vespalib/util/stash.h>
+
#include <vespa/log/log.h>
LOG_SETUP(".features.internalmaxreduceprodjoin");
diff --git a/searchlib/src/vespa/searchlib/features/item_raw_score_feature.cpp b/searchlib/src/vespa/searchlib/features/item_raw_score_feature.cpp
index 45baf646656..89f7751a369 100644
--- a/searchlib/src/vespa/searchlib/features/item_raw_score_feature.cpp
+++ b/searchlib/src/vespa/searchlib/features/item_raw_score_feature.cpp
@@ -3,11 +3,11 @@
#include "item_raw_score_feature.h"
#include "valuefeature.h"
#include "utils.h"
+#include <vespa/vespalib/util/stash.h>
using namespace search::fef;
-namespace search {
-namespace features {
+namespace search::features {
void
ItemRawScoreExecutor::execute(uint32_t docId)
@@ -89,6 +89,4 @@ ItemRawScoreBlueprint::resolve(const IQueryEnvironment &env,
return handles;
}
-
-} // namespace features
-} // namespace search
+}
diff --git a/searchlib/src/vespa/searchlib/features/jarowinklerdistancefeature.cpp b/searchlib/src/vespa/searchlib/features/jarowinklerdistancefeature.cpp
index a5e3e2da5ba..dc689459ff3 100644
--- a/searchlib/src/vespa/searchlib/features/jarowinklerdistancefeature.cpp
+++ b/searchlib/src/vespa/searchlib/features/jarowinklerdistancefeature.cpp
@@ -6,6 +6,8 @@
#include <vespa/searchlib/fef/properties.h>
#include <vespa/vespalib/util/stringfmt.h>
#include <vespa/vespalib/locale/c.h>
+#include <vespa/vespalib/util/stash.h>
+
namespace search::features {
@@ -168,7 +170,7 @@ JaroWinklerDistanceBlueprint::setup(const search::fef::IIndexEnvironment &env,
search::fef::Blueprint::UP
JaroWinklerDistanceBlueprint::createInstance() const
{
- return search::fef::Blueprint::UP(new JaroWinklerDistanceBlueprint());
+ return std::make_unique<JaroWinklerDistanceBlueprint>();
}
search::fef::FeatureExecutor &
diff --git a/searchlib/src/vespa/searchlib/features/matchcountfeature.cpp b/searchlib/src/vespa/searchlib/features/matchcountfeature.cpp
index fd453e17eb1..c061ace4854 100644
--- a/searchlib/src/vespa/searchlib/features/matchcountfeature.cpp
+++ b/searchlib/src/vespa/searchlib/features/matchcountfeature.cpp
@@ -3,11 +3,11 @@
#include "matchcountfeature.h"
#include "utils.h"
#include "valuefeature.h"
+#include <vespa/vespalib/util/stash.h>
using namespace search::fef;
-namespace search {
-namespace features {
+namespace search::features {
MatchCountExecutor::MatchCountExecutor(uint32_t fieldId, const IQueryEnvironment &env)
: FeatureExecutor(),
@@ -43,7 +43,7 @@ MatchCountExecutor::handle_bind_match_data(const MatchData &md)
MatchCountBlueprint::MatchCountBlueprint() :
Blueprint("matchCount"),
- _field(NULL)
+ _field(nullptr)
{
}
@@ -63,7 +63,7 @@ MatchCountBlueprint::setup(const IIndexEnvironment &, const ParameterList & para
Blueprint::UP
MatchCountBlueprint::createInstance() const
{
- return Blueprint::UP(new MatchCountBlueprint());
+ return std::make_unique<MatchCountBlueprint>();
}
FeatureExecutor &
@@ -75,5 +75,4 @@ MatchCountBlueprint::createExecutor(const IQueryEnvironment & queryEnv, vespalib
return stash.create<MatchCountExecutor>(_field->id(), queryEnv);
}
-} // namespace features
-} // namespace search
+}
diff --git a/searchlib/src/vespa/searchlib/features/matchesfeature.cpp b/searchlib/src/vespa/searchlib/features/matchesfeature.cpp
index f4788ee74c8..a99c2330ee3 100644
--- a/searchlib/src/vespa/searchlib/features/matchesfeature.cpp
+++ b/searchlib/src/vespa/searchlib/features/matchesfeature.cpp
@@ -4,11 +4,12 @@
#include "utils.h"
#include "valuefeature.h"
#include <vespa/searchlib/fef/fieldinfo.h>
+#include <vespa/vespalib/util/stash.h>
+
using namespace search::fef;
-namespace search {
-namespace features {
+namespace search::features {
MatchesExecutor::MatchesExecutor(uint32_t fieldId,
const search::fef::IQueryEnvironment &env,
@@ -47,7 +48,7 @@ MatchesExecutor::handle_bind_match_data(const MatchData &md)
MatchesBlueprint::MatchesBlueprint() :
Blueprint("matches"),
- _field(NULL),
+ _field(nullptr),
_termIdx(std::numeric_limits<uint32_t>::max())
{
}
@@ -73,7 +74,7 @@ MatchesBlueprint::setup(const IIndexEnvironment &,
Blueprint::UP
MatchesBlueprint::createInstance() const
{
- return Blueprint::UP(new MatchesBlueprint());
+ return std::make_unique<MatchesBlueprint>();
}
FeatureExecutor &
@@ -89,5 +90,4 @@ MatchesBlueprint::createExecutor(const IQueryEnvironment & queryEnv, vespalib::S
}
}
-} // namespace features
-} // namespace search
+}
diff --git a/searchlib/src/vespa/searchlib/features/matchfeature.cpp b/searchlib/src/vespa/searchlib/features/matchfeature.cpp
index 7210b8b67e9..f6843df1a2f 100644
--- a/searchlib/src/vespa/searchlib/features/matchfeature.cpp
+++ b/searchlib/src/vespa/searchlib/features/matchfeature.cpp
@@ -2,17 +2,16 @@
#include "matchfeature.h"
#include "utils.h"
-#include <vespa/searchlib/fef/featurenamebuilder.h>
#include <vespa/searchlib/fef/fieldinfo.h>
#include <vespa/searchlib/fef/indexproperties.h>
#include <vespa/searchlib/fef/properties.h>
-#include <vespa/vespalib/util/stringfmt.h>
+#include <vespa/vespalib/util/stash.h>
+
using namespace search::fef;
using CollectionType = FieldInfo::CollectionType;
-namespace search {
-namespace features {
+namespace search::features {
MatchExecutor::MatchExecutor(const MatchParams & params) :
FeatureExecutor(),
@@ -46,9 +45,7 @@ MatchBlueprint::MatchBlueprint() :
{
}
-MatchBlueprint::~MatchBlueprint()
-{
-}
+MatchBlueprint::~MatchBlueprint() = default;
void
MatchBlueprint::visitDumpFeatures(const IIndexEnvironment & env,
@@ -61,7 +58,7 @@ MatchBlueprint::visitDumpFeatures(const IIndexEnvironment & env,
Blueprint::UP
MatchBlueprint::createInstance() const
{
- return Blueprint::UP(new MatchBlueprint());
+ return std::make_unique<MatchBlueprint>();
}
bool
@@ -101,6 +98,4 @@ MatchBlueprint::createExecutor(const IQueryEnvironment &env, vespalib::Stash &st
return stash.create<MatchExecutor>(_params);
}
-
-} // namespace features
-} // namespace search
+}
diff --git a/searchlib/src/vespa/searchlib/features/native_dot_product_feature.cpp b/searchlib/src/vespa/searchlib/features/native_dot_product_feature.cpp
index 7865e32849f..5ede14130ec 100644
--- a/searchlib/src/vespa/searchlib/features/native_dot_product_feature.cpp
+++ b/searchlib/src/vespa/searchlib/features/native_dot_product_feature.cpp
@@ -2,11 +2,11 @@
#include "native_dot_product_feature.h"
#include "utils.h"
+#include <vespa/vespalib/util/stash.h>
using namespace search::fef;
-namespace search {
-namespace features {
+namespace search::features {
NativeDotProductExecutor::NativeDotProductExecutor(const search::fef::IQueryEnvironment &env)
: FeatureExecutor(),
@@ -80,5 +80,4 @@ NativeDotProductBlueprint::createExecutor(const IQueryEnvironment &queryEnv, ves
}
}
-} // namespace features
-} // namespace search
+}
diff --git a/searchlib/src/vespa/searchlib/features/nativeattributematchfeature.cpp b/searchlib/src/vespa/searchlib/features/nativeattributematchfeature.cpp
index 5dda159f629..865ea9fc3c4 100644
--- a/searchlib/src/vespa/searchlib/features/nativeattributematchfeature.cpp
+++ b/searchlib/src/vespa/searchlib/features/nativeattributematchfeature.cpp
@@ -7,11 +7,11 @@
#include <vespa/searchlib/fef/indexproperties.h>
#include <vespa/searchlib/fef/itablemanager.h>
#include <vespa/searchlib/fef/properties.h>
+#include <vespa/vespalib/util/stash.h>
using namespace search::fef;
-namespace search {
-namespace features {
+namespace search::features {
feature_t
NativeAttributeMatchExecutor::calculateScore(const CachedTermData &td, const TermFieldMatchData &tfmd)
@@ -115,7 +115,7 @@ NativeAttributeMatchBlueprint::visitDumpFeatures(const IIndexEnvironment & env,
Blueprint::UP
NativeAttributeMatchBlueprint::createInstance() const
{
- return Blueprint::UP(new NativeAttributeMatchBlueprint());
+ return std::make_unique<NativeAttributeMatchBlueprint>();
}
fef::ParameterDescriptions
@@ -160,6 +160,4 @@ NativeAttributeMatchBlueprint::createExecutor(const IQueryEnvironment &env, vesp
return NativeAttributeMatchExecutor::createExecutor(env, _params, stash);
}
-
-} // namespace features
-} // namespace search
+}
diff --git a/searchlib/src/vespa/searchlib/features/nativefieldmatchfeature.cpp b/searchlib/src/vespa/searchlib/features/nativefieldmatchfeature.cpp
index 089a8102d6e..2b6841750ad 100644
--- a/searchlib/src/vespa/searchlib/features/nativefieldmatchfeature.cpp
+++ b/searchlib/src/vespa/searchlib/features/nativefieldmatchfeature.cpp
@@ -7,11 +7,11 @@
#include <vespa/searchlib/fef/indexproperties.h>
#include <vespa/searchlib/fef/itablemanager.h>
#include <vespa/searchlib/fef/properties.h>
+#include <vespa/vespalib/util/stash.h>
using namespace search::fef;
-namespace search {
-namespace features {
+namespace search::features {
const uint32_t NativeFieldMatchParam::NOT_DEF_FIELD_LENGTH(std::numeric_limits<uint32_t>::max());
@@ -95,9 +95,7 @@ NativeFieldMatchBlueprint::NativeFieldMatchBlueprint() :
{
}
-NativeFieldMatchBlueprint::~NativeFieldMatchBlueprint()
-{
-}
+NativeFieldMatchBlueprint::~NativeFieldMatchBlueprint() = default;
void
NativeFieldMatchBlueprint::visitDumpFeatures(const IIndexEnvironment & env,
@@ -110,7 +108,7 @@ NativeFieldMatchBlueprint::visitDumpFeatures(const IIndexEnvironment & env,
Blueprint::UP
NativeFieldMatchBlueprint::createInstance() const
{
- return Blueprint::UP(new NativeFieldMatchBlueprint());
+ return std::make_unique<NativeFieldMatchBlueprint>();
}
bool
@@ -181,5 +179,4 @@ NativeFieldMatchBlueprint::createExecutor(const IQueryEnvironment &env, vespalib
}
}
-} // namespace features
-} // namespace search
+}
diff --git a/searchlib/src/vespa/searchlib/features/nativeproximityfeature.cpp b/searchlib/src/vespa/searchlib/features/nativeproximityfeature.cpp
index a31d9207e05..2809417e382 100644
--- a/searchlib/src/vespa/searchlib/features/nativeproximityfeature.cpp
+++ b/searchlib/src/vespa/searchlib/features/nativeproximityfeature.cpp
@@ -7,12 +7,12 @@
#include <vespa/searchlib/fef/indexproperties.h>
#include <vespa/searchlib/fef/itablemanager.h>
#include <vespa/searchlib/fef/properties.h>
+#include <vespa/vespalib/util/stash.h>
#include <map>
using namespace search::fef;
-namespace search {
-namespace features {
+namespace search::features {
feature_t
NativeProximityExecutor::calculateScoreForField(const FieldSetup & fs, uint32_t docId)
@@ -136,9 +136,7 @@ NativeProximityBlueprint::NativeProximityBlueprint() :
{
}
-NativeProximityBlueprint::~NativeProximityBlueprint()
-{
-}
+NativeProximityBlueprint::~NativeProximityBlueprint() = default;
void
NativeProximityBlueprint::visitDumpFeatures(const IIndexEnvironment & env,
@@ -151,7 +149,7 @@ NativeProximityBlueprint::visitDumpFeatures(const IIndexEnvironment & env,
Blueprint::UP
NativeProximityBlueprint::createInstance() const
{
- return Blueprint::UP(new NativeProximityBlueprint());
+ return std::make_unique<NativeProximityBlueprint>();
}
bool
@@ -168,12 +166,12 @@ NativeProximityBlueprint::setup(const IIndexEnvironment & env,
NativeProximityParam & param = _params.vector[fieldId];
param.field = true;
if ((param.proximityTable =
- util::lookupTable(env, getBaseName(), "proximityTable", info->name(), _defaultProximityBoost)) == NULL)
+ util::lookupTable(env, getBaseName(), "proximityTable", info->name(), _defaultProximityBoost)) == nullptr)
{
return false;
}
if ((param.revProximityTable =
- util::lookupTable(env, getBaseName(), "reverseProximityTable", info->name(), _defaultRevProximityBoost)) == NULL)
+ util::lookupTable(env, getBaseName(), "reverseProximityTable", info->name(), _defaultRevProximityBoost)) == nullptr)
{
return false;
}
@@ -190,7 +188,7 @@ NativeProximityBlueprint::setup(const IIndexEnvironment & env,
if (NativeRankBlueprint::useTableNormalization(env)) {
const Table * fp = param.proximityTable;
const Table * rp = param.revProximityTable;
- if (fp != NULL && rp != NULL) {
+ if (fp != nullptr && rp != nullptr) {
double value = (fp->max() * param.proximityImportance) +
(rp->max() * (1 - param.proximityImportance));
_params.setMaxTableSums(fieldId, value);
@@ -217,6 +215,4 @@ NativeProximityBlueprint::createExecutor(const IQueryEnvironment &env, vespalib:
}
-
-} // namespace features
-} // namespace search
+}
diff --git a/searchlib/src/vespa/searchlib/features/nativerankfeature.cpp b/searchlib/src/vespa/searchlib/features/nativerankfeature.cpp
index b519c4f4b7f..a980c265484 100644
--- a/searchlib/src/vespa/searchlib/features/nativerankfeature.cpp
+++ b/searchlib/src/vespa/searchlib/features/nativerankfeature.cpp
@@ -4,6 +4,7 @@
#include "valuefeature.h"
#include "utils.h"
#include <vespa/searchlib/fef/properties.h>
+#include <vespa/vespalib/util/stash.h>
#include <sstream>
#include <vespa/log/log.h>
@@ -30,8 +31,7 @@ buildFeatureName(const vespalib::string & baseName, const search::features::Fiel
}
-namespace search {
-namespace features {
+namespace search::features {
FieldWrapper::FieldWrapper(const IIndexEnvironment & env,
const ParameterList & fields,
@@ -93,7 +93,7 @@ NativeRankBlueprint::visitDumpFeatures(const IIndexEnvironment & env,
Blueprint::UP
NativeRankBlueprint::createInstance() const
{
- return Blueprint::UP(new NativeRankBlueprint());
+ return std::make_unique<NativeRankBlueprint>();
}
bool
@@ -168,6 +168,4 @@ NativeRankBlueprint::useTableNormalization(const search::fef::IIndexEnvironment
return (!(norm.found() && (norm.get() == vespalib::string("false"))));
}
-
-} // namespace features
-} // namespace search
+}
diff --git a/searchlib/src/vespa/searchlib/features/nowfeature.cpp b/searchlib/src/vespa/searchlib/features/nowfeature.cpp
index 074acb2e890..d6059592cf3 100644
--- a/searchlib/src/vespa/searchlib/features/nowfeature.cpp
+++ b/searchlib/src/vespa/searchlib/features/nowfeature.cpp
@@ -3,6 +3,7 @@
#include "nowfeature.h"
#include <vespa/searchlib/fef/queryproperties.h>
#include <vespa/searchlib/fef/properties.h>
+#include <vespa/vespalib/util/stash.h>
#include <chrono>
namespace search::features {
diff --git a/searchlib/src/vespa/searchlib/features/proximityfeature.cpp b/searchlib/src/vespa/searchlib/features/proximityfeature.cpp
index f625e30f378..daeb4af6569 100644
--- a/searchlib/src/vespa/searchlib/features/proximityfeature.cpp
+++ b/searchlib/src/vespa/searchlib/features/proximityfeature.cpp
@@ -2,13 +2,11 @@
#include "proximityfeature.h"
#include "utils.h"
-#include <vespa/searchlib/fef/featurenamebuilder.h>
#include <vespa/searchlib/fef/fieldinfo.h>
#include <vespa/searchlib/fef/itermdata.h>
-#include <vespa/vespalib/util/stringfmt.h>
+#include <vespa/vespalib/util/stash.h>
-namespace search {
-namespace features {
+namespace search::features {
ProximityConfig::ProximityConfig() :
fieldId(search::fef::IllegalHandle),
@@ -139,7 +137,7 @@ ProximityBlueprint::setup(const search::fef::IIndexEnvironment &env,
search::fef::Blueprint::UP
ProximityBlueprint::createInstance() const
{
- return search::fef::Blueprint::UP(new ProximityBlueprint());
+ return std::make_unique<ProximityBlueprint>();
}
search::fef::FeatureExecutor &
@@ -148,4 +146,4 @@ ProximityBlueprint::createExecutor(const search::fef::IQueryEnvironment &env, ve
return stash.create<ProximityExecutor>(env, _config);
}
-}}
+}
diff --git a/searchlib/src/vespa/searchlib/features/querycompletenessfeature.cpp b/searchlib/src/vespa/searchlib/features/querycompletenessfeature.cpp
index b4b6a1b0eb4..56e8810e14b 100644
--- a/searchlib/src/vespa/searchlib/features/querycompletenessfeature.cpp
+++ b/searchlib/src/vespa/searchlib/features/querycompletenessfeature.cpp
@@ -3,15 +3,14 @@
#include "querycompletenessfeature.h"
#include "utils.h"
#include <vespa/searchlib/fef/featurenamebuilder.h>
-#include <vespa/searchlib/fef/fieldinfo.h>
#include <vespa/searchlib/fef/itermdata.h>
+#include <vespa/vespalib/util/stash.h>
#include <limits>
#include <vespa/log/log.h>
LOG_SETUP(".features.querycompleteness");
-namespace search {
-namespace features {
+namespace search::features {
QueryCompletenessConfig::QueryCompletenessConfig() :
fieldId(search::fef::IllegalHandle),
@@ -106,7 +105,7 @@ QueryCompletenessBlueprint::setup(const search::fef::IIndexEnvironment &env,
search::fef::Blueprint::UP
QueryCompletenessBlueprint::createInstance() const
{
- return search::fef::Blueprint::UP(new QueryCompletenessBlueprint());
+ return std::make_unique<QueryCompletenessBlueprint>();
}
search::fef::FeatureExecutor &
@@ -115,4 +114,4 @@ QueryCompletenessBlueprint::createExecutor(const search::fef::IQueryEnvironment
return stash.create<QueryCompletenessExecutor>(env, _config);
}
-}}
+}
diff --git a/searchlib/src/vespa/searchlib/features/querytermcountfeature.cpp b/searchlib/src/vespa/searchlib/features/querytermcountfeature.cpp
index dfc4af059a1..cbaf9e97cb6 100644
--- a/searchlib/src/vespa/searchlib/features/querytermcountfeature.cpp
+++ b/searchlib/src/vespa/searchlib/features/querytermcountfeature.cpp
@@ -4,15 +4,14 @@
#include "valuefeature.h"
#include <vespa/searchlib/fef/properties.h>
#include <vespa/searchlib/fef/fieldinfo.h>
-#include <vespa/searchlib/fef/fieldtype.h>
#include <vespa/searchlib/fef/featurenamebuilder.h>
#include <vespa/searchlib/fef/itermdata.h>
-#include <vespa/searchlib/fef/handle.h>
+#include <vespa/vespalib/util/stash.h>
+
using namespace search::fef;
-namespace search {
-namespace features {
+namespace search::features {
QueryTermCountBlueprint::QueryTermCountBlueprint() :
Blueprint("queryTermCount")
@@ -30,7 +29,7 @@ QueryTermCountBlueprint::visitDumpFeatures(const IIndexEnvironment & env,
Blueprint::UP
QueryTermCountBlueprint::createInstance() const
{
- return Blueprint::UP(new QueryTermCountBlueprint());
+ return std::make_unique<QueryTermCountBlueprint>();
}
bool
@@ -49,6 +48,4 @@ QueryTermCountBlueprint::createExecutor(const IQueryEnvironment &env, vespalib::
return stash.create<ValueExecutor>(values);
}
-
-} // namespace features
-} // namespace search
+}
diff --git a/searchlib/src/vespa/searchlib/features/random_normal_feature.cpp b/searchlib/src/vespa/searchlib/features/random_normal_feature.cpp
index dd0c67df45c..8d4af8fd88d 100644
--- a/searchlib/src/vespa/searchlib/features/random_normal_feature.cpp
+++ b/searchlib/src/vespa/searchlib/features/random_normal_feature.cpp
@@ -3,6 +3,7 @@
#include "random_normal_feature.h"
#include "utils.h"
#include <vespa/searchlib/fef/properties.h>
+#include <vespa/vespalib/util/stash.h>
#include <chrono>
#include <vespa/log/log.h>
diff --git a/searchlib/src/vespa/searchlib/features/random_normal_stable_feature.cpp b/searchlib/src/vespa/searchlib/features/random_normal_stable_feature.cpp
index bde0cedaed0..f3d33c7dc29 100644
--- a/searchlib/src/vespa/searchlib/features/random_normal_stable_feature.cpp
+++ b/searchlib/src/vespa/searchlib/features/random_normal_stable_feature.cpp
@@ -3,6 +3,7 @@
#include "random_normal_stable_feature.h"
#include "utils.h"
#include <vespa/searchlib/fef/properties.h>
+#include <vespa/vespalib/util/stash.h>
#include <vespa/log/log.h>
LOG_SETUP(".features.randomnormalstablefeature");
@@ -41,7 +42,7 @@ RandomNormalStableBlueprint::visitDumpFeatures(const search::fef::IIndexEnvironm
search::fef::Blueprint::UP
RandomNormalStableBlueprint::createInstance() const
{
- return search::fef::Blueprint::UP(new RandomNormalStableBlueprint());
+ return std::make_unique<RandomNormalStableBlueprint>();
}
bool
@@ -75,5 +76,4 @@ RandomNormalStableBlueprint::createExecutor(const search::fef::IQueryEnvironment
return stash.create<RandomNormalStableExecutor>(seed, _mean, _stddev);
}
-
}
diff --git a/searchlib/src/vespa/searchlib/features/randomfeature.cpp b/searchlib/src/vespa/searchlib/features/randomfeature.cpp
index 18b0cf616d4..95daebd0452 100644
--- a/searchlib/src/vespa/searchlib/features/randomfeature.cpp
+++ b/searchlib/src/vespa/searchlib/features/randomfeature.cpp
@@ -3,7 +3,9 @@
#include "randomfeature.h"
#include "utils.h"
#include <vespa/searchlib/fef/properties.h>
+#include <vespa/vespalib/util/stash.h>
#include <chrono>
+
#include <vespa/log/log.h>
LOG_SETUP(".features.randomfeature");
@@ -75,5 +77,4 @@ RandomBlueprint::createExecutor(const fef::IQueryEnvironment &env, vespalib::Sta
return stash.create<RandomExecutor>(seed, matchSeed);
}
-
}
diff --git a/searchlib/src/vespa/searchlib/features/raw_score_feature.cpp b/searchlib/src/vespa/searchlib/features/raw_score_feature.cpp
index 61355581214..ea696c75eff 100644
--- a/searchlib/src/vespa/searchlib/features/raw_score_feature.cpp
+++ b/searchlib/src/vespa/searchlib/features/raw_score_feature.cpp
@@ -2,6 +2,7 @@
#include "raw_score_feature.h"
#include "utils.h"
+#include <vespa/vespalib/util/stash.h>
using namespace search::fef;
diff --git a/searchlib/src/vespa/searchlib/features/reverseproximityfeature.cpp b/searchlib/src/vespa/searchlib/features/reverseproximityfeature.cpp
index c27936332d2..436e9bc0a0c 100644
--- a/searchlib/src/vespa/searchlib/features/reverseproximityfeature.cpp
+++ b/searchlib/src/vespa/searchlib/features/reverseproximityfeature.cpp
@@ -2,14 +2,11 @@
#include "reverseproximityfeature.h"
#include "utils.h"
-#include <vespa/searchlib/fef/featurenamebuilder.h>
#include <vespa/searchlib/fef/fieldinfo.h>
-#include <vespa/searchlib/fef/fieldtype.h>
#include <vespa/searchlib/fef/itermdata.h>
-#include <vespa/vespalib/util/stringfmt.h>
+#include <vespa/vespalib/util/stash.h>
-namespace search {
-namespace features {
+namespace search::features {
ReverseProximityConfig::ReverseProximityConfig() :
fieldId(search::fef::IllegalHandle),
@@ -126,7 +123,7 @@ ReverseProximityBlueprint::setup(const search::fef::IIndexEnvironment &env,
search::fef::Blueprint::UP
ReverseProximityBlueprint::createInstance() const
{
- return search::fef::Blueprint::UP(new ReverseProximityBlueprint());
+ return std::make_unique<ReverseProximityBlueprint>();
}
search::fef::FeatureExecutor &
@@ -135,4 +132,4 @@ ReverseProximityBlueprint::createExecutor(const search::fef::IQueryEnvironment &
return stash.create<ReverseProximityExecutor>(env, _config);
}
-}}
+}
diff --git a/searchlib/src/vespa/searchlib/features/subqueries_feature.cpp b/searchlib/src/vespa/searchlib/features/subqueries_feature.cpp
index 6c52b6edb76..aaff46af93e 100644
--- a/searchlib/src/vespa/searchlib/features/subqueries_feature.cpp
+++ b/searchlib/src/vespa/searchlib/features/subqueries_feature.cpp
@@ -2,11 +2,11 @@
#include "subqueries_feature.h"
#include "utils.h"
+#include <vespa/vespalib/util/stash.h>
using namespace search::fef;
-namespace search {
-namespace features {
+namespace search::features {
SubqueriesExecutor::SubqueriesExecutor(const IQueryEnvironment &env,
uint32_t fieldId)
@@ -59,5 +59,4 @@ SubqueriesBlueprint::createExecutor(const IQueryEnvironment &queryEnv, vespalib:
return stash.create<SubqueriesExecutor>(queryEnv, _field->id());
}
-} // namespace features
-} // namespace search
+}
diff --git a/searchlib/src/vespa/searchlib/features/term_field_md_feature.cpp b/searchlib/src/vespa/searchlib/features/term_field_md_feature.cpp
index a4ca8524140..7fd487d8bdd 100644
--- a/searchlib/src/vespa/searchlib/features/term_field_md_feature.cpp
+++ b/searchlib/src/vespa/searchlib/features/term_field_md_feature.cpp
@@ -2,10 +2,10 @@
#include "term_field_md_feature.h"
#include "utils.h"
-#include <vespa/searchlib/fef/fieldinfo.h>
#include <vespa/searchlib/fef/indexproperties.h>
#include <vespa/searchlib/fef/itablemanager.h>
#include <vespa/searchlib/fef/properties.h>
+#include <vespa/vespalib/util/stash.h>
#include <cassert>
using namespace search::fef;
@@ -83,7 +83,7 @@ TermFieldMdBlueprint::visitDumpFeatures(const IIndexEnvironment &,
Blueprint::UP
TermFieldMdBlueprint::createInstance() const
{
- return Blueprint::UP(new TermFieldMdBlueprint());
+ return std::make_unique<TermFieldMdBlueprint>();
}
bool
diff --git a/searchlib/src/vespa/searchlib/features/termdistancefeature.cpp b/searchlib/src/vespa/searchlib/features/termdistancefeature.cpp
index fb38a49d6eb..e9f48421fcf 100644
--- a/searchlib/src/vespa/searchlib/features/termdistancefeature.cpp
+++ b/searchlib/src/vespa/searchlib/features/termdistancefeature.cpp
@@ -5,11 +5,11 @@
#include "utils.h"
#include <vespa/searchlib/fef/fieldinfo.h>
#include <vespa/searchlib/fef/properties.h>
+#include <vespa/vespalib/util/stash.h>
using namespace search::fef;
-namespace search {
-namespace features {
+namespace search::features {
TermDistanceExecutor::TermDistanceExecutor(const IQueryEnvironment & env,
@@ -61,7 +61,7 @@ TermDistanceBlueprint::visitDumpFeatures(const IIndexEnvironment &,
Blueprint::UP
TermDistanceBlueprint::createInstance() const
{
- return Blueprint::UP(new TermDistanceBlueprint());
+ return std::make_unique<TermDistanceBlueprint>();
}
bool
@@ -97,6 +97,4 @@ TermDistanceBlueprint::createExecutor(const IQueryEnvironment &env, vespalib::St
}
}
-
-} // namespace features
-} // namespace search
+}
diff --git a/searchlib/src/vespa/searchlib/features/termeditdistancefeature.cpp b/searchlib/src/vespa/searchlib/features/termeditdistancefeature.cpp
index 5990d62cb25..433bf6134b8 100644
--- a/searchlib/src/vespa/searchlib/features/termeditdistancefeature.cpp
+++ b/searchlib/src/vespa/searchlib/features/termeditdistancefeature.cpp
@@ -6,6 +6,8 @@
#include <vespa/searchlib/fef/properties.h>
#include <vespa/vespalib/util/stringfmt.h>
#include <vespa/vespalib/locale/c.h>
+#include <vespa/vespalib/util/stash.h>
+
#include <vespa/log/log.h>
LOG_SETUP(".features.termeditdistance");
@@ -219,7 +221,7 @@ TermEditDistanceBlueprint::setup(const search::fef::IIndexEnvironment &env,
search::fef::Blueprint::UP
TermEditDistanceBlueprint::createInstance() const
{
- return search::fef::Blueprint::UP(new TermEditDistanceBlueprint());
+ return std::make_unique<TermEditDistanceBlueprint>();
}
search::fef::FeatureExecutor &
diff --git a/searchlib/src/vespa/searchlib/features/termfeature.cpp b/searchlib/src/vespa/searchlib/features/termfeature.cpp
index d6df25cc2b9..90540726227 100644
--- a/searchlib/src/vespa/searchlib/features/termfeature.cpp
+++ b/searchlib/src/vespa/searchlib/features/termfeature.cpp
@@ -4,15 +4,14 @@
#include "utils.h"
#include <vespa/searchlib/fef/featurenamebuilder.h>
#include <vespa/searchlib/fef/fieldinfo.h>
-#include <vespa/searchlib/fef/fieldtype.h>
#include <vespa/searchlib/fef/properties.h>
#include <vespa/searchlib/fef/itermdata.h>
#include <vespa/vespalib/util/stringfmt.h>
+#include <vespa/vespalib/util/stash.h>
using namespace search::fef;
-namespace search {
-namespace features {
+namespace search::features {
TermExecutor::TermExecutor(const search::fef::IQueryEnvironment &env,
uint32_t termId) :
@@ -21,7 +20,7 @@ TermExecutor::TermExecutor(const search::fef::IQueryEnvironment &env,
_connectedness(util::lookupConnectedness(env, termId)),
_significance(0)
{
- if (_termData != NULL) {
+ if (_termData != nullptr) {
feature_t fallback = util::getSignificance(*_termData);
_significance = util::lookupSignificance(env, termId, fallback);
}
@@ -30,7 +29,7 @@ TermExecutor::TermExecutor(const search::fef::IQueryEnvironment &env,
void
TermExecutor::execute(uint32_t)
{
- if (_termData == NULL) { // this query term is not present in the query
+ if (_termData == nullptr) { // this query term is not present in the query
outputs().set_number(0, 0.0f); // connectedness
outputs().set_number(1, 0.0f); // significance (1 - frequency)
outputs().set_number(2, 0.0f); // weight
@@ -76,7 +75,7 @@ TermBlueprint::setup(const search::fef::IIndexEnvironment &,
search::fef::Blueprint::UP
TermBlueprint::createInstance() const
{
- return search::fef::Blueprint::UP(new TermBlueprint());
+ return std::make_unique<TermBlueprint>();
}
search::fef::FeatureExecutor &
@@ -85,4 +84,4 @@ TermBlueprint::createExecutor(const search::fef::IQueryEnvironment &env, vespali
return stash.create<TermExecutor>(env, _termId);
}
-}}
+}
diff --git a/searchlib/src/vespa/searchlib/features/terminfofeature.cpp b/searchlib/src/vespa/searchlib/features/terminfofeature.cpp
index 4f32cda8c86..ca7a5d8e248 100644
--- a/searchlib/src/vespa/searchlib/features/terminfofeature.cpp
+++ b/searchlib/src/vespa/searchlib/features/terminfofeature.cpp
@@ -3,30 +3,28 @@
#include "terminfofeature.h"
#include "valuefeature.h"
#include <vespa/searchlib/fef/properties.h>
-#include <vespa/searchlib/fef/fieldinfo.h>
-#include <vespa/searchlib/fef/fieldtype.h>
#include <vespa/searchlib/fef/featurenamebuilder.h>
#include <vespa/searchlib/fef/itermdata.h>
-#include <vespa/searchlib/fef/handle.h>
+#include <vespa/vespalib/util/stash.h>
-namespace search {
-namespace features {
+using namespace search::fef;
+namespace search::features {
TermInfoBlueprint::TermInfoBlueprint()
- : search::fef::Blueprint("termInfo"),
+ : Blueprint("termInfo"),
_termIdx(0)
{
}
void
-TermInfoBlueprint::visitDumpFeatures(const search::fef::IIndexEnvironment &,
- search::fef::IDumpFeatureVisitor &) const
+TermInfoBlueprint::visitDumpFeatures(const IIndexEnvironment &,
+ IDumpFeatureVisitor &) const
{
}
bool
-TermInfoBlueprint::setup(const search::fef::IIndexEnvironment &,
- const search::fef::ParameterList & params)
+TermInfoBlueprint::setup(const IIndexEnvironment &,
+ const ParameterList & params)
{
_termIdx = params[0].asInteger();
describeOutput("queryidx", "The index of the first term with the given "
@@ -34,8 +32,8 @@ TermInfoBlueprint::setup(const search::fef::IIndexEnvironment &,
return true;
}
-search::fef::FeatureExecutor &
-TermInfoBlueprint::createExecutor(const search::fef::IQueryEnvironment &queryEnv, vespalib::Stash &stash) const
+FeatureExecutor &
+TermInfoBlueprint::createExecutor(const IQueryEnvironment &queryEnv, vespalib::Stash &stash) const
{
feature_t queryIdx = -1.0;
if (queryEnv.getNumTerms() > _termIdx) {
@@ -46,5 +44,4 @@ TermInfoBlueprint::createExecutor(const search::fef::IQueryEnvironment &queryEnv
return stash.create<ValueExecutor>(values);
}
-} // namespace features
-} // namespace search
+}
diff --git a/searchlib/src/vespa/searchlib/features/text_similarity_feature.cpp b/searchlib/src/vespa/searchlib/features/text_similarity_feature.cpp
index 3cdb20a16e5..a0e0a8759a0 100644
--- a/searchlib/src/vespa/searchlib/features/text_similarity_feature.cpp
+++ b/searchlib/src/vespa/searchlib/features/text_similarity_feature.cpp
@@ -3,6 +3,7 @@
#include "text_similarity_feature.h"
#include <vespa/searchlib/fef/itermdata.h>
#include <vespa/searchlib/fef/featurenamebuilder.h>
+#include <vespa/vespalib/util/stash.h>
namespace search::features {
@@ -194,7 +195,7 @@ TextSimilarityBlueprint::visitDumpFeatures(const fef::IIndexEnvironment &env,
fef::Blueprint::UP
TextSimilarityBlueprint::createInstance() const
{
- return Blueprint::UP(new TextSimilarityBlueprint());
+ return std::make_unique<TextSimilarityBlueprint>();
}
bool
@@ -218,6 +219,4 @@ TextSimilarityBlueprint::createExecutor(const fef::IQueryEnvironment &env, vespa
return stash.create<TextSimilarityExecutor>(env, _field_id);
}
-//-----------------------------------------------------------------------------
-
}
diff --git a/searchlib/src/vespa/searchlib/features/valuefeature.cpp b/searchlib/src/vespa/searchlib/features/valuefeature.cpp
index 339cc5431f1..2b91cf1688b 100644
--- a/searchlib/src/vespa/searchlib/features/valuefeature.cpp
+++ b/searchlib/src/vespa/searchlib/features/valuefeature.cpp
@@ -2,12 +2,13 @@
#include "valuefeature.h"
#include <vespa/vespalib/stllike/asciistream.h>
+#include <vespa/vespalib/util/stash.h>
-namespace search {
-namespace features {
+using namespace search::fef;
+namespace search::features {
ValueExecutor::ValueExecutor(const std::vector<feature_t> & values) :
- search::fef::FeatureExecutor(),
+ FeatureExecutor(),
_values(values)
{
}
@@ -27,22 +28,20 @@ SingleZeroValueExecutor::execute(uint32_t)
}
ValueBlueprint::ValueBlueprint() :
- search::fef::Blueprint("value"),
+ Blueprint("value"),
_values()
{
}
-ValueBlueprint::~ValueBlueprint() {}
+ValueBlueprint::~ValueBlueprint() = default;
void
-ValueBlueprint::visitDumpFeatures(const search::fef::IIndexEnvironment &,
- search::fef::IDumpFeatureVisitor &) const
+ValueBlueprint::visitDumpFeatures(const IIndexEnvironment &, IDumpFeatureVisitor &) const
{
}
bool
-ValueBlueprint::setup(const search::fef::IIndexEnvironment &,
- const search::fef::ParameterList & params)
+ValueBlueprint::setup(const IIndexEnvironment &, const ParameterList & params)
{
for (uint32_t i = 0; i < params.size(); ++i) {
_values.push_back(params[i].asDouble());
@@ -56,13 +55,12 @@ ValueBlueprint::setup(const search::fef::IIndexEnvironment &,
return true;
}
-search::fef::FeatureExecutor &
-ValueBlueprint::createExecutor(const search::fef::IQueryEnvironment &queryEnv, vespalib::Stash &stash) const
+FeatureExecutor &
+ValueBlueprint::createExecutor(const IQueryEnvironment &queryEnv, vespalib::Stash &stash) const
{
(void) queryEnv;
return stash.create<ValueExecutor>(_values);
}
-} // namespace features
-} // namespace search
+}
diff --git a/searchlib/src/vespa/searchlib/fef/blueprintresolver.cpp b/searchlib/src/vespa/searchlib/fef/blueprintresolver.cpp
index cd2cd949a91..82a69009e82 100644
--- a/searchlib/src/vespa/searchlib/fef/blueprintresolver.cpp
+++ b/searchlib/src/vespa/searchlib/fef/blueprintresolver.cpp
@@ -85,9 +85,21 @@ struct Compiler : public Blueprint::DependencyHandler {
}
compile_error = true;
}
+ fixup_feature_map();
return FeatureRef();
}
+ void fixup_feature_map() {
+ auto itr = feature_map.begin();
+ while (itr != feature_map.end()) {
+ if (itr->second.executor >= spec_list.size()) {
+ itr = feature_map.erase(itr);
+ } else {
+ ++itr;
+ }
+ }
+ }
+
FeatureRef verify_type(const FeatureNameParser &parser, FeatureRef ref, Accept accept_type) {
const auto &spec = spec_list[ref.executor];
bool is_object = spec.output_types[ref.output];
diff --git a/searchlib/src/vespa/searchlib/fef/rank_program.cpp b/searchlib/src/vespa/searchlib/fef/rank_program.cpp
index d39be693806..bd8abe41afb 100644
--- a/searchlib/src/vespa/searchlib/fef/rank_program.cpp
+++ b/searchlib/src/vespa/searchlib/fef/rank_program.cpp
@@ -6,6 +6,9 @@
#include <algorithm>
#include <cassert>
+#include <vespa/log/log.h>
+LOG_SETUP(".fef.rankprogram");
+
using vespalib::Stash;
namespace search::fef {
@@ -42,7 +45,7 @@ struct OverrideVisitor : public IPropertiesVisitor
{
auto pos = feature_map.find(key);
if (pos != feature_map.end()) {
- overrides.push_back(Override(pos->second, vespalib::locale::c::strtod(values.get().c_str(), nullptr)));
+ overrides.emplace_back(pos->second, vespalib::locale::c::strtod(values.get().c_str(), nullptr));
}
}
};
@@ -175,6 +178,7 @@ RankProgram::setup(const MatchData &md,
auto override_end = overrides.end();
const auto &specs = _resolver->getExecutorSpecs();
+ _executors.reserve(specs.size());
for (uint32_t i = 0; i < specs.size(); ++i) {
vespalib::ArrayRef<NumberOrObject> outputs = _hot_stash.create_array<NumberOrObject>(specs[i].output_types.size());
StashSelector stash(_hot_stash, _cold_stash);
@@ -216,6 +220,8 @@ RankProgram::setup(const MatchData &md,
}
}
assert(_executors.size() == specs.size());
+ LOG(debug, "Num executors = %ld, hot stash = %ld, cold stash = %ld, match data fields = %d",
+ _executors.size(), _hot_stash.count_used(), _cold_stash.count_used(), md.getNumTermFields());
}
FeatureResolver
diff --git a/searchlib/src/vespa/searchlib/fef/rank_program.h b/searchlib/src/vespa/searchlib/fef/rank_program.h
index 0e5c390162a..aa6f77abce4 100644
--- a/searchlib/src/vespa/searchlib/fef/rank_program.h
+++ b/searchlib/src/vespa/searchlib/fef/rank_program.h
@@ -9,8 +9,8 @@
#include "feature_resolver.h"
#include <vespa/vespalib/stllike/string.h>
#include <vespa/vespalib/util/array.h>
+#include <vespa/vespalib/util/stash.h>
#include <set>
-#include <vector>
namespace search::fef {
diff --git a/searchlib/src/vespa/searchlib/fef/ranksetup.cpp b/searchlib/src/vespa/searchlib/fef/ranksetup.cpp
index ee88be8ad00..88f4a07d95d 100644
--- a/searchlib/src/vespa/searchlib/fef/ranksetup.cpp
+++ b/searchlib/src/vespa/searchlib/fef/ranksetup.cpp
@@ -12,7 +12,7 @@ class VisitorAdapter : public search::fef::IDumpFeatureVisitor
{
search::fef::BlueprintResolver &_resolver;
public:
- VisitorAdapter(search::fef::BlueprintResolver &resolver)
+ explicit VisitorAdapter(search::fef::BlueprintResolver &resolver)
: _resolver(resolver) {}
void visitDumpFeature(const vespalib::string &name) override {
_resolver.addSeed(name);
@@ -60,7 +60,8 @@ RankSetup::RankSetup(const BlueprintFactory &factory, const IIndexEnvironment &i
_diversityCutoffFactor(10.0),
_diversityCutoffStrategy("loose"),
_softTimeoutEnabled(false),
- _softTimeoutTailCost(0.1)
+ _softTimeoutTailCost(0.1),
+ _softTimeoutFactor(0.5)
{ }
RankSetup::~RankSetup() = default;
@@ -71,13 +72,13 @@ RankSetup::configure()
setFirstPhaseRank(rank::FirstPhase::lookup(_indexEnv.getProperties()));
setSecondPhaseRank(rank::SecondPhase::lookup(_indexEnv.getProperties()));
std::vector<vespalib::string> summaryFeatures = summary::Feature::lookup(_indexEnv.getProperties());
- for (uint32_t i = 0; i < summaryFeatures.size(); ++i) {
- addSummaryFeature(summaryFeatures[i]);
+ for (const auto & feature : summaryFeatures) {
+ addSummaryFeature(feature);
}
setIgnoreDefaultRankFeatures(dump::IgnoreDefaultFeatures::check(_indexEnv.getProperties()));
std::vector<vespalib::string> dumpFeatures = dump::Feature::lookup(_indexEnv.getProperties());
- for (uint32_t i = 0; i < dumpFeatures.size(); ++i) {
- addDumpFeature(dumpFeatures[i]);
+ for (const auto & feature : dumpFeatures) {
+ addDumpFeature(feature);
}
split_unpacking_iterators(matching::SplitUnpackingIterators::check(_indexEnv.getProperties()));
delay_unpacking_iterators(matching::DelayUnpackingIterators::check(_indexEnv.getProperties()));
@@ -159,15 +160,15 @@ RankSetup::compile()
_compileError = true;
}
}
- for (uint32_t i = 0; i < _summaryFeatures.size(); ++i) {
- _summary_resolver->addSeed(_summaryFeatures[i]);
+ for (const auto & feature :_summaryFeatures) {
+ _summary_resolver->addSeed(feature);
}
if (!_ignoreDefaultRankFeatures) {
VisitorAdapter adapter(*_dumpResolver);
_factory.visitDumpFeatures(_indexEnv, adapter);
}
- for (uint32_t i = 0; i < _dumpFeatures.size(); ++i) {
- _dumpResolver->addSeed(_dumpFeatures[i]);
+ for (const auto & feature : _dumpFeatures) {
+ _dumpResolver->addSeed(feature);
}
_indexEnv.hintFeatureMotivation(IIndexEnvironment::RANK);
_compileError |= !_first_phase_resolver->compile();
diff --git a/searchlib/src/vespa/searchlib/fef/ranksetup.h b/searchlib/src/vespa/searchlib/fef/ranksetup.h
index d543794b347..e1cd78d41a9 100644
--- a/searchlib/src/vespa/searchlib/fef/ranksetup.h
+++ b/searchlib/src/vespa/searchlib/fef/ranksetup.h
@@ -397,10 +397,10 @@ public:
// them to be ready to use. Also keep in mind that creating a rank
// program is cheap while setting it up is more expensive.
- RankProgram::UP create_first_phase_program() const { return RankProgram::UP(new RankProgram(_first_phase_resolver)); }
- RankProgram::UP create_second_phase_program() const { return RankProgram::UP(new RankProgram(_second_phase_resolver)); }
- RankProgram::UP create_summary_program() const { return RankProgram::UP(new RankProgram(_summary_resolver)); }
- RankProgram::UP create_dump_program() const { return RankProgram::UP(new RankProgram(_dumpResolver)); }
+ RankProgram::UP create_first_phase_program() const { return std::make_unique<RankProgram>(_first_phase_resolver); }
+ RankProgram::UP create_second_phase_program() const { return std::make_unique<RankProgram>(_second_phase_resolver); }
+ RankProgram::UP create_summary_program() const { return std::make_unique<RankProgram>(_summary_resolver); }
+ RankProgram::UP create_dump_program() const { return std::make_unique<RankProgram>(_dumpResolver); }
/**
* Here you can do some preprocessing. State must be stored in the IObjectStore.
diff --git a/searchlib/src/vespa/searchlib/fef/test/plugin/cfgvalue.cpp b/searchlib/src/vespa/searchlib/fef/test/plugin/cfgvalue.cpp
index 31e99ef9953..8be0961f999 100644
--- a/searchlib/src/vespa/searchlib/fef/test/plugin/cfgvalue.cpp
+++ b/searchlib/src/vespa/searchlib/fef/test/plugin/cfgvalue.cpp
@@ -2,11 +2,10 @@
#include "cfgvalue.h"
#include <vespa/searchlib/fef/properties.h>
+#include <vespa/vespalib/util/stash.h>
#include <sstream>
-namespace search {
-namespace fef {
-namespace test {
+namespace search::fef::test {
CfgValueBlueprint::CfgValueBlueprint() :
Blueprint("test_cfgvalue"),
@@ -14,9 +13,7 @@ CfgValueBlueprint::CfgValueBlueprint() :
{
}
-CfgValueBlueprint::~CfgValueBlueprint()
-{
-}
+CfgValueBlueprint::~CfgValueBlueprint() = default;
void
CfgValueBlueprint::visitDumpFeatures(const IIndexEnvironment &indexEnv, IDumpFeatureVisitor &visitor) const
@@ -59,6 +56,4 @@ CfgValueBlueprint::createExecutor(const IQueryEnvironment & queryEnv, vespalib::
return stash.create<search::features::ValueExecutor>(_values);
}
-} // namespace test
-} // namespace fef
-} // namespace search
+}
diff --git a/searchlib/src/vespa/searchlib/fef/test/plugin/chain.cpp b/searchlib/src/vespa/searchlib/fef/test/plugin/chain.cpp
index 86754c2c22d..017b916ad76 100644
--- a/searchlib/src/vespa/searchlib/fef/test/plugin/chain.cpp
+++ b/searchlib/src/vespa/searchlib/fef/test/plugin/chain.cpp
@@ -1,11 +1,10 @@
// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
#include "chain.h"
+#include <vespa/vespalib/util/stash.h>
#include <sstream>
-namespace search {
-namespace fef {
-namespace test {
+namespace search::fef::test {
ChainExecutor::ChainExecutor() :
FeatureExecutor()
@@ -67,6 +66,4 @@ ChainBlueprint::createExecutor(const IQueryEnvironment &queryEnv, vespalib::Stas
return stash.create<ChainExecutor>();
}
-} // namespace test
-} // namespace fef
-} // namespace search
+}
diff --git a/searchlib/src/vespa/searchlib/fef/test/plugin/double.cpp b/searchlib/src/vespa/searchlib/fef/test/plugin/double.cpp
index d9ec8b13e57..6f8ebd57fb0 100644
--- a/searchlib/src/vespa/searchlib/fef/test/plugin/double.cpp
+++ b/searchlib/src/vespa/searchlib/fef/test/plugin/double.cpp
@@ -3,6 +3,7 @@
#include "double.h"
#include <vespa/searchlib/fef/featurenamebuilder.h>
#include <vespa/vespalib/stllike/asciistream.h>
+#include <vespa/vespalib/util/stash.h>
#include <cassert>
namespace search::fef::test {
diff --git a/searchlib/src/vespa/searchlib/fef/test/plugin/query.cpp b/searchlib/src/vespa/searchlib/fef/test/plugin/query.cpp
index 4b4b10c4d25..1e57d6252ee 100644
--- a/searchlib/src/vespa/searchlib/fef/test/plugin/query.cpp
+++ b/searchlib/src/vespa/searchlib/fef/test/plugin/query.cpp
@@ -4,6 +4,7 @@
#include <vespa/searchlib/features/valuefeature.h>
#include <vespa/searchlib/fef/properties.h>
#include <vespa/vespalib/locale/c.h>
+#include <vespa/vespalib/util/stash.h>
#include <sstream>
namespace search::fef::test {
@@ -32,7 +33,7 @@ QueryBlueprint::createExecutor(const IQueryEnvironment &queryEnv, vespalib::Stas
{
std::vector<feature_t> values;
std::string val = queryEnv.getProperties().lookup(_key).get("0.0");
- values.push_back(vespalib::locale::c::strtod(val.data(), NULL));
+ values.push_back(vespalib::locale::c::strtod(val.data(), nullptr));
return stash.create<search::features::ValueExecutor>(values);
}
diff --git a/searchlib/src/vespa/searchlib/fef/test/plugin/staticrank.cpp b/searchlib/src/vespa/searchlib/fef/test/plugin/staticrank.cpp
index c1b8b940245..c5871b23e77 100644
--- a/searchlib/src/vespa/searchlib/fef/test/plugin/staticrank.cpp
+++ b/searchlib/src/vespa/searchlib/fef/test/plugin/staticrank.cpp
@@ -2,10 +2,9 @@
#include "staticrank.h"
#include <vespa/searchcommon/attribute/attributecontent.h>
+#include <vespa/vespalib/util/stash.h>
-namespace search {
-namespace fef {
-namespace test {
+namespace search::fef::test {
StaticRankExecutor::StaticRankExecutor(const search::attribute::IAttributeVector * attribute) :
FeatureExecutor(),
@@ -17,7 +16,7 @@ void
StaticRankExecutor::execute(uint32_t docId)
{
search::attribute::FloatContent staticRank;
- if (_attribute != NULL) {
+ if (_attribute != nullptr) {
staticRank.allocate(_attribute->getMaxValueCount());
staticRank.fill(*_attribute, docId);
}
@@ -31,9 +30,7 @@ StaticRankBlueprint::StaticRankBlueprint() :
{
}
-StaticRankBlueprint::~StaticRankBlueprint()
-{
-}
+StaticRankBlueprint::~StaticRankBlueprint() = default;
bool
StaticRankBlueprint::setup(const IIndexEnvironment & indexEnv, const StringVector & params)
@@ -54,6 +51,4 @@ StaticRankBlueprint::createExecutor(const IQueryEnvironment & queryEnv, vespalib
return stash.create<StaticRankExecutor>(av);
}
-} // namespace test
-} // namespace fef
-} // namespace search
+}
diff --git a/searchlib/src/vespa/searchlib/fef/test/plugin/sum.cpp b/searchlib/src/vespa/searchlib/fef/test/plugin/sum.cpp
index b5025d53cbd..4362a7f0860 100644
--- a/searchlib/src/vespa/searchlib/fef/test/plugin/sum.cpp
+++ b/searchlib/src/vespa/searchlib/fef/test/plugin/sum.cpp
@@ -2,10 +2,9 @@
#include "sum.h"
#include <vespa/searchlib/fef/featurenamebuilder.h>
+#include <vespa/vespalib/util/stash.h>
-namespace search {
-namespace fef {
-namespace test {
+namespace search::fef::test {
void
SumExecutor::execute(uint32_t)
@@ -73,6 +72,4 @@ SumBlueprint::createExecutor(const IQueryEnvironment &queryEnv, vespalib::Stash
return stash.create<SumExecutor>();
}
-} // namespace test
-} // namespace fef
-} // namespace search
+}
diff --git a/searchlib/src/vespa/searchlib/fef/test/plugin/unbox.cpp b/searchlib/src/vespa/searchlib/fef/test/plugin/unbox.cpp
index 7b3876fada0..e30b4893e15 100644
--- a/searchlib/src/vespa/searchlib/fef/test/plugin/unbox.cpp
+++ b/searchlib/src/vespa/searchlib/fef/test/plugin/unbox.cpp
@@ -1,6 +1,7 @@
// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
#include "unbox.h"
+#include <vespa/vespalib/util/stash.h>
namespace search::fef::test {
diff --git a/searchlib/src/vespa/searchlib/fef/test/test_features.cpp b/searchlib/src/vespa/searchlib/fef/test/test_features.cpp
index f924acd65de..c32776dc88d 100644
--- a/searchlib/src/vespa/searchlib/fef/test/test_features.cpp
+++ b/searchlib/src/vespa/searchlib/fef/test/test_features.cpp
@@ -3,6 +3,8 @@
#include <vespa/vespalib/testkit/test_kit.h>
#include "test_features.h"
#include <vespa/vespalib/locale/c.h>
+#include <vespa/vespalib/util/stash.h>
+
using vespalib::eval::DoubleValue;
using vespalib::eval::ValueType;
diff --git a/searchlib/src/vespa/searchlib/queryeval/CMakeLists.txt b/searchlib/src/vespa/searchlib/queryeval/CMakeLists.txt
index de2919443ff..0dcb0393473 100644
--- a/searchlib/src/vespa/searchlib/queryeval/CMakeLists.txt
+++ b/searchlib/src/vespa/searchlib/queryeval/CMakeLists.txt
@@ -32,6 +32,7 @@ vespa_add_library(searchlib_queryeval OBJECT
nearest_neighbor_blueprint.cpp
nearest_neighbor_iterator.cpp
nearsearch.cpp
+ nns_index_iterator.cpp
orsearch.cpp
predicate_blueprint.cpp
predicate_search.cpp
diff --git a/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_blueprint.cpp b/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_blueprint.cpp
index 8be6263221a..d4aa2aaa1d7 100644
--- a/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_blueprint.cpp
+++ b/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_blueprint.cpp
@@ -3,6 +3,7 @@
#include "emptysearch.h"
#include "nearest_neighbor_blueprint.h"
#include "nearest_neighbor_iterator.h"
+#include "nns_index_iterator.h"
#include <vespa/searchlib/fef/termfieldmatchdataarray.h>
#include <vespa/eval/tensor/dense/dense_tensor_view.h>
#include <vespa/searchlib/tensor/dense_tensor_attribute.h>
@@ -17,20 +18,51 @@ NearestNeighborBlueprint::NearestNeighborBlueprint(const queryeval::FieldSpec& f
_attr_tensor(attr_tensor),
_query_tensor(std::move(query_tensor)),
_target_num_hits(target_num_hits),
- _distance_heap(target_num_hits)
+ _distance_heap(target_num_hits),
+ _found_hits()
{
- setEstimate(HitEstimate(_attr_tensor.getNumDocs(), false));
+ uint32_t est_hits = _attr_tensor.getNumDocs();
+ if (_attr_tensor.nearest_neighbor_index()) {
+ est_hits = std::min(target_num_hits, est_hits);
+ }
+ setEstimate(HitEstimate(est_hits, false));
}
NearestNeighborBlueprint::~NearestNeighborBlueprint() = default;
+void
+NearestNeighborBlueprint::perform_top_k()
+{
+ auto nns_index = _attr_tensor.nearest_neighbor_index();
+ if (nns_index) {
+ auto lhs_type = _query_tensor->fast_type();
+ auto rhs_type = _attr_tensor.getTensorType();
+ // XXX deal with different cell types later
+ if (lhs_type == rhs_type) {
+ auto lhs = _query_tensor->cellsRef();
+ uint32_t k = _target_num_hits;
+ uint32_t explore_k = k + 100; // XXX hardcoded for now
+ _found_hits = nns_index->find_top_k(k, lhs, explore_k);
+ }
+ }
+}
+
+void
+NearestNeighborBlueprint::fetchPostings(const ExecuteInfo &execInfo) {
+ if (execInfo.isStrict()) {
+ perform_top_k();
+ }
+}
+
std::unique_ptr<SearchIterator>
NearestNeighborBlueprint::createLeafSearch(const search::fef::TermFieldMatchDataArray& tfmda, bool strict) const
{
assert(tfmda.size() == 1);
fef::TermFieldMatchData &tfmd = *tfmda[0]; // always search in only one field
+ if (strict && ! _found_hits.empty()) {
+ return NnsIndexIterator::create(tfmd, _found_hits);
+ }
const vespalib::tensor::DenseTensorView &qT = *_query_tensor;
-
return NearestNeighborIterator::create(strict, tfmd, qT, _attr_tensor, _distance_heap);
}
diff --git a/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_blueprint.h b/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_blueprint.h
index 019f8e31842..ab4413c487a 100644
--- a/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_blueprint.h
+++ b/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_blueprint.h
@@ -3,6 +3,7 @@
#include "blueprint.h"
#include "nearest_neighbor_distance_heap.h"
+#include <vespa/searchlib/tensor/nearest_neighbor_index.h>
namespace vespalib::tensor { class DenseTensorView; }
namespace search::tensor { class DenseTensorAttribute; }
@@ -21,7 +22,9 @@ private:
std::unique_ptr<vespalib::tensor::DenseTensorView> _query_tensor;
uint32_t _target_num_hits;
mutable NearestNeighborDistanceHeap _distance_heap;
+ std::vector<search::tensor::NearestNeighborIndex::Neighbor> _found_hits;
+ void perform_top_k();
public:
NearestNeighborBlueprint(const queryeval::FieldSpec& field,
const tensor::DenseTensorAttribute& attr_tensor,
@@ -38,6 +41,7 @@ public:
bool strict) const override;
void visitMembers(vespalib::ObjectVisitor& visitor) const override;
bool always_needs_unpack() const override;
+ void fetchPostings(const ExecuteInfo &execInfo) override;
};
}
diff --git a/searchlib/src/vespa/searchlib/queryeval/nns_index_iterator.cpp b/searchlib/src/vespa/searchlib/queryeval/nns_index_iterator.cpp
new file mode 100644
index 00000000000..18e0213e092
--- /dev/null
+++ b/searchlib/src/vespa/searchlib/queryeval/nns_index_iterator.cpp
@@ -0,0 +1,68 @@
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include "nns_index_iterator.h"
+#include <vespa/searchlib/tensor/nearest_neighbor_index.h>
+#include <cmath>
+
+using Neighbor = search::tensor::NearestNeighborIndex::Neighbor;
+
+namespace search::queryeval {
+
+/**
+ * Search iterator for K nearest neighbor matching,
+ * where the actual search is done up front and this class
+ * just iterates over a vector held by the blueprint.
+ **/
+class NeighborVectorIterator : public NnsIndexIterator
+{
+private:
+ fef::TermFieldMatchData &_tfmd;
+ const std::vector<Neighbor> &_hits;
+ uint32_t _idx;
+ double _last_sq_dist;
+public:
+ NeighborVectorIterator(fef::TermFieldMatchData &tfmd,
+ const std::vector<Neighbor> &hits)
+ : _tfmd(tfmd),
+ _hits(hits),
+ _idx(0),
+ _last_sq_dist(0.0)
+ {}
+
+ void initRange(uint32_t begin_id, uint32_t end_id) override {
+ SearchIterator::initRange(begin_id, end_id);
+ _idx = 0;
+ }
+
+ void doSeek(uint32_t docId) override {
+ while (_idx < _hits.size()) {
+ uint32_t hit_id = _hits[_idx].docid;
+ if (hit_id < docId) {
+ ++_idx;
+ } else if (hit_id < getEndId()) {
+ setDocId(hit_id);
+ _last_sq_dist = _hits[_idx].distance;
+ return;
+ } else {
+ _idx = _hits.size();
+ }
+ }
+ setAtEnd();
+ }
+
+ void doUnpack(uint32_t docId) override {
+ _tfmd.setRawScore(docId, sqrt(_last_sq_dist));
+ }
+
+ Trinary is_strict() const override { return Trinary::True; }
+};
+
+std::unique_ptr<NnsIndexIterator>
+NnsIndexIterator::create(
+ fef::TermFieldMatchData &tfmd,
+ const std::vector<Neighbor> &hits)
+{
+ return std::make_unique<NeighborVectorIterator>(tfmd, hits);
+}
+
+} // namespace
diff --git a/searchlib/src/vespa/searchlib/queryeval/nns_index_iterator.h b/searchlib/src/vespa/searchlib/queryeval/nns_index_iterator.h
new file mode 100644
index 00000000000..9ffd0df94eb
--- /dev/null
+++ b/searchlib/src/vespa/searchlib/queryeval/nns_index_iterator.h
@@ -0,0 +1,20 @@
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include "searchiterator.h"
+#include <vespa/searchlib/fef/termfieldmatchdata.h>
+#include <vespa/searchlib/tensor/nearest_neighbor_index.h>
+
+namespace search::queryeval {
+
+class NnsIndexIterator : public SearchIterator
+{
+public:
+ using Hit = search::tensor::NearestNeighborIndex::Neighbor;
+ static std::unique_ptr<NnsIndexIterator> create(
+ fef::TermFieldMatchData &tfmd,
+ const std::vector<Hit> &hits);
+};
+
+} // namespace
diff --git a/searchlib/src/vespa/searchlib/tensor/CMakeLists.txt b/searchlib/src/vespa/searchlib/tensor/CMakeLists.txt
index 9175168248c..0bdcd53af77 100644
--- a/searchlib/src/vespa/searchlib/tensor/CMakeLists.txt
+++ b/searchlib/src/vespa/searchlib/tensor/CMakeLists.txt
@@ -1,16 +1,18 @@
# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
vespa_add_library(searchlib_tensor OBJECT
SOURCES
+ default_nearest_neighbor_index_factory.cpp
dense_tensor_attribute.cpp
dense_tensor_attribute_saver.cpp
dense_tensor_store.cpp
generic_tensor_attribute.cpp
+ generic_tensor_attribute_saver.cpp
generic_tensor_store.cpp
hnsw_index.cpp
imported_tensor_attribute_vector.cpp
imported_tensor_attribute_vector_read_guard.cpp
+ nearest_neighbor_index.cpp
tensor_attribute.cpp
- generic_tensor_attribute_saver.cpp
tensor_store.cpp
DEPENDS
)
diff --git a/searchlib/src/vespa/searchlib/tensor/default_nearest_neighbor_index_factory.cpp b/searchlib/src/vespa/searchlib/tensor/default_nearest_neighbor_index_factory.cpp
new file mode 100644
index 00000000000..68efe6417c0
--- /dev/null
+++ b/searchlib/src/vespa/searchlib/tensor/default_nearest_neighbor_index_factory.cpp
@@ -0,0 +1,51 @@
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include "default_nearest_neighbor_index_factory.h"
+#include "distance_functions.h"
+#include "hnsw_index.h"
+#include "random_level_generator.h"
+#include <vespa/searchcommon/attribute/config.h>
+
+namespace search::tensor {
+
+using vespalib::eval::ValueType;
+
+namespace {
+
+class LevelZeroGenerator : public RandomLevelGenerator {
+ uint32_t max_level() override { return 0; }
+};
+
+DistanceFunction::UP
+make_distance_function(ValueType::CellType cell_type)
+{
+ if (cell_type == ValueType::CellType::FLOAT) {
+ return std::make_unique<SquaredEuclideanDistance<float>>();
+ } else {
+ return std::make_unique<SquaredEuclideanDistance<double>>();
+ }
+}
+
+RandomLevelGenerator::UP
+make_random_level_generator()
+{
+ // TODO: Make generator that results in hierarchical graph.
+ return std::make_unique<LevelZeroGenerator>();
+}
+
+}
+
+std::unique_ptr<NearestNeighborIndex>
+DefaultNearestNeighborIndexFactory::make(const DocVectorAccess& vectors,
+ vespalib::eval::ValueType::CellType cell_type,
+ const search::attribute::HnswIndexParams& params) const
+{
+ HnswIndex::Config cfg(params.max_links_per_node() * 2,
+ params.max_links_per_node(),
+ params.neighbors_to_explore_at_insert(),
+ true);
+ return std::make_unique<HnswIndex>(vectors, make_distance_function(cell_type), make_random_level_generator(), cfg);
+}
+
+}
+
diff --git a/searchlib/src/vespa/searchlib/tensor/default_nearest_neighbor_index_factory.h b/searchlib/src/vespa/searchlib/tensor/default_nearest_neighbor_index_factory.h
new file mode 100644
index 00000000000..ea784efdb51
--- /dev/null
+++ b/searchlib/src/vespa/searchlib/tensor/default_nearest_neighbor_index_factory.h
@@ -0,0 +1,19 @@
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include "nearest_neighbor_index_factory.h"
+
+namespace search::tensor {
+
+/**
+ * Factory that instantiates the production hnsw index.
+ */
+class DefaultNearestNeighborIndexFactory : public NearestNeighborIndexFactory {
+public:
+ std::unique_ptr<NearestNeighborIndex> make(const DocVectorAccess& vectors,
+ vespalib::eval::ValueType::CellType cell_type,
+ const search::attribute::HnswIndexParams& params) const override;
+};
+
+}
diff --git a/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute.cpp b/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute.cpp
index a2b9f136ed9..171340e07f1 100644
--- a/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute.cpp
+++ b/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute.cpp
@@ -2,6 +2,7 @@
#include "dense_tensor_attribute.h"
#include "dense_tensor_attribute_saver.h"
+#include "nearest_neighbor_index.h"
#include "tensor_attribute.hpp"
#include <vespa/eval/tensor/tensor.h>
#include <vespa/eval/tensor/dense/mutable_dense_tensor_view.h>
@@ -55,11 +56,23 @@ TensorReader::is_present() {
}
-DenseTensorAttribute::DenseTensorAttribute(vespalib::stringref baseFileName,
- const Config &cfg)
+void
+DenseTensorAttribute::consider_remove_from_index(DocId docid)
+{
+ if (_index && _refVector[docid].valid()) {
+ _index->remove_document(docid);
+ }
+}
+
+DenseTensorAttribute::DenseTensorAttribute(vespalib::stringref baseFileName, const Config& cfg,
+ const NearestNeighborIndexFactory& index_factory)
: TensorAttribute(baseFileName, cfg, _denseTensorStore),
- _denseTensorStore(cfg.tensorType())
+ _denseTensorStore(cfg.tensorType()),
+ _index()
{
+ if (cfg.hnsw_index_params().has_value()) {
+ _index = index_factory.make(*this, cfg.tensorType().cell_type(), cfg.hnsw_index_params().value());
+ }
}
@@ -69,12 +82,23 @@ DenseTensorAttribute::~DenseTensorAttribute()
_tensorStore.clearHoldLists();
}
+uint32_t
+DenseTensorAttribute::clearDoc(DocId docId)
+{
+ consider_remove_from_index(docId);
+ return TensorAttribute::clearDoc(docId);
+}
+
void
DenseTensorAttribute::setTensor(DocId docId, const Tensor &tensor)
{
checkTensorType(tensor);
+ consider_remove_from_index(docId);
EntryRef ref = _denseTensorStore.setTensor(tensor);
setTensorRef(docId, ref);
+ if (_index) {
+ _index->add_document(docId);
+ }
}
@@ -120,6 +144,11 @@ DenseTensorAttribute::onLoad()
auto raw = _denseTensorStore.allocRawBuffer();
tensorReader.readTensor(raw.data, _denseTensorStore.getBufSize());
_refVector.push_back(raw.ref);
+ if (_index) {
+ // This ensures that get_vector() (via getTensor()) is able to find the newly added tensor.
+ setCommittedDocIdLimit(lid + 1);
+ _index->add_document(lid);
+ }
} else {
_refVector.push_back(EntryRef());
}
@@ -154,4 +183,12 @@ DenseTensorAttribute::getVersion() const
return DENSE_TENSOR_ATTRIBUTE_VERSION;
}
+vespalib::tensor::TypedCells
+DenseTensorAttribute::get_vector(uint32_t docid) const
+{
+ MutableDenseTensorView tensor_view(_denseTensorStore.type());
+ getTensor(docid, tensor_view);
+ return tensor_view.cellsRef();
+}
+
}
diff --git a/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute.h b/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute.h
index 593741cef39..f9a8a81b56b 100644
--- a/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute.h
+++ b/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute.h
@@ -2,35 +2,47 @@
#pragma once
-#include "tensor_attribute.h"
+#include "default_nearest_neighbor_index_factory.h"
#include "dense_tensor_store.h"
+#include "doc_vector_access.h"
+#include "tensor_attribute.h"
+#include <memory>
-namespace vespalib { namespace tensor { class MutableDenseTensorView; }}
+namespace vespalib::tensor { class MutableDenseTensorView; }
-namespace search {
+namespace search::tensor {
-namespace tensor {
+class NearestNeighborIndex;
/**
* Attribute vector class used to store dense tensors for all
* documents in memory.
*/
-class DenseTensorAttribute : public TensorAttribute
-{
+class DenseTensorAttribute : public TensorAttribute, public DocVectorAccess {
+private:
DenseTensorStore _denseTensorStore;
+ std::unique_ptr<NearestNeighborIndex> _index;
+
+ void consider_remove_from_index(DocId docid);
+
public:
- DenseTensorAttribute(vespalib::stringref baseFileName, const Config &cfg);
+ DenseTensorAttribute(vespalib::stringref baseFileName, const Config& cfg,
+ const NearestNeighborIndexFactory& index_factory = DefaultNearestNeighborIndexFactory());
virtual ~DenseTensorAttribute();
- virtual void setTensor(DocId docId, const Tensor &tensor) override;
- virtual std::unique_ptr<Tensor> getTensor(DocId docId) const override;
- virtual void getTensor(DocId docId, vespalib::tensor::MutableDenseTensorView &tensor) const override;
- virtual bool onLoad() override;
- virtual std::unique_ptr<AttributeSaver> onInitSave(vespalib::stringref fileName) override;
- virtual void compactWorst() override;
- virtual uint32_t getVersion() const override;
+ // Implements TensorAttribute
+ uint32_t clearDoc(DocId docId) override;
+ void setTensor(DocId docId, const Tensor &tensor) override;
+ std::unique_ptr<Tensor> getTensor(DocId docId) const override;
+ void getTensor(DocId docId, vespalib::tensor::MutableDenseTensorView &tensor) const override;
+ bool onLoad() override;
+ std::unique_ptr<AttributeSaver> onInitSave(vespalib::stringref fileName) override;
+ void compactWorst() override;
+ uint32_t getVersion() const override;
+
+ // Implements DocVectorAccess
+ vespalib::tensor::TypedCells get_vector(uint32_t docid) const override;
+
+ const NearestNeighborIndex* nearest_neighbor_index() const { return _index.get(); }
};
-
-} // namespace search::tensor
-
-} // namespace search
+}
diff --git a/searchlib/src/vespa/searchlib/tensor/distance_function.h b/searchlib/src/vespa/searchlib/tensor/distance_function.h
index 8dfb77ddccb..b682824c805 100644
--- a/searchlib/src/vespa/searchlib/tensor/distance_function.h
+++ b/searchlib/src/vespa/searchlib/tensor/distance_function.h
@@ -2,6 +2,8 @@
#pragma once
+#include <memory>
+
namespace vespalib::tensor { struct TypedCells; }
namespace search::tensor {
@@ -14,6 +16,7 @@ namespace search::tensor {
*/
class DistanceFunction {
public:
+ using UP = std::unique_ptr<DistanceFunction>;
virtual ~DistanceFunction() {}
virtual double calc(const vespalib::tensor::TypedCells& lhs, const vespalib::tensor::TypedCells& rhs) const = 0;
};
diff --git a/searchlib/src/vespa/searchlib/tensor/distance_functions.h b/searchlib/src/vespa/searchlib/tensor/distance_functions.h
index 1e8727e92aa..494d1a859b6 100644
--- a/searchlib/src/vespa/searchlib/tensor/distance_functions.h
+++ b/searchlib/src/vespa/searchlib/tensor/distance_functions.h
@@ -3,6 +3,7 @@
#pragma once
#include "distance_function.h"
+#include <vespa/eval/tensor/dense/typed_cells.h>
namespace search::tensor {
diff --git a/searchlib/src/vespa/searchlib/tensor/hnsw_index.cpp b/searchlib/src/vespa/searchlib/tensor/hnsw_index.cpp
index be53b758841..54779408b37 100644
--- a/searchlib/src/vespa/searchlib/tensor/hnsw_index.cpp
+++ b/searchlib/src/vespa/searchlib/tensor/hnsw_index.cpp
@@ -44,7 +44,7 @@ HnswIndex::max_links_for_level(uint32_t level) const
uint32_t
HnswIndex::make_node_for_document(uint32_t docid)
{
- uint32_t max_level = _level_generator.max_level();
+ uint32_t max_level = _level_generator->max_level();
// TODO: Add capping on num_levels
uint32_t num_levels = max_level + 1;
// Note: The level array instance lives as long as the document is present in the index.
@@ -136,7 +136,7 @@ HnswIndex::select_neighbors(const HnswCandidateVector& neighbors, uint32_t max_l
}
void
-HnswIndex::connect_new_node(uint32_t docid, const LinkArray& neighbors, uint32_t level)
+HnswIndex::connect_new_node(uint32_t docid, const LinkArrayRef &neighbors, uint32_t level)
{
set_link_array(docid, level, neighbors);
for (uint32_t neighbor_docid : neighbors) {
@@ -170,11 +170,11 @@ double
HnswIndex::calc_distance(const TypedCells& lhs, uint32_t rhs_docid) const
{
auto rhs = get_vector(rhs_docid);
- return _distance_func.calc(lhs, rhs);
+ return _distance_func->calc(lhs, rhs);
}
HnswCandidate
-HnswIndex::find_nearest_in_layer(const TypedCells& input, const HnswCandidate& entry_point, uint32_t level)
+HnswIndex::find_nearest_in_layer(const TypedCells& input, const HnswCandidate& entry_point, uint32_t level) const
{
HnswCandidate nearest = entry_point;
bool keep_searching = true;
@@ -192,7 +192,7 @@ HnswIndex::find_nearest_in_layer(const TypedCells& input, const HnswCandidate& e
}
void
-HnswIndex::search_layer(const TypedCells& input, uint32_t neighbors_to_find, FurthestPriQ& best_neighbors, uint32_t level)
+HnswIndex::search_layer(const TypedCells& input, uint32_t neighbors_to_find, FurthestPriQ& best_neighbors, uint32_t level) const
{
NearestPriQ candidates;
// TODO: Add proper handling of visited set.
@@ -227,11 +227,11 @@ HnswIndex::search_layer(const TypedCells& input, uint32_t neighbors_to_find, Fur
}
}
-HnswIndex::HnswIndex(const DocVectorAccess& vectors, const DistanceFunction& distance_func,
- RandomLevelGenerator& level_generator, const Config& cfg)
+HnswIndex::HnswIndex(const DocVectorAccess& vectors, DistanceFunction::UP distance_func,
+ RandomLevelGenerator::UP level_generator, const Config& cfg)
: _vectors(vectors),
- _distance_func(distance_func),
- _level_generator(level_generator),
+ _distance_func(std::move(distance_func)),
+ _level_generator(std::move(level_generator)),
_cfg(cfg),
_node_refs(),
_nodes(make_default_node_store_config()),
@@ -310,24 +310,32 @@ HnswIndex::remove_document(uint32_t docid)
_node_refs[docid].store_release(invalid);
}
-std::vector<uint32_t>
-HnswIndex::find_top_k(uint32_t k, TypedCells vector, uint32_t explore_k)
+struct NeighborsByDocId {
+ bool operator() (const NearestNeighborIndex::Neighbor &lhs,
+ const NearestNeighborIndex::Neighbor &rhs)
+ {
+ return (lhs.docid < rhs.docid);
+ }
+};
+
+std::vector<NearestNeighborIndex::Neighbor>
+HnswIndex::find_top_k(uint32_t k, TypedCells vector, uint32_t explore_k) const
{
- std::vector<uint32_t> result;
+ std::vector<Neighbor> result;
FurthestPriQ candidates = top_k_candidates(vector, std::max(k, explore_k));
while (candidates.size() > k) {
candidates.pop();
}
result.reserve(candidates.size());
for (const HnswCandidate & hit : candidates.peek()) {
- result.emplace_back(hit.docid);
+ result.emplace_back(hit.docid, hit.distance);
}
- std::sort(result.begin(), result.end());
+ std::sort(result.begin(), result.end(), NeighborsByDocId());
return result;
}
FurthestPriQ
-HnswIndex::top_k_candidates(const TypedCells &vector, uint32_t k)
+HnswIndex::top_k_candidates(const TypedCells &vector, uint32_t k) const
{
FurthestPriQ best_neighbors;
if (_entry_level < 0) {
@@ -363,5 +371,28 @@ HnswIndex::get_node(uint32_t docid) const
return HnswNode(result);
}
+void
+HnswIndex::set_node(uint32_t docid, const HnswNode &node)
+{
+ _node_refs.ensure_size(docid + 1, AtomicEntryRef());
+ // A document cannot be added twice.
+ assert(!_node_refs[docid].load_acquire().valid());
+
+ // make new node
+ size_t num_levels = node.size();
+ assert(num_levels > 0);
+ LevelArray levels(num_levels, AtomicEntryRef());
+ auto node_ref = _nodes.add(levels);
+ _node_refs[docid].store_release(node_ref);
+
+ for (size_t level = 0; level < num_levels; ++level) {
+ connect_new_node(docid, node.level(level), level);
+ }
+ int max_level = num_levels - 1;
+ if (_entry_level < max_level) {
+ _entry_docid = docid;
+ _entry_level = max_level;
+ }
}
+}
diff --git a/searchlib/src/vespa/searchlib/tensor/hnsw_index.h b/searchlib/src/vespa/searchlib/tensor/hnsw_index.h
index 814148072ca..a8129032c11 100644
--- a/searchlib/src/vespa/searchlib/tensor/hnsw_index.h
+++ b/searchlib/src/vespa/searchlib/tensor/hnsw_index.h
@@ -2,10 +2,12 @@
#pragma once
+#include "distance_function.h"
#include "doc_vector_access.h"
#include "hnsw_index_utils.h"
#include "hnsw_node.h"
#include "nearest_neighbor_index.h"
+#include "random_level_generator.h"
#include <vespa/eval/tensor/dense/typed_cells.h>
#include <vespa/searchlib/common/bitvector.h>
#include <vespa/vespalib/datastore/array_store.h>
@@ -15,9 +17,6 @@
namespace search::tensor {
-class DistanceFunction;
-class RandomLevelGenerator;
-
/**
* Implementation of a hierarchical navigable small world graph (HNSW)
* that is used for approximate K-nearest neighbor search.
@@ -82,8 +81,8 @@ protected:
using TypedCells = vespalib::tensor::TypedCells;
const DocVectorAccess& _vectors;
- const DistanceFunction& _distance_func;
- RandomLevelGenerator& _level_generator;
+ DistanceFunction::UP _distance_func;
+ RandomLevelGenerator::UP _level_generator;
Config _cfg;
NodeRefVector _node_refs;
NodeStore _nodes;
@@ -111,7 +110,7 @@ protected:
LinkArray select_neighbors_heuristic(const HnswCandidateVector& neighbors, uint32_t max_links) const;
LinkArray select_neighbors_simple(const HnswCandidateVector& neighbors, uint32_t max_links) const;
LinkArray select_neighbors(const HnswCandidateVector& neighbors, uint32_t max_links) const;
- void connect_new_node(uint32_t docid, const LinkArray& neighbors, uint32_t level);
+ void connect_new_node(uint32_t docid, const LinkArrayRef &neighbors, uint32_t level);
void remove_link_to(uint32_t remove_from, uint32_t remove_id, uint32_t level);
inline TypedCells get_vector(uint32_t docid) const {
@@ -124,18 +123,20 @@ protected:
/**
* Performs a greedy search in the given layer to find the candidate that is nearest the input vector.
*/
- HnswCandidate find_nearest_in_layer(const TypedCells& input, const HnswCandidate& entry_point, uint32_t level);
- void search_layer(const TypedCells& input, uint32_t neighbors_to_find, FurthestPriQ& found_neighbors, uint32_t level);
+ HnswCandidate find_nearest_in_layer(const TypedCells& input, const HnswCandidate& entry_point, uint32_t level) const;
+ void search_layer(const TypedCells& input, uint32_t neighbors_to_find, FurthestPriQ& found_neighbors, uint32_t level) const;
public:
- HnswIndex(const DocVectorAccess& vectors, const DistanceFunction& distance_func,
- RandomLevelGenerator& level_generator, const Config& cfg);
+ HnswIndex(const DocVectorAccess& vectors, DistanceFunction::UP distance_func,
+ RandomLevelGenerator::UP level_generator, const Config& cfg);
~HnswIndex() override;
+ const Config& config() const { return _cfg; }
+
void add_document(uint32_t docid) override;
void remove_document(uint32_t docid) override;
- std::vector<uint32_t> find_top_k(uint32_t k, TypedCells vector, uint32_t explore_k) override;
- FurthestPriQ top_k_candidates(const TypedCells &vector, uint32_t k);
+ std::vector<Neighbor> find_top_k(uint32_t k, TypedCells vector, uint32_t explore_k) const override;
+ FurthestPriQ top_k_candidates(const TypedCells &vector, uint32_t k) const;
// TODO: Add support for generation handling and cleanup (transfer_hold_lists, trim_hold_lists)
@@ -144,8 +145,7 @@ public:
// Should only be used by unit tests.
HnswNode get_node(uint32_t docid) const;
-
- // TODO: Implement set_node() as well for use in unit tests.
+ void set_node(uint32_t docid, const HnswNode &node);
};
}
diff --git a/searchlib/src/vespa/searchlib/tensor/nearest_neighbor_index.cpp b/searchlib/src/vespa/searchlib/tensor/nearest_neighbor_index.cpp
new file mode 100644
index 00000000000..f31230af381
--- /dev/null
+++ b/searchlib/src/vespa/searchlib/tensor/nearest_neighbor_index.cpp
@@ -0,0 +1,3 @@
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include "nearest_neighbor_index.h"
diff --git a/searchlib/src/vespa/searchlib/tensor/nearest_neighbor_index.h b/searchlib/src/vespa/searchlib/tensor/nearest_neighbor_index.h
index 2ae322fe76e..f933af0147e 100644
--- a/searchlib/src/vespa/searchlib/tensor/nearest_neighbor_index.h
+++ b/searchlib/src/vespa/searchlib/tensor/nearest_neighbor_index.h
@@ -13,10 +13,20 @@ namespace search::tensor {
*/
class NearestNeighborIndex {
public:
+ struct Neighbor {
+ uint32_t docid;
+ double distance;
+ Neighbor(uint32_t id, double dist)
+ : docid(id), distance(dist)
+ {}
+ Neighbor() : docid(0), distance(0.0) {}
+ };
virtual ~NearestNeighborIndex() {}
virtual void add_document(uint32_t docid) = 0;
virtual void remove_document(uint32_t docid) = 0;
- virtual std::vector<uint32_t> find_top_k(uint32_t k, vespalib::tensor::TypedCells vector, uint32_t explore_k) = 0;
+ virtual std::vector<Neighbor> find_top_k(uint32_t k,
+ vespalib::tensor::TypedCells vector,
+ uint32_t explore_k) const = 0;
};
}
diff --git a/searchlib/src/vespa/searchlib/tensor/nearest_neighbor_index_factory.h b/searchlib/src/vespa/searchlib/tensor/nearest_neighbor_index_factory.h
new file mode 100644
index 00000000000..c09403df5e0
--- /dev/null
+++ b/searchlib/src/vespa/searchlib/tensor/nearest_neighbor_index_factory.h
@@ -0,0 +1,26 @@
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <vespa/eval/eval/value_type.h>
+#include <memory>
+
+namespace search::attribute { class HnswIndexParams; }
+
+namespace search::tensor {
+
+class DocVectorAccess;
+class NearestNeighborIndex;
+
+/**
+ * Factory interface used to instantiate an index used for (approximate) nearest neighbor search.
+ */
+class NearestNeighborIndexFactory {
+public:
+ virtual ~NearestNeighborIndexFactory() {}
+ virtual std::unique_ptr<NearestNeighborIndex> make(const DocVectorAccess& vectors,
+ vespalib::eval::ValueType::CellType cell_type,
+ const search::attribute::HnswIndexParams& params) const = 0;
+};
+
+}
diff --git a/searchlib/src/vespa/searchlib/tensor/random_level_generator.h b/searchlib/src/vespa/searchlib/tensor/random_level_generator.h
index 0fcac977d9d..0f4c7c34445 100644
--- a/searchlib/src/vespa/searchlib/tensor/random_level_generator.h
+++ b/searchlib/src/vespa/searchlib/tensor/random_level_generator.h
@@ -2,6 +2,8 @@
#pragma once
+#include <memory>
+
namespace search::tensor {
/**
@@ -9,6 +11,7 @@ namespace search::tensor {
*/
class RandomLevelGenerator {
public:
+ using UP = std::unique_ptr<RandomLevelGenerator>;
virtual ~RandomLevelGenerator() {}
virtual uint32_t max_level() = 0;
};
diff --git a/searchsummary/src/tests/docsummary/matched_elements_filter/matched_elements_filter_test.cpp b/searchsummary/src/tests/docsummary/matched_elements_filter/matched_elements_filter_test.cpp
index 9019a212f3f..dff3acc5b89 100644
--- a/searchsummary/src/tests/docsummary/matched_elements_filter/matched_elements_filter_test.cpp
+++ b/searchsummary/src/tests/docsummary/matched_elements_filter/matched_elements_filter_test.cpp
@@ -246,6 +246,7 @@ TEST_F(MatchedElementsFilterTest, filters_elements_in_array_field_value_when_inp
expect_filtered("array_in_doc", {0, 1, 2}, "[{'name':'a','weight':3},"
"{'name':'b','weight':5},"
"{'name':'c','weight':7}]");
+ expect_filtered("array_in_doc", {0, 1, 100}, "[]");
}
TEST_F(MatchedElementsFilterTest, struct_field_mapper_is_setup_for_array_field_value)
@@ -276,6 +277,7 @@ TEST_F(MatchedElementsFilterTest, filters_elements_in_map_field_value_when_input
expect_filtered("map_in_doc", {0, 1, 2}, "[{'key':'a','value':{'name':'a','weight':3}},"
"{'key':'b','value':{'name':'b','weight':5}},"
"{'key':'c','value':{'name':'c','weight':7}}]");
+ expect_filtered("map_in_doc", {0, 1, 100}, "[]");
}
TEST_F(MatchedElementsFilterTest, struct_field_mapper_is_setup_for_map_field_value)
diff --git a/searchsummary/src/vespa/searchsummary/docsummary/summaryfieldconverter.cpp b/searchsummary/src/vespa/searchsummary/docsummary/summaryfieldconverter.cpp
index 7368a199569..ada14bf17f5 100644
--- a/searchsummary/src/vespa/searchsummary/docsummary/summaryfieldconverter.cpp
+++ b/searchsummary/src/vespa/searchsummary/docsummary/summaryfieldconverter.cpp
@@ -389,9 +389,11 @@ private:
MapFieldValueInserter map_inserter(_inserter, _tokenize);
if (filter_matching_elements()) {
assert(v.has_no_erased_keys());
- for (uint32_t id_to_keep : (*_matching_elems)) {
- auto entry = v[id_to_keep];
- map_inserter.insert_entry(*entry.first, *entry.second);
+ if (!_matching_elems->empty() && _matching_elems->back() < v.size()) {
+ for (uint32_t id_to_keep : (*_matching_elems)) {
+ auto entry = v[id_to_keep];
+ map_inserter.insert_entry(*entry.first, *entry.second);
+ }
}
} else {
for (const auto &entry : v) {
@@ -406,8 +408,10 @@ private:
ArrayInserter ai(a);
SlimeFiller conv(ai, _tokenize);
if (filter_matching_elements()) {
- for (uint32_t id_to_keep : (*_matching_elems)) {
- value[id_to_keep].accept(conv);
+ if (!_matching_elems->empty() && _matching_elems->back() < value.size()) {
+ for (uint32_t id_to_keep : (*_matching_elems)) {
+ value[id_to_keep].accept(conv);
+ }
}
} else {
for (const FieldValue &fv : value) {
diff --git a/security-tools/pom.xml b/security-tools/pom.xml
index 38b14ce957f..195e2d06311 100644
--- a/security-tools/pom.xml
+++ b/security-tools/pom.xml
@@ -57,6 +57,7 @@
<exclude>META-INF/*.SF</exclude>
<exclude>META-INF/*.DSA</exclude>
<exclude>META-INF/*.RSA</exclude>
+ <exclude>META-INF/versions/*/module-info.class</exclude>
</excludes>
</filter>
</filters>
diff --git a/security-tools/src/main/java/com/yahoo/vespa/security/tool/securityenv/Main.java b/security-tools/src/main/java/com/yahoo/vespa/security/tool/securityenv/Main.java
index 367d7b9dd83..c314d17e018 100644
--- a/security-tools/src/main/java/com/yahoo/vespa/security/tool/securityenv/Main.java
+++ b/security-tools/src/main/java/com/yahoo/vespa/security/tool/securityenv/Main.java
@@ -54,6 +54,9 @@ public class Main {
MixedMode mixedMode = TransportSecurityUtils.getInsecureMixedMode(envVars);
if (options.isPresent() && mixedMode != MixedMode.PLAINTEXT_CLIENT_MIXED_SERVER) {
outputVariables.put(OutputVariable.TLS_ENABLED, "1");
+ if (options.get().isHostnameValidationDisabled()) {
+ outputVariables.put(OutputVariable.DISABLE_HOSTNAME_VALIDATION, "1");
+ }
options.get().getCaCertificatesFile()
.ifPresent(caCertFile -> outputVariables.put(OutputVariable.CA_CERTIFICATE, caCertFile.toString()));
options.get().getCertificatesFile()
diff --git a/security-tools/src/main/java/com/yahoo/vespa/security/tool/securityenv/OutputVariable.java b/security-tools/src/main/java/com/yahoo/vespa/security/tool/securityenv/OutputVariable.java
index dd248d05aac..9a90a145f30 100644
--- a/security-tools/src/main/java/com/yahoo/vespa/security/tool/securityenv/OutputVariable.java
+++ b/security-tools/src/main/java/com/yahoo/vespa/security/tool/securityenv/OutputVariable.java
@@ -10,7 +10,8 @@ enum OutputVariable {
TLS_ENABLED("VESPA_TLS_ENABLED", "Set to '1' if TLS is enabled in Vespa"),
CA_CERTIFICATE("VESPA_TLS_CA_CERT", "Path to CA certificates file"),
CERTIFICATE("VESPA_TLS_CERT", "Path to certificate file"),
- PRIVATE_KEY("VESPA_TLS_PRIVATE_KEY", "Path to private key file");
+ PRIVATE_KEY("VESPA_TLS_PRIVATE_KEY", "Path to private key file"),
+ DISABLE_HOSTNAME_VALIDATION("VESPA_TLS_HOSTNAME_VALIDATION_DISABLED", "Set to '1' if TLS hostname validation is disabled");
private final String variableName;
private final String description;
diff --git a/security-tools/src/main/sh/vespa-curl-wrapper b/security-tools/src/main/sh/vespa-curl-wrapper
index e286e121f64..b4fd9224a8a 100755
--- a/security-tools/src/main/sh/vespa-curl-wrapper
+++ b/security-tools/src/main/sh/vespa-curl-wrapper
@@ -88,6 +88,11 @@ then
CURL_PARAMETERS=("${CURL_PARAMETERS[@]/http:/https:}")
fi
+if [ -n "${VESPA_TLS_HOSTNAME_VALIDATION_DISABLED}" ]
+then
+ CURL_PARAMETERS=("--insecure" "${CURL_PARAMETERS[@]}")
+fi
+
if [ -n "${VESPA_TLS_CA_CERT}" ]
then
CURL_PARAMETERS=("--cacert" "${VESPA_TLS_CA_CERT}" "${CURL_PARAMETERS[@]}")
diff --git a/security-tools/src/test/java/com/yahoo/vespa/security/tool/securityenv/MainTest.java b/security-tools/src/test/java/com/yahoo/vespa/security/tool/securityenv/MainTest.java
index b563ebd14f4..45626820f4d 100644
--- a/security-tools/src/test/java/com/yahoo/vespa/security/tool/securityenv/MainTest.java
+++ b/security-tools/src/test/java/com/yahoo/vespa/security/tool/securityenv/MainTest.java
@@ -106,6 +106,7 @@ public class MainTest {
TransportSecurityOptions options = new TransportSecurityOptions.Builder()
.withCertificates(Paths.get("/path/to/certificate"), Paths.get("/path/to/key"))
.withCaCertificates(Paths.get("/path/to/cacerts"))
+ .withHostnameValidationDisabled(true)
.build();
Path configFile = tmpFolder.newFile().toPath();
options.toJsonFile(configFile);
diff --git a/security-tools/src/test/resources/bash-output.txt b/security-tools/src/test/resources/bash-output.txt
index c07c667af47..182dc177d42 100644
--- a/security-tools/src/test/resources/bash-output.txt
+++ b/security-tools/src/test/resources/bash-output.txt
@@ -2,3 +2,4 @@ VESPA_TLS_ENABLED="1"; export VESPA_TLS_ENABLED;
VESPA_TLS_CA_CERT="/path/to/cacerts"; export VESPA_TLS_CA_CERT;
VESPA_TLS_CERT="/path/to/certificate"; export VESPA_TLS_CERT;
VESPA_TLS_PRIVATE_KEY="/path/to/key"; export VESPA_TLS_PRIVATE_KEY;
+VESPA_TLS_HOSTNAME_VALIDATION_DISABLED="1"; export VESPA_TLS_HOSTNAME_VALIDATION_DISABLED;
diff --git a/security-tools/src/test/resources/csh-output.txt b/security-tools/src/test/resources/csh-output.txt
index 2b6716de92b..2e6cd886c26 100644
--- a/security-tools/src/test/resources/csh-output.txt
+++ b/security-tools/src/test/resources/csh-output.txt
@@ -2,3 +2,4 @@ setenv VESPA_TLS_ENABLED "1";
setenv VESPA_TLS_CA_CERT "/path/to/cacerts";
setenv VESPA_TLS_CERT "/path/to/certificate";
setenv VESPA_TLS_PRIVATE_KEY "/path/to/key";
+setenv VESPA_TLS_HOSTNAME_VALIDATION_DISABLED "1";
diff --git a/security-tools/src/test/resources/expected-help-output.txt b/security-tools/src/test/resources/expected-help-output.txt
index 7d125fe15a2..33ad3b6d232 100644
--- a/security-tools/src/test/resources/expected-help-output.txt
+++ b/security-tools/src/test/resources/expected-help-output.txt
@@ -9,3 +9,5 @@ The output may include the following variables:
- 'VESPA_TLS_CA_CERT': Path to CA certificates file
- 'VESPA_TLS_CERT': Path to certificate file
- 'VESPA_TLS_PRIVATE_KEY': Path to private key file
+ - 'VESPA_TLS_HOSTNAME_VALIDATION_DISABLED': Set to '1' if TLS hostname
+validation is disabled
diff --git a/security-tools/src/test/resources/no-security-output.txt b/security-tools/src/test/resources/no-security-output.txt
index 3467f1316b5..257a2747ee2 100644
--- a/security-tools/src/test/resources/no-security-output.txt
+++ b/security-tools/src/test/resources/no-security-output.txt
@@ -2,3 +2,4 @@ unset VESPA_TLS_ENABLED;
unset VESPA_TLS_CA_CERT;
unset VESPA_TLS_CERT;
unset VESPA_TLS_PRIVATE_KEY;
+unset VESPA_TLS_HOSTNAME_VALIDATION_DISABLED;
diff --git a/security-utils/src/main/java/com/yahoo/security/SslContextBuilder.java b/security-utils/src/main/java/com/yahoo/security/SslContextBuilder.java
index d2b98fd20d9..f3932c84a17 100644
--- a/security-utils/src/main/java/com/yahoo/security/SslContextBuilder.java
+++ b/security-utils/src/main/java/com/yahoo/security/SslContextBuilder.java
@@ -35,6 +35,7 @@ public class SslContextBuilder {
private TrustManagerFactory trustManagerFactory = TrustManagerUtils::createDefaultX509TrustManager;
private KeyManagerFactory keyManagerFactory = KeyManagerUtils::createDefaultX509KeyManager;
private X509ExtendedKeyManager keyManager;
+ private X509ExtendedTrustManager trustManager;
public SslContextBuilder() {}
@@ -121,15 +122,25 @@ public class SslContextBuilder {
return this;
}
+ /**
+ * Note: Callee is responsible for configuring the trust manager.
+ * Any truststore configured by {@link #withTrustStore(KeyStore)} or the other overloads will be ignored.
+ */
+ public SslContextBuilder withTrustManager(X509ExtendedTrustManager trustManager) {
+ this.trustManager = trustManager;
+ return this;
+ }
+
public SSLContext build() {
try {
SSLContext sslContext = SSLContext.getInstance(TlsContext.SSL_CONTEXT_VERSION);
- TrustManager[] trustManagers = new TrustManager[] { trustManagerFactory.createTrustManager(trustStoreSupplier.get()) };
+ X509ExtendedTrustManager trustManager = this.trustManager != null
+ ? this.trustManager
+ : trustManagerFactory.createTrustManager(trustStoreSupplier.get());
X509ExtendedKeyManager keyManager = this.keyManager != null
? this.keyManager
: keyManagerFactory.createKeyManager(keyStoreSupplier.get(), keyStorePassword);
- KeyManager[] keyManagers = new KeyManager[] {keyManager};
- sslContext.init(keyManagers, trustManagers, null);
+ sslContext.init(new KeyManager[] {keyManager}, new TrustManager[] {trustManager}, null);
return sslContext;
} catch (GeneralSecurityException e) {
throw new RuntimeException(e);
diff --git a/security-utils/src/main/java/com/yahoo/security/tls/ConfigFileBasedTlsContext.java b/security-utils/src/main/java/com/yahoo/security/tls/ConfigFileBasedTlsContext.java
index f746480b126..28854c59b2c 100644
--- a/security-utils/src/main/java/com/yahoo/security/tls/ConfigFileBasedTlsContext.java
+++ b/security-utils/src/main/java/com/yahoo/security/tls/ConfigFileBasedTlsContext.java
@@ -12,11 +12,9 @@ import com.yahoo.security.tls.policy.AuthorizedPeers;
import javax.net.ssl.SSLContext;
import javax.net.ssl.SSLEngine;
import javax.net.ssl.SSLParameters;
-import javax.net.ssl.X509ExtendedTrustManager;
import java.io.IOException;
import java.io.UncheckedIOException;
import java.lang.ref.WeakReference;
-import java.nio.file.Files;
import java.nio.file.Path;
import java.security.KeyStore;
import java.time.Duration;
@@ -110,12 +108,14 @@ public class ConfigFileBasedTlsContext implements TlsContext {
MutableX509TrustManager mutableTrustManager,
MutableX509KeyManager mutableKeyManager,
PeerAuthentication peerAuthentication) {
+
+ HostnameVerification hostnameVerification = options.isHostnameValidationDisabled() ? HostnameVerification.DISABLED : HostnameVerification.ENABLED;
+ PeerAuthorizerTrustManager authorizerTrustManager = options.getAuthorizedPeers()
+ .map(authorizedPeers -> new PeerAuthorizerTrustManager(authorizedPeers, mode, hostnameVerification, mutableTrustManager))
+ .orElseGet(() -> new PeerAuthorizerTrustManager(new AuthorizedPeers(com.yahoo.vespa.jdk8compat.Set.of()), AuthorizationMode.DISABLE, hostnameVerification, mutableTrustManager));
SSLContext sslContext = new SslContextBuilder()
.withKeyManager(mutableKeyManager)
- .withTrustManagerFactory(
- ignoredTruststore -> options.getAuthorizedPeers()
- .map(authorizedPeers -> (X509ExtendedTrustManager) new PeerAuthorizerTrustManager(authorizedPeers, mode, mutableTrustManager))
- .orElseGet(() -> new PeerAuthorizerTrustManager(new AuthorizedPeers(com.yahoo.vespa.jdk8compat.Set.of()), AuthorizationMode.DISABLE, mutableTrustManager)))
+ .withTrustManager(authorizerTrustManager)
.build();
List<String> acceptedCiphers = options.getAcceptedCiphers();
Set<String> ciphers = acceptedCiphers.isEmpty() ? TlsContext.ALLOWED_CIPHER_SUITES : new HashSet<>(acceptedCiphers);
diff --git a/security-utils/src/main/java/com/yahoo/security/tls/DefaultTlsContext.java b/security-utils/src/main/java/com/yahoo/security/tls/DefaultTlsContext.java
index c3f10a464a5..def3e49be4d 100644
--- a/security-utils/src/main/java/com/yahoo/security/tls/DefaultTlsContext.java
+++ b/security-utils/src/main/java/com/yahoo/security/tls/DefaultTlsContext.java
@@ -34,8 +34,9 @@ public class DefaultTlsContext implements TlsContext {
List<X509Certificate> caCertificates,
AuthorizedPeers authorizedPeers,
AuthorizationMode mode,
- PeerAuthentication peerAuthentication) {
- this(createSslContext(certificates, privateKey, caCertificates, authorizedPeers, mode), peerAuthentication);
+ PeerAuthentication peerAuthentication,
+ HostnameVerification hostnameVerification) {
+ this(createSslContext(certificates, privateKey, caCertificates, authorizedPeers, mode, hostnameVerification), peerAuthentication);
}
public DefaultTlsContext(SSLContext sslContext, PeerAuthentication peerAuthentication) {
@@ -120,7 +121,8 @@ public class DefaultTlsContext implements TlsContext {
PrivateKey privateKey,
List<X509Certificate> caCertificates,
AuthorizedPeers authorizedPeers,
- AuthorizationMode mode) {
+ AuthorizationMode mode,
+ HostnameVerification hostnameVerification) {
SslContextBuilder builder = new SslContextBuilder();
if (!certificates.isEmpty()) {
builder.withKeyStore(privateKey, certificates);
@@ -129,12 +131,12 @@ public class DefaultTlsContext implements TlsContext {
builder.withTrustStore(caCertificates);
}
if (authorizedPeers != null) {
- builder.withTrustManagerFactory(truststore -> new PeerAuthorizerTrustManager(authorizedPeers, mode, truststore));
+ builder.withTrustManagerFactory(truststore -> new PeerAuthorizerTrustManager(authorizedPeers, mode, hostnameVerification, truststore));
} else {
- builder.withTrustManagerFactory(truststore -> new PeerAuthorizerTrustManager(new AuthorizedPeers(com.yahoo.vespa.jdk8compat.Set.of()), AuthorizationMode.DISABLE, truststore));
+ builder.withTrustManagerFactory(truststore -> new PeerAuthorizerTrustManager(
+ new AuthorizedPeers(com.yahoo.vespa.jdk8compat.Set.of()), AuthorizationMode.DISABLE, hostnameVerification, truststore));
}
return builder.build();
}
-
}
diff --git a/security-utils/src/main/java/com/yahoo/security/tls/HostnameVerification.java b/security-utils/src/main/java/com/yahoo/security/tls/HostnameVerification.java
new file mode 100644
index 00000000000..a41edc6dc44
--- /dev/null
+++ b/security-utils/src/main/java/com/yahoo/security/tls/HostnameVerification.java
@@ -0,0 +1,7 @@
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.security.tls;
+
+/**
+ * @author bjorncs
+ */
+public enum HostnameVerification { ENABLED, DISABLED }
diff --git a/security-utils/src/main/java/com/yahoo/security/tls/authz/PeerAuthorizerTrustManager.java b/security-utils/src/main/java/com/yahoo/security/tls/authz/PeerAuthorizerTrustManager.java
index 3ddd0861f39..03358190e8a 100644
--- a/security-utils/src/main/java/com/yahoo/security/tls/authz/PeerAuthorizerTrustManager.java
+++ b/security-utils/src/main/java/com/yahoo/security/tls/authz/PeerAuthorizerTrustManager.java
@@ -3,6 +3,7 @@ package com.yahoo.security.tls.authz;
import com.yahoo.security.X509CertificateUtils;
import com.yahoo.security.tls.AuthorizationMode;
+import com.yahoo.security.tls.HostnameVerification;
import com.yahoo.security.tls.TrustManagerUtils;
import com.yahoo.security.tls.policy.AuthorizedPeers;
@@ -14,7 +15,6 @@ import java.net.Socket;
import java.security.KeyStore;
import java.security.cert.CertificateException;
import java.security.cert.X509Certificate;
-import java.util.Objects;
import java.util.Optional;
import java.util.logging.Logger;
@@ -33,15 +33,23 @@ public class PeerAuthorizerTrustManager extends X509ExtendedTrustManager {
private final PeerAuthorizer authorizer;
private final X509ExtendedTrustManager defaultTrustManager;
private final AuthorizationMode mode;
+ private final HostnameVerification hostnameVerification;
- public PeerAuthorizerTrustManager(AuthorizedPeers authorizedPeers, AuthorizationMode mode, X509ExtendedTrustManager defaultTrustManager) {
+ public PeerAuthorizerTrustManager(AuthorizedPeers authorizedPeers,
+ AuthorizationMode mode,
+ HostnameVerification hostnameVerification,
+ X509ExtendedTrustManager defaultTrustManager) {
this.authorizer = new PeerAuthorizer(authorizedPeers);
this.mode = mode;
+ this.hostnameVerification = hostnameVerification;
this.defaultTrustManager = defaultTrustManager;
}
- public PeerAuthorizerTrustManager(AuthorizedPeers authorizedPeers, AuthorizationMode mode, KeyStore truststore) {
- this(authorizedPeers, mode, TrustManagerUtils.createDefaultX509TrustManager(truststore));
+ public PeerAuthorizerTrustManager(AuthorizedPeers authorizedPeers,
+ AuthorizationMode mode,
+ HostnameVerification hostnameVerification,
+ KeyStore truststore) {
+ this(authorizedPeers, mode, hostnameVerification, TrustManagerUtils.createDefaultX509TrustManager(truststore));
}
@Override
@@ -58,28 +66,26 @@ public class PeerAuthorizerTrustManager extends X509ExtendedTrustManager {
@Override
public void checkClientTrusted(X509Certificate[] chain, String authType, Socket socket) throws CertificateException {
- overrideHostnameVerification(socket);
defaultTrustManager.checkClientTrusted(chain, authType, socket);
authorizePeer(chain[0], authType, true, null);
}
@Override
public void checkServerTrusted(X509Certificate[] chain, String authType, Socket socket) throws CertificateException {
- overrideHostnameVerification(socket);
+ overrideHostnameVerificationForClient(socket);
defaultTrustManager.checkServerTrusted(chain, authType, socket);
authorizePeer(chain[0], authType, false, null);
}
@Override
public void checkClientTrusted(X509Certificate[] chain, String authType, SSLEngine sslEngine) throws CertificateException {
- overrideHostnameVerification(sslEngine);
defaultTrustManager.checkClientTrusted(chain, authType, sslEngine);
authorizePeer(chain[0], authType, true, sslEngine);
}
@Override
public void checkServerTrusted(X509Certificate[] chain, String authType, SSLEngine sslEngine) throws CertificateException {
- overrideHostnameVerification(sslEngine);
+ overrideHostnameVerificationForClient(sslEngine);
defaultTrustManager.checkServerTrusted(chain, authType, sslEngine);
authorizePeer(chain[0], authType, false, sslEngine);
}
@@ -121,31 +127,44 @@ public class PeerAuthorizerTrustManager extends X509ExtendedTrustManager {
certificate.getSubjectX500Principal(), X509CertificateUtils.getSubjectAlternativeNames(certificate), authType, isVerifyingClient);
}
- private static void overrideHostnameVerification(SSLEngine engine) {
+ private void overrideHostnameVerificationForClient(SSLEngine engine) {
SSLParameters params = engine.getSSLParameters();
- if (overrideHostnameVerification(params)) {
+ if (overrideHostnameVerificationForClient(params)) {
engine.setSSLParameters(params);
}
}
- private static void overrideHostnameVerification(Socket socket) {
+ private void overrideHostnameVerificationForClient(Socket socket) {
if (socket instanceof SSLSocket) {
SSLSocket sslSocket = (SSLSocket) socket;
SSLParameters params = sslSocket.getSSLParameters();
- if (overrideHostnameVerification(params)) {
+ if (overrideHostnameVerificationForClient(params)) {
sslSocket.setSSLParameters(params);
}
}
}
- // Disable the default hostname verification that is performed by underlying trust manager when 'HTTPS' is used as endpoint identification algorithm.
- // Some http clients, notably the new http client in Java 11, does not allow user configuration of the endpoint algorithm or custom HostnameVerifier.
- private static boolean overrideHostnameVerification(SSLParameters params) {
- if (Objects.equals("HTTPS", params.getEndpointIdentificationAlgorithm())) {
- params.setEndpointIdentificationAlgorithm("");
- return true;
+ // Overrides the endpoint identification algorithm specified in the ssl parameters of the ssl engine/socket.
+ // The underlying trust manager will perform hostname verification if endpoint identification algorithm is set to 'HTTPS'.
+ // Returns true if the parameter instance was modified
+ private boolean overrideHostnameVerificationForClient(SSLParameters params) {
+ String configuredAlgorithm = params.getEndpointIdentificationAlgorithm();
+ switch (hostnameVerification) {
+ case ENABLED:
+ if (!"HTTPS".equals(configuredAlgorithm)) {
+ params.setEndpointIdentificationAlgorithm("HTTPS");
+ return true;
+ }
+ return false;
+ case DISABLED:
+ if (configuredAlgorithm != null && !configuredAlgorithm.isEmpty()) {
+ params.setEndpointIdentificationAlgorithm(""); // disable any configured endpoint identification algorithm
+ return true;
+ }
+ return false;
+ default:
+ throw new IllegalStateException("Unknown host verification type: " + hostnameVerification);
}
- return false;
}
}
diff --git a/security-utils/src/test/java/com/yahoo/security/tls/DefaultTlsContextTest.java b/security-utils/src/test/java/com/yahoo/security/tls/DefaultTlsContextTest.java
index 727a64ae934..00928187f55 100644
--- a/security-utils/src/test/java/com/yahoo/security/tls/DefaultTlsContextTest.java
+++ b/security-utils/src/test/java/com/yahoo/security/tls/DefaultTlsContextTest.java
@@ -46,7 +46,9 @@ public class DefaultTlsContextTest {
singletonList(new RequiredPeerCredential(RequiredPeerCredential.Field.CN, new HostGlobPattern("dummy"))))));
DefaultTlsContext tlsContext =
- new DefaultTlsContext(singletonList(certificate), keyPair.getPrivate(), singletonList(certificate), authorizedPeers, AuthorizationMode.ENFORCE, PeerAuthentication.NEED);
+ new DefaultTlsContext(
+ singletonList(certificate), keyPair.getPrivate(), singletonList(certificate), authorizedPeers,
+ AuthorizationMode.ENFORCE, PeerAuthentication.NEED, HostnameVerification.ENABLED);
SSLEngine sslEngine = tlsContext.createSslEngine();
assertThat(sslEngine).isNotNull();
diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModel.java b/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModel.java
index f559e9336c8..1cfb70560b8 100644
--- a/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModel.java
+++ b/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModel.java
@@ -4,9 +4,9 @@ package com.yahoo.vespa.service.duper;
import com.yahoo.config.model.api.ApplicationInfo;
import com.yahoo.config.provision.ApplicationId;
import com.yahoo.log.LogLevel;
+import com.yahoo.vespa.service.monitor.DuperModelListener;
import java.util.ArrayList;
-import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.TreeMap;
@@ -22,12 +22,16 @@ public class DuperModel {
private final Map<ApplicationId, ApplicationInfo> applications = new TreeMap<>();
private final List<DuperModelListener> listeners = new ArrayList<>();
+ private boolean isComplete = false;
public void registerListener(DuperModelListener listener) {
applications.values().forEach(listener::applicationActivated);
listeners.add(listener);
}
+ public void setCompleteness(boolean isComplete) { this.isComplete = isComplete; }
+ public boolean isComplete() { return isComplete; }
+
public boolean contains(ApplicationId applicationId) {
return applications.containsKey(applicationId);
}
@@ -47,6 +51,6 @@ public class DuperModel {
public List<ApplicationInfo> getApplicationInfos() {
logger.log(LogLevel.DEBUG, "Applications in duper model: " + applications.values().size());
- return Collections.unmodifiableList(new ArrayList<>(applications.values()));
+ return List.copyOf(applications.values());
}
}
diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModelManager.java b/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModelManager.java
index 885368810a8..15c461c7f59 100644
--- a/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModelManager.java
+++ b/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModelManager.java
@@ -12,6 +12,8 @@ import com.yahoo.config.provision.HostName;
import com.yahoo.config.provision.SystemName;
import com.yahoo.vespa.flags.FlagSource;
import com.yahoo.vespa.service.monitor.DuperModelInfraApi;
+import com.yahoo.vespa.service.monitor.DuperModelListener;
+import com.yahoo.vespa.service.monitor.DuperModelProvider;
import com.yahoo.vespa.service.monitor.InfraApplicationApi;
import java.util.ArrayList;
@@ -27,7 +29,7 @@ import java.util.stream.Stream;
/**
* @author hakonhall
*/
-public class DuperModelManager implements DuperModelInfraApi {
+public class DuperModelManager implements DuperModelProvider, DuperModelInfraApi {
// Infrastructure applications
static final ControllerHostApplication controllerHostApplication = new ControllerHostApplication();
@@ -45,6 +47,8 @@ public class DuperModelManager implements DuperModelInfraApi {
// The set of active infrastructure ApplicationInfo. Not all are necessarily in the DuperModel for historical reasons.
private final Set<ApplicationId> activeInfraInfos = new HashSet<>(10);
+ private boolean superModelIsComplete = false;
+ private boolean infraApplicationsIsComplete = false;
@Inject
public DuperModelManager(ConfigserverConfig configServerConfig, FlagSource flagSource, SuperModelProvider superModelProvider) {
@@ -53,7 +57,7 @@ public class DuperModelManager implements DuperModelInfraApi {
superModelProvider, new DuperModel(), flagSource, SystemName.from(configServerConfig.system()));
}
- /** For testing */
+ /** Non-private for testing */
DuperModelManager(boolean multitenant, boolean isController, SuperModelProvider superModelProvider, DuperModel duperModel, FlagSource flagSource, SystemName system) {
this.duperModel = duperModel;
@@ -86,6 +90,16 @@ public class DuperModelManager implements DuperModelInfraApi {
duperModel.remove(applicationId);
}
}
+
+ @Override
+ public void notifyOfCompleteness(SuperModel superModel) {
+ synchronized (monitor) {
+ if (!superModelIsComplete) {
+ superModelIsComplete = true;
+ maybeSetDuperModelAsComplete();
+ }
+ }
+ }
});
}
@@ -93,6 +107,7 @@ public class DuperModelManager implements DuperModelInfraApi {
* Synchronously call {@link DuperModelListener#applicationActivated(ApplicationInfo) listener.applicationActivated()}
* for each currently active application, and forward future changes.
*/
+ @Override
public void registerListener(DuperModelListener listener) {
synchronized (monitor) {
duperModel.registerListener(listener);
@@ -148,9 +163,25 @@ public class DuperModelManager implements DuperModelInfraApi {
}
}
+ @Override
+ public void infraApplicationsIsNowComplete() {
+ synchronized (monitor) {
+ if (!infraApplicationsIsComplete) {
+ infraApplicationsIsComplete = true;
+ maybeSetDuperModelAsComplete();
+ }
+ }
+ }
+
public List<ApplicationInfo> getApplicationInfos() {
synchronized (monitor) {
return duperModel.getApplicationInfos();
}
}
+
+ private void maybeSetDuperModelAsComplete() {
+ if (superModelIsComplete && infraApplicationsIsComplete) {
+ duperModel.setCompleteness(true);
+ }
+ }
}
diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/health/HealthMonitorManager.java b/service-monitor/src/main/java/com/yahoo/vespa/service/health/HealthMonitorManager.java
index 3cc7010e209..d6e15f6af4e 100644
--- a/service-monitor/src/main/java/com/yahoo/vespa/service/health/HealthMonitorManager.java
+++ b/service-monitor/src/main/java/com/yahoo/vespa/service/health/HealthMonitorManager.java
@@ -92,6 +92,10 @@ public class HealthMonitorManager implements MonitorManager, HealthMonitorApi {
}
@Override
+ public void bootstrapComplete() {
+ }
+
+ @Override
public ServiceStatusInfo getStatus(ApplicationId applicationId,
ClusterId clusterId,
ServiceType serviceType,
diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/manager/MonitorManager.java b/service-monitor/src/main/java/com/yahoo/vespa/service/manager/MonitorManager.java
index dd781a02cef..a7579d3f0da 100644
--- a/service-monitor/src/main/java/com/yahoo/vespa/service/manager/MonitorManager.java
+++ b/service-monitor/src/main/java/com/yahoo/vespa/service/manager/MonitorManager.java
@@ -1,7 +1,7 @@
// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.service.manager;
-import com.yahoo.vespa.service.duper.DuperModelListener;
+import com.yahoo.vespa.service.monitor.DuperModelListener;
import com.yahoo.vespa.service.monitor.ServiceStatusProvider;
/**
diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/manager/UnionMonitorManager.java b/service-monitor/src/main/java/com/yahoo/vespa/service/manager/UnionMonitorManager.java
index 3490ad4a5d2..2aacc3eadac 100644
--- a/service-monitor/src/main/java/com/yahoo/vespa/service/manager/UnionMonitorManager.java
+++ b/service-monitor/src/main/java/com/yahoo/vespa/service/manager/UnionMonitorManager.java
@@ -51,4 +51,8 @@ public class UnionMonitorManager implements MonitorManager {
slobrokMonitorManager.applicationRemoved(id);
healthMonitorManager.applicationRemoved(id);
}
+
+ @Override
+ public void bootstrapComplete() {
+ }
}
diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/DuperModelInfraApi.java b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/DuperModelInfraApi.java
index d08bba2bd3d..f9e47b6b80a 100644
--- a/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/DuperModelInfraApi.java
+++ b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/DuperModelInfraApi.java
@@ -27,4 +27,7 @@ public interface DuperModelInfraApi {
/** Update the DuperModel: A supported infrastructure application has been removed or is not active. */
void infraApplicationRemoved(ApplicationId applicationId);
+
+ /** All infra applications that are supposed to activate on config server bootstrap has been activated. */
+ void infraApplicationsIsNowComplete();
}
diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModelListener.java b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/DuperModelListener.java
index a969b6c3f40..f664e5246ca 100644
--- a/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModelListener.java
+++ b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/DuperModelListener.java
@@ -1,34 +1,45 @@
// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.service.duper;
+package com.yahoo.vespa.service.monitor;
import com.yahoo.config.model.api.ApplicationInfo;
import com.yahoo.config.model.api.SuperModel;
import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.vespa.service.duper.DuperModel;
/**
* Interface for listening for changes to the {@link DuperModel}.
*
- * @author hakon
+ * @author hakonhall
*/
public interface DuperModelListener {
/**
* An application has been activated:
*
* <ul>
- * <li>A synthetic application like the config server application has been added/"activated"
+ * <li>A synthetic application like the config server application has been added/activated
* <li>A super model application has been activated (see
* {@link com.yahoo.config.model.api.SuperModelListener#applicationActivated(SuperModel, ApplicationInfo)
* SuperModelListener}
* </ul>
*
- * No other threads will concurrently call any methods on this interface.
+ * <p>No other threads will concurrently call any methods on this interface.</p>
*/
void applicationActivated(ApplicationInfo application);
/**
* Application has been removed.
*
- * No other threads will concurrently call any methods on this interface.
+ * <p>No other threads will concurrently call any methods on this interface.</p>
*/
void applicationRemoved(ApplicationId id);
+
+ /**
+ * During bootstrap of the config server, a number of applications are activated before
+ * resuming normal operations: The normal "tenant" application (making the super model) and
+ * the relevant infrastructure applications. Once all of these have been activated, this method
+ * will be invoked.
+ *
+ * <p>No other threads will concurrently call any methods on this interface.</p>
+ */
+ void bootstrapComplete();
}
diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/DuperModelProvider.java b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/DuperModelProvider.java
new file mode 100644
index 00000000000..a90fa418054
--- /dev/null
+++ b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/DuperModelProvider.java
@@ -0,0 +1,6 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.service.monitor;
+
+public interface DuperModelProvider {
+ void registerListener(DuperModelListener listener);
+}
diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/slobrok/SlobrokMonitorManagerImpl.java b/service-monitor/src/main/java/com/yahoo/vespa/service/slobrok/SlobrokMonitorManagerImpl.java
index e3ea48ca9fe..e7a8d33ea14 100644
--- a/service-monitor/src/main/java/com/yahoo/vespa/service/slobrok/SlobrokMonitorManagerImpl.java
+++ b/service-monitor/src/main/java/com/yahoo/vespa/service/slobrok/SlobrokMonitorManagerImpl.java
@@ -73,6 +73,10 @@ public class SlobrokMonitorManagerImpl implements SlobrokApi, MonitorManager {
}
@Override
+ public void bootstrapComplete() {
+ }
+
+ @Override
public List<Mirror.Entry> lookup(ApplicationId id, String pattern) {
synchronized (monitor) {
SlobrokMonitor slobrokMonitor = slobrokMonitors.get(id);
diff --git a/service-monitor/src/test/java/com/yahoo/vespa/service/duper/DuperModelTest.java b/service-monitor/src/test/java/com/yahoo/vespa/service/duper/DuperModelTest.java
index 31fd266649a..dc90035be71 100644
--- a/service-monitor/src/test/java/com/yahoo/vespa/service/duper/DuperModelTest.java
+++ b/service-monitor/src/test/java/com/yahoo/vespa/service/duper/DuperModelTest.java
@@ -3,6 +3,7 @@ package com.yahoo.vespa.service.duper;
import com.yahoo.config.model.api.ApplicationInfo;
import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.vespa.service.monitor.DuperModelListener;
import org.junit.Before;
import org.junit.Test;
diff --git a/service-monitor/src/test/java/com/yahoo/vespa/service/model/ExampleModel.java b/service-monitor/src/test/java/com/yahoo/vespa/service/model/ExampleModel.java
index 0f7c0dde357..3fb10f1f24e 100644
--- a/service-monitor/src/test/java/com/yahoo/vespa/service/model/ExampleModel.java
+++ b/service-monitor/src/test/java/com/yahoo/vespa/service/model/ExampleModel.java
@@ -47,7 +47,7 @@ public class ExampleModel {
Map<ApplicationId, ApplicationInfo> applicationInfos = new HashMap<>();
applicationInfos.put(applicationInfo.getApplicationId(), applicationInfo);
- return new SuperModel(applicationInfos);
+ return new SuperModel(applicationInfos, true);
}
public static ApplicationBuilder createApplication(String tenant,
diff --git a/storage/src/tests/distributor/garbagecollectiontest.cpp b/storage/src/tests/distributor/garbagecollectiontest.cpp
index 65c1ac726b5..776cfc14d84 100644
--- a/storage/src/tests/distributor/garbagecollectiontest.cpp
+++ b/storage/src/tests/distributor/garbagecollectiontest.cpp
@@ -3,6 +3,7 @@
#include <vespa/storageapi/message/removelocation.h>
#include <vespa/storage/distributor/operations/idealstate/garbagecollectionoperation.h>
#include <vespa/storage/distributor/idealstatemanager.h>
+#include <vespa/storage/distributor/idealstatemetricsset.h>
#include <tests/distributor/distributortestutil.h>
#include <vespa/storage/distributor/distributor.h>
#include <vespa/document/test/make_document_bucket.h>
@@ -35,11 +36,13 @@ struct GarbageCollectionOperationTest : Test, DistributorTestUtil {
}
// FIXME fragile to assume that send order == node index, but that's the way it currently works
- void reply_to_nth_request(GarbageCollectionOperation& op, size_t n, uint32_t bucket_info_checksum) {
+ void reply_to_nth_request(GarbageCollectionOperation& op, size_t n,
+ uint32_t bucket_info_checksum, uint32_t n_docs_removed) {
auto msg = _sender.command(n);
assert(msg->getType() == api::MessageType::REMOVELOCATION);
std::shared_ptr<api::StorageReply> reply(msg->makeReply());
auto& gc_reply = dynamic_cast<api::RemoveLocationReply&>(*reply);
+ gc_reply.set_documents_removed(n_docs_removed);
gc_reply.setBucketInfo(api::BucketInfo(bucket_info_checksum, 90, 500));
op.receive(_sender, reply);
@@ -56,6 +59,13 @@ struct GarbageCollectionOperationTest : Test, DistributorTestUtil {
<< entry->getNode(i)->getBucketInfo();
}
}
+
+ uint32_t gc_removed_documents_metric() {
+ auto metric_base = getIdealStateManager().getMetrics().operations[IdealStateOperation::GARBAGE_COLLECTION];
+ auto gc_metrics = std::dynamic_pointer_cast<GcMetricSet>(metric_base);
+ assert(gc_metrics);
+ return gc_metrics->documents_removed.getValue();
+ }
};
TEST_F(GarbageCollectionOperationTest, simple) {
@@ -63,29 +73,34 @@ TEST_F(GarbageCollectionOperationTest, simple) {
op->start(_sender, framework::MilliSecTime(0));
ASSERT_EQ(2, _sender.commands().size());
+ EXPECT_EQ(0u, gc_removed_documents_metric());
for (uint32_t i = 0; i < 2; ++i) {
std::shared_ptr<api::StorageCommand> msg = _sender.command(i);
ASSERT_EQ(msg->getType(), api::MessageType::REMOVELOCATION);
auto& tmp = dynamic_cast<api::RemoveLocationCommand&>(*msg);
EXPECT_EQ("music.date < 34", tmp.getDocumentSelection());
- reply_to_nth_request(*op, i, 777 + i);
+ reply_to_nth_request(*op, i, 777 + i, 50);
}
ASSERT_NO_FATAL_FAILURE(assert_bucket_db_contains({api::BucketInfo(777, 90, 500), api::BucketInfo(778, 90, 500)}, 34));
+ EXPECT_EQ(50u, gc_removed_documents_metric());
}
TEST_F(GarbageCollectionOperationTest, replica_bucket_info_not_added_to_db_until_all_replies_received) {
auto op = create_op();
op->start(_sender, framework::MilliSecTime(0));
ASSERT_EQ(2, _sender.commands().size());
+ EXPECT_EQ(0u, gc_removed_documents_metric());
// Respond to 1st request. Should _not_ cause bucket info to be merged into the database yet
- reply_to_nth_request(*op, 0, 1234);
+ reply_to_nth_request(*op, 0, 1234, 70);
ASSERT_NO_FATAL_FAILURE(assert_bucket_db_contains({api::BucketInfo(250, 50, 300), api::BucketInfo(250, 50, 300)}, 0));
// Respond to 2nd request. This _should_ cause bucket info to be merged into the database.
- reply_to_nth_request(*op, 1, 4567);
+ reply_to_nth_request(*op, 1, 4567, 60);
ASSERT_NO_FATAL_FAILURE(assert_bucket_db_contains({api::BucketInfo(1234, 90, 500), api::BucketInfo(4567, 90, 500)}, 34));
+
+ EXPECT_EQ(70u, gc_removed_documents_metric()); // Use max of received metrics
}
TEST_F(GarbageCollectionOperationTest, gc_bucket_info_does_not_overwrite_later_sequenced_bucket_info_writes) {
@@ -93,10 +108,10 @@ TEST_F(GarbageCollectionOperationTest, gc_bucket_info_does_not_overwrite_later_s
op->start(_sender, framework::MilliSecTime(0));
ASSERT_EQ(2, _sender.commands().size());
- reply_to_nth_request(*op, 0, 1234);
+ reply_to_nth_request(*op, 0, 1234, 0);
// Change to replica on node 0 happens after GC op, but before GC info is merged into the DB. Must not be lost.
insertBucketInfo(op->getBucketId(), 0, 7777, 100, 2000);
- reply_to_nth_request(*op, 1, 4567);
+ reply_to_nth_request(*op, 1, 4567, 0);
// Bucket info for node 0 is that of the later sequenced operation, _not_ from the earlier GC op.
ASSERT_NO_FATAL_FAILURE(assert_bucket_db_contains({api::BucketInfo(7777, 100, 2000), api::BucketInfo(4567, 90, 500)}, 34));
}
diff --git a/storage/src/tests/persistence/processalltest.cpp b/storage/src/tests/persistence/processalltest.cpp
index 8c0f8853d2d..83f243ed1b2 100644
--- a/storage/src/tests/persistence/processalltest.cpp
+++ b/storage/src/tests/persistence/processalltest.cpp
@@ -23,11 +23,15 @@ TEST_F(ProcessAllHandlerTest, remove_location) {
api::RemoveLocationCommand removeLocation("id.user == 4", makeDocumentBucket(bucketId));
ProcessAllHandler handler(getEnv(), getPersistenceProvider());
spi::Context context(documentapi::LoadType::DEFAULT, 0, 0);
- handler.handleRemoveLocation(removeLocation, context);
+ auto tracker = handler.handleRemoveLocation(removeLocation, context);
EXPECT_EQ("DocEntry(1234, 1, id:mail:testdoctype1:n=4:3619.html)\n"
"DocEntry(2345, 1, id:mail:testdoctype1:n=4:4008.html)\n",
dumpBucket(bucketId));
+
+ auto reply = std::dynamic_pointer_cast<api::RemoveLocationReply>(tracker->getReply());
+ ASSERT_TRUE(reply.get() != nullptr);
+ EXPECT_EQ(2u, reply->documents_removed());
}
TEST_F(ProcessAllHandlerTest, remove_location_document_subset) {
@@ -44,7 +48,7 @@ TEST_F(ProcessAllHandlerTest, remove_location_document_subset) {
api::RemoveLocationCommand
removeLocation("testdoctype1.headerval % 2 == 0", makeDocumentBucket(bucketId));
spi::Context context(documentapi::LoadType::DEFAULT, 0, 0);
- handler.handleRemoveLocation(removeLocation, context);
+ auto tracker = handler.handleRemoveLocation(removeLocation, context);
EXPECT_EQ("DocEntry(100, 1, id:mail:testdoctype1:n=4:3619.html)\n"
"DocEntry(101, 0, Doc(id:mail:testdoctype1:n=4:33113.html))\n"
@@ -57,6 +61,10 @@ TEST_F(ProcessAllHandlerTest, remove_location_document_subset) {
"DocEntry(108, 1, id:mail:testdoctype1:n=4:42967.html)\n"
"DocEntry(109, 0, Doc(id:mail:testdoctype1:n=4:6925.html))\n",
dumpBucket(bucketId));
+
+ auto reply = std::dynamic_pointer_cast<api::RemoveLocationReply>(tracker->getReply());
+ ASSERT_TRUE(reply.get() != nullptr);
+ EXPECT_EQ(5u, reply->documents_removed());
}
TEST_F(ProcessAllHandlerTest, remove_location_throws_exception_on_unknown_doc_type) {
diff --git a/storage/src/vespa/storage/distributor/bucketdb/bucketdbmetricupdater.cpp b/storage/src/vespa/storage/distributor/bucketdb/bucketdbmetricupdater.cpp
index 61e67b40f44..c211e775326 100644
--- a/storage/src/vespa/storage/distributor/bucketdb/bucketdbmetricupdater.cpp
+++ b/storage/src/vespa/storage/distributor/bucketdb/bucketdbmetricupdater.cpp
@@ -4,8 +4,7 @@
#include <vespa/storage/distributor/distributormetricsset.h>
#include <vespa/storage/distributor/idealstatemetricsset.h>
-namespace storage {
-namespace distributor {
+namespace storage::distributor {
BucketDBMetricUpdater::Stats::Stats()
: _docCount(0),
@@ -27,9 +26,7 @@ BucketDBMetricUpdater::BucketDBMetricUpdater()
{
}
-BucketDBMetricUpdater::~BucketDBMetricUpdater()
-{
-}
+BucketDBMetricUpdater::~BucketDBMetricUpdater() = default;
void
BucketDBMetricUpdater::resetStats()
@@ -148,5 +145,4 @@ BucketDBMetricUpdater::reset()
resetStats();
}
-} // distributor
-} // storage
+} // storage::distributor
diff --git a/storage/src/vespa/storage/distributor/bucketdb/bucketdbmetricupdater.h b/storage/src/vespa/storage/distributor/bucketdb/bucketdbmetricupdater.h
index 6e15ee03d12..7ef8479866f 100644
--- a/storage/src/vespa/storage/distributor/bucketdb/bucketdbmetricupdater.h
+++ b/storage/src/vespa/storage/distributor/bucketdb/bucketdbmetricupdater.h
@@ -7,12 +7,9 @@
#include <unordered_map>
-namespace storage {
+namespace storage::distributor {
class DistributorMetricSet;
-
-namespace distributor {
-
class IdealStateMetricSet;
class BucketDBMetricUpdater {
@@ -107,5 +104,4 @@ private:
void resetStats();
};
-} // distributor
-} // storage
+} // storage::distributor
diff --git a/storage/src/vespa/storage/distributor/distributorinterface.h b/storage/src/vespa/storage/distributor/distributorinterface.h
index aba58e112dc..b17bcd56d19 100644
--- a/storage/src/vespa/storage/distributor/distributorinterface.h
+++ b/storage/src/vespa/storage/distributor/distributorinterface.h
@@ -12,10 +12,10 @@ namespace storage::api { class MergeBucketReply; }
namespace storage::lib { class ClusterStateBundle; }
namespace storage {
class DistributorConfiguration;
- class DistributorMetricSet;
}
namespace storage::distributor {
+class DistributorMetricSet;
class PendingMessageTracker;
class DistributorInterface : public DistributorMessageSender
diff --git a/storage/src/vespa/storage/distributor/distributormetricsset.cpp b/storage/src/vespa/storage/distributor/distributormetricsset.cpp
index 244406ca6fb..98e96f9294f 100644
--- a/storage/src/vespa/storage/distributor/distributormetricsset.cpp
+++ b/storage/src/vespa/storage/distributor/distributormetricsset.cpp
@@ -3,7 +3,7 @@
#include <vespa/metrics/loadmetric.hpp>
#include <vespa/metrics/summetric.hpp>
-namespace storage {
+namespace storage::distributor {
using metrics::MetricSet;
diff --git a/storage/src/vespa/storage/distributor/distributormetricsset.h b/storage/src/vespa/storage/distributor/distributormetricsset.h
index 1e4730b8de6..b5be72e8c14 100644
--- a/storage/src/vespa/storage/distributor/distributormetricsset.h
+++ b/storage/src/vespa/storage/distributor/distributormetricsset.h
@@ -7,7 +7,7 @@
#include <vespa/metrics/metrics.h>
#include <vespa/documentapi/loadtypes/loadtypeset.h>
-namespace storage {
+namespace storage::distributor {
class DistributorMetricSet : public metrics::MetricSet
{
diff --git a/storage/src/vespa/storage/distributor/externaloperationhandler.h b/storage/src/vespa/storage/distributor/externaloperationhandler.h
index 96875a3644a..60cad15a791 100644
--- a/storage/src/vespa/storage/distributor/externaloperationhandler.h
+++ b/storage/src/vespa/storage/distributor/externaloperationhandler.h
@@ -13,11 +13,11 @@
namespace storage {
-class DistributorMetricSet;
class PersistenceOperationMetricSet;
namespace distributor {
+class DistributorMetricSet;
class Distributor;
class MaintenanceOperationGenerator;
class DirectDispatchSender;
diff --git a/storage/src/vespa/storage/distributor/idealstatemetricsset.cpp b/storage/src/vespa/storage/distributor/idealstatemetricsset.cpp
index d72f4a80ef4..fd193ad6fd8 100644
--- a/storage/src/vespa/storage/distributor/idealstatemetricsset.cpp
+++ b/storage/src/vespa/storage/distributor/idealstatemetricsset.cpp
@@ -6,7 +6,7 @@ namespace storage {
namespace distributor {
OperationMetricSet::OperationMetricSet(const std::string& name, metrics::Metric::Tags tags, const std::string& description, MetricSet* owner)
- : MetricSet(name, tags, description, owner),
+ : MetricSet(name, std::move(tags), description, owner),
pending("pending",
{{"logdefault"},{"yamasdefault"}},
"The number of operations pending", this),
@@ -16,14 +16,25 @@ OperationMetricSet::OperationMetricSet(const std::string& name, metrics::Metric:
failed("done_failed",
{{"logdefault"},{"yamasdefault"}},
"The number of operations that failed", this)
-{ }
+{}
-OperationMetricSet::~OperationMetricSet() { }
+OperationMetricSet::~OperationMetricSet() = default;
+
+GcMetricSet::GcMetricSet(const std::string& name, metrics::Metric::Tags tags, const std::string& description, MetricSet* owner)
+ : OperationMetricSet(name, std::move(tags), description, owner),
+ documents_removed("documents_removed",
+ {{"logdefault"},{"yamasdefault"}},
+ "Number of documents removed by GC operations", this)
+{}
+
+GcMetricSet::~GcMetricSet() = default;
void
IdealStateMetricSet::createOperationMetrics() {
typedef IdealStateOperation ISO;
operations.resize(ISO::OPERATION_COUNT);
+ // Note: naked new is used instead of make_shared due to the latter not being
+ // able to properly transitively deduce the types for the tag initializer lists.
operations[ISO::DELETE_BUCKET] = std::shared_ptr<OperationMetricSet>(
new OperationMetricSet("delete_bucket",
{{"logdefault"},{"yamasdefault"}},
@@ -45,9 +56,9 @@ IdealStateMetricSet::createOperationMetrics() {
{{"logdefault"},{"yamasdefault"}},
"Operations to set active/ready state for bucket copies", this));
operations[ISO::GARBAGE_COLLECTION] = std::shared_ptr<OperationMetricSet>(
- new OperationMetricSet("garbage_collection",
- {{"logdefault"},{"yamasdefault"}},
- "Operations to garbage collect data from buckets", this));
+ new GcMetricSet("garbage_collection",
+ {{"logdefault"},{"yamasdefault"}},
+ "Operations to garbage collect data from buckets", this));
}
IdealStateMetricSet::IdealStateMetricSet()
@@ -81,7 +92,7 @@ IdealStateMetricSet::IdealStateMetricSet()
createOperationMetrics();
}
-IdealStateMetricSet::~IdealStateMetricSet() { }
+IdealStateMetricSet::~IdealStateMetricSet() = default;
void IdealStateMetricSet::setPendingOperations(const std::vector<uint64_t>& newMetrics) {
for (uint32_t i = 0; i < IdealStateOperation::OPERATION_COUNT; i++) {
diff --git a/storage/src/vespa/storage/distributor/idealstatemetricsset.h b/storage/src/vespa/storage/distributor/idealstatemetricsset.h
index 7bb472b4a2c..2679da17598 100644
--- a/storage/src/vespa/storage/distributor/idealstatemetricsset.h
+++ b/storage/src/vespa/storage/distributor/idealstatemetricsset.h
@@ -16,13 +16,21 @@ public:
metrics::LongCountMetric failed;
OperationMetricSet(const std::string& name, metrics::Metric::Tags tags, const std::string& description, MetricSet* owner);
- ~OperationMetricSet();
+ ~OperationMetricSet() override;
+};
+
+struct GcMetricSet : OperationMetricSet {
+ metrics::LongCountMetric documents_removed;
+
+ GcMetricSet(const std::string& name, metrics::Metric::Tags tags,
+ const std::string& description, MetricSet* owner);
+ ~GcMetricSet() override;
};
class IdealStateMetricSet : public metrics::MetricSet
{
public:
- std::vector<std::shared_ptr<OperationMetricSet> > operations;
+ std::vector<std::shared_ptr<OperationMetricSet>> operations;
metrics::LongValueMetric idealstate_diff;
metrics::LongValueMetric buckets_toofewcopies;
metrics::LongValueMetric buckets_toomanycopies;
@@ -35,7 +43,7 @@ public:
void createOperationMetrics();
IdealStateMetricSet();
- ~IdealStateMetricSet();
+ ~IdealStateMetricSet() override;
void setPendingOperations(const std::vector<uint64_t>& newMetrics);
};
diff --git a/storage/src/vespa/storage/distributor/operations/idealstate/garbagecollectionoperation.cpp b/storage/src/vespa/storage/distributor/operations/idealstate/garbagecollectionoperation.cpp
index c674add80f7..fc127c2e0eb 100644
--- a/storage/src/vespa/storage/distributor/operations/idealstate/garbagecollectionoperation.cpp
+++ b/storage/src/vespa/storage/distributor/operations/idealstate/garbagecollectionoperation.cpp
@@ -2,6 +2,7 @@
#include "garbagecollectionoperation.h"
#include <vespa/storage/distributor/idealstatemanager.h>
+#include <vespa/storage/distributor/idealstatemetricsset.h>
#include <vespa/storage/distributor/distributor.h>
#include <vespa/storage/distributor/distributor_bucket_space.h>
#include <vespa/storageapi/message/removelocation.h>
@@ -9,19 +10,18 @@
#include <vespa/log/log.h>
LOG_SETUP(".distributor.operation.idealstate.remove");
-using namespace storage::distributor;
+namespace storage::distributor {
GarbageCollectionOperation::GarbageCollectionOperation(const std::string& clusterName, const BucketAndNodes& nodes)
: IdealStateOperation(nodes),
_tracker(clusterName),
- _replica_info()
+ _replica_info(),
+ _max_documents_removed(0)
{}
GarbageCollectionOperation::~GarbageCollectionOperation() = default;
-void
-GarbageCollectionOperation::onStart(DistributorMessageSender& sender)
-{
+void GarbageCollectionOperation::onStart(DistributorMessageSender& sender) {
BucketDatabase::Entry entry = _bucketSpace->getBucketDatabase().get(getBucketId());
std::vector<uint16_t> nodes = entry->getNodes();
@@ -43,7 +43,7 @@ GarbageCollectionOperation::onStart(DistributorMessageSender& sender)
void
GarbageCollectionOperation::onReceive(DistributorMessageSender&,
- const std::shared_ptr<api::StorageReply>& reply)
+ const std::shared_ptr<api::StorageReply>& reply)
{
auto* rep = dynamic_cast<api::RemoveLocationReply*>(reply.get());
assert(rep != nullptr);
@@ -53,6 +53,7 @@ GarbageCollectionOperation::onReceive(DistributorMessageSender&,
if (!rep->getResult().failed()) {
_replica_info.emplace_back(_manager->getDistributorComponent().getUniqueTimestamp(),
node, rep->getBucketInfo());
+ _max_documents_removed = std::max(_max_documents_removed, rep->documents_removed());
} else {
_ok = false;
}
@@ -61,6 +62,7 @@ GarbageCollectionOperation::onReceive(DistributorMessageSender&,
if (_ok) {
merge_received_bucket_info_into_db();
}
+ update_gc_metrics();
done();
}
}
@@ -76,8 +78,16 @@ void GarbageCollectionOperation::merge_received_bucket_info_into_db() {
}
}
+void GarbageCollectionOperation::update_gc_metrics() {
+ auto metric_base = _manager->getMetrics().operations[IdealStateOperation::GARBAGE_COLLECTION];
+ auto gc_metrics = std::dynamic_pointer_cast<GcMetricSet>(metric_base);
+ assert(gc_metrics);
+ gc_metrics->documents_removed.inc(_max_documents_removed);
+}
+
bool
-GarbageCollectionOperation::shouldBlockThisOperation(uint32_t, uint8_t) const
-{
+GarbageCollectionOperation::shouldBlockThisOperation(uint32_t, uint8_t) const {
return true;
}
+
+}
diff --git a/storage/src/vespa/storage/distributor/operations/idealstate/garbagecollectionoperation.h b/storage/src/vespa/storage/distributor/operations/idealstate/garbagecollectionoperation.h
index 47ea11bb328..28de9592a63 100644
--- a/storage/src/vespa/storage/distributor/operations/idealstate/garbagecollectionoperation.h
+++ b/storage/src/vespa/storage/distributor/operations/idealstate/garbagecollectionoperation.h
@@ -26,8 +26,10 @@ protected:
MessageTracker _tracker;
private:
std::vector<BucketCopy> _replica_info;
+ uint32_t _max_documents_removed;
void merge_received_bucket_info_into_db();
+ void update_gc_metrics();
};
}
diff --git a/storage/src/vespa/storage/distributor/persistence_operation_metric_set.h b/storage/src/vespa/storage/distributor/persistence_operation_metric_set.h
index d951d7ceba2..1299fdad2ad 100644
--- a/storage/src/vespa/storage/distributor/persistence_operation_metric_set.h
+++ b/storage/src/vespa/storage/distributor/persistence_operation_metric_set.h
@@ -16,7 +16,7 @@ class PersistenceFailuresMetricSet : public metrics::MetricSet
{
public:
explicit PersistenceFailuresMetricSet(metrics::MetricSet* owner);
- ~PersistenceFailuresMetricSet();
+ ~PersistenceFailuresMetricSet() override;
metrics::SumMetric<metrics::LongCountMetric> sum;
metrics::LongCountMetric notready;
@@ -44,7 +44,7 @@ public:
PersistenceFailuresMetricSet failures;
PersistenceOperationMetricSet(const std::string& name, metrics::MetricSet* owner = nullptr);
- ~PersistenceOperationMetricSet();
+ ~PersistenceOperationMetricSet() override;
MetricSet * clone(std::vector<Metric::UP>& ownerList, CopyType copyType,
metrics::MetricSet* owner, bool includeUnused) const override;
diff --git a/storage/src/vespa/storage/persistence/processallhandler.cpp b/storage/src/vespa/storage/persistence/processallhandler.cpp
index 8c951a9f50d..5b94a3da027 100644
--- a/storage/src/vespa/storage/persistence/processallhandler.cpp
+++ b/storage/src/vespa/storage/persistence/processallhandler.cpp
@@ -23,6 +23,7 @@ public:
spi::PersistenceProvider& _provider;
const spi::Bucket& _bucket;
spi::Context& _context;
+ uint32_t _n_removed;
UnrevertableRemoveEntryProcessor(
spi::PersistenceProvider& provider,
@@ -30,7 +31,9 @@ public:
spi::Context& context)
: _provider(provider),
_bucket(bucket),
- _context(context) {}
+ _context(context),
+ _n_removed(0)
+ {}
void process(spi::DocEntry& entry) override {
spi::RemoveResult removeResult = _provider.remove(
@@ -45,13 +48,14 @@ public:
<< removeResult.getErrorMessage();
throw std::runtime_error(ss.str());
}
+ ++_n_removed;
}
};
class StatEntryProcessor : public BucketProcessor::EntryProcessor {
public:
std::ostream& ost;
- StatEntryProcessor(std::ostream& o)
+ explicit StatEntryProcessor(std::ostream& o)
: ost(o) {};
void process(spi::DocEntry& e) override {
@@ -97,7 +101,9 @@ ProcessAllHandler::handleRemoveLocation(api::RemoveLocationCommand& cmd,
context);
spi::Result result = _spi.flush(bucket, context);
uint32_t code = _env.convertErrorCode(result);
- if (code != 0) {
+ if (code == 0) {
+ tracker->setReply(std::make_shared<api::RemoveLocationReply>(cmd, processor._n_removed));
+ } else {
tracker->fail(code, result.getErrorMessage());
}
diff --git a/storageapi/src/tests/mbusprot/storageprotocoltest.cpp b/storageapi/src/tests/mbusprot/storageprotocoltest.cpp
index 2f959e40e2a..2e5eb115844 100644
--- a/storageapi/src/tests/mbusprot/storageprotocoltest.cpp
+++ b/storageapi/src/tests/mbusprot/storageprotocoltest.cpp
@@ -522,8 +522,15 @@ TEST_P(StorageProtocolTest, remove_location) {
EXPECT_EQ("id.group == \"mygroup\"", cmd2->getDocumentSelection());
EXPECT_EQ(_bucket, cmd2->getBucket());
- auto reply = std::make_shared<RemoveLocationReply>(*cmd2);
+ uint32_t n_docs_removed = 12345;
+ auto reply = std::make_shared<RemoveLocationReply>(*cmd2, n_docs_removed);
auto reply2 = copyReply(reply);
+ if (GetParam().getMajor() == 7) {
+ // Statistics are only available for protobuf-enabled version.
+ EXPECT_EQ(n_docs_removed, reply2->documents_removed());
+ } else {
+ EXPECT_EQ(0, reply2->documents_removed());
+ }
}
TEST_P(StorageProtocolTest, create_visitor) {
diff --git a/storageapi/src/vespa/storageapi/mbusprot/protobuf/feed.proto b/storageapi/src/vespa/storageapi/mbusprot/protobuf/feed.proto
index 810f88f588f..12dbaf59146 100644
--- a/storageapi/src/vespa/storageapi/mbusprot/protobuf/feed.proto
+++ b/storageapi/src/vespa/storageapi/mbusprot/protobuf/feed.proto
@@ -90,7 +90,12 @@ message RemoveLocationRequest {
bytes document_selection = 2;
}
+message RemoveLocationStats {
+ uint32 documents_removed = 1;
+}
+
message RemoveLocationResponse {
BucketInfo bucket_info = 1;
BucketId remapped_bucket_id = 2;
+ RemoveLocationStats stats = 3;
}
diff --git a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization7.cpp b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization7.cpp
index 9751fd1be98..90c8d1c7d2a 100644
--- a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization7.cpp
+++ b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization7.cpp
@@ -643,7 +643,9 @@ void ProtocolSerialization7::onEncode(GBBuf& buf, const api::RemoveLocationComma
}
void ProtocolSerialization7::onEncode(GBBuf& buf, const api::RemoveLocationReply& msg) const {
- encode_bucket_info_response<protobuf::RemoveLocationResponse>(buf, msg, no_op_encode);
+ encode_bucket_info_response<protobuf::RemoveLocationResponse>(buf, msg, [&](auto& res) {
+ res.mutable_stats()->set_documents_removed(msg.documents_removed());
+ });
}
api::StorageCommand::UP ProtocolSerialization7::onDecodeRemoveLocationCommand(BBuf& buf) const {
@@ -653,8 +655,11 @@ api::StorageCommand::UP ProtocolSerialization7::onDecodeRemoveLocationCommand(BB
}
api::StorageReply::UP ProtocolSerialization7::onDecodeRemoveLocationReply(const SCmd& cmd, BBuf& buf) const {
- return decode_bucket_info_response<protobuf::RemoveLocationResponse>(buf, [&]([[maybe_unused]] auto& res) {
- return std::make_unique<api::RemoveLocationReply>(static_cast<const api::RemoveLocationCommand&>(cmd));
+ return decode_bucket_info_response<protobuf::RemoveLocationResponse>(buf, [&](auto& res) {
+ uint32_t documents_removed = (res.has_stats() ? res.stats().documents_removed() : 0u);
+ return std::make_unique<api::RemoveLocationReply>(
+ static_cast<const api::RemoveLocationCommand&>(cmd),
+ documents_removed);
});
}
diff --git a/storageapi/src/vespa/storageapi/message/removelocation.cpp b/storageapi/src/vespa/storageapi/message/removelocation.cpp
index b53584601ef..49c9d22f5ee 100644
--- a/storageapi/src/vespa/storageapi/message/removelocation.cpp
+++ b/storageapi/src/vespa/storageapi/message/removelocation.cpp
@@ -25,8 +25,9 @@ RemoveLocationCommand::print(std::ostream& out, bool verbose, const std::string&
BucketInfoCommand::print(out, verbose, indent);
}
-RemoveLocationReply::RemoveLocationReply(const RemoveLocationCommand& cmd)
- : BucketInfoReply(cmd)
+RemoveLocationReply::RemoveLocationReply(const RemoveLocationCommand& cmd, uint32_t docs_removed)
+ : BucketInfoReply(cmd),
+ _documents_removed(docs_removed)
{
}
diff --git a/storageapi/src/vespa/storageapi/message/removelocation.h b/storageapi/src/vespa/storageapi/message/removelocation.h
index 46555497035..812cc8c413b 100644
--- a/storageapi/src/vespa/storageapi/message/removelocation.h
+++ b/storageapi/src/vespa/storageapi/message/removelocation.h
@@ -11,7 +11,7 @@ class RemoveLocationCommand : public BucketInfoCommand
{
public:
RemoveLocationCommand(vespalib::stringref documentSelection, const document::Bucket &bucket);
- ~RemoveLocationCommand();
+ ~RemoveLocationCommand() override;
void print(std::ostream& out, bool verbose, const std::string& indent) const override;
const vespalib::string& getDocumentSelection() const { return _documentSelection; }
@@ -22,8 +22,13 @@ private:
class RemoveLocationReply : public BucketInfoReply
{
+ uint32_t _documents_removed;
public:
- RemoveLocationReply(const RemoveLocationCommand& cmd);
+ explicit RemoveLocationReply(const RemoveLocationCommand& cmd, uint32_t docs_removed = 0);
+ void set_documents_removed(uint32_t docs_removed) noexcept {
+ _documents_removed = docs_removed;
+ }
+ uint32_t documents_removed() const noexcept { return _documents_removed; }
DECLARE_STORAGEREPLY(RemoveLocationReply, onRemoveLocationReply)
};
diff --git a/vbench/src/vbench/vbench/vbench.cpp b/vbench/src/vbench/vbench/vbench.cpp
index 4f6efadfbdd..58854af705e 100644
--- a/vbench/src/vbench/vbench/vbench.cpp
+++ b/vbench/src/vbench/vbench/vbench.cpp
@@ -29,11 +29,13 @@ CryptoEngine::SP setup_crypto(const vespalib::slime::Inspector &tls) {
if (!tls.valid()) {
return std::make_shared<vespalib::NullCryptoEngine>();
}
- vespalib::net::tls::TransportSecurityOptions
- tls_opts(maybe_load(tls["ca-certificates"]),
- maybe_load(tls["certificates"]),
- maybe_load(tls["private-key"]));
- return std::make_shared<vespalib::TlsCryptoEngine>(tls_opts);
+ auto ts_builder = vespalib::net::tls::TransportSecurityOptions::Params().
+ ca_certs_pem(maybe_load(tls["ca-certificates"])).
+ cert_chain_pem(maybe_load(tls["certificates"])).
+ private_key_pem(maybe_load(tls["private-key"])).
+ authorized_peers(vespalib::net::tls::AuthorizedPeers::allow_all_authenticated()).
+ disable_hostname_validation(true); // TODO configurable or default false!
+ return std::make_shared<vespalib::TlsCryptoEngine>(vespalib::net::tls::TransportSecurityOptions(std::move(ts_builder)));
}
} // namespace vbench::<unnamed>
diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/runner/Runner.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/runner/Runner.java
index 0e202d1f348..59953fbe002 100644
--- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/runner/Runner.java
+++ b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/runner/Runner.java
@@ -70,7 +70,6 @@ public class Runner {
Optional.ofNullable(commandLineArgs.getFile()),
commandLineArgs.getAddRootElementToXml());
-
int intervalOfLogging = commandLineArgs.getVerbose()
? commandLineArgs.getWhenVerboseEnabledPrintMessageForEveryXDocuments()
: Integer.MAX_VALUE;
@@ -86,13 +85,15 @@ public class Runner {
if (commandLineArgs.getVerbose()) {
System.err.println(feedClient.getStatsAsJson());
- double fileSizeMb = ((double) new File(commandLineArgs.getFile()).length()) / 1024.0 / 1024.0;
double transferTimeSec = ((double) sendTotalTimeMs) / 1000.0;
- System.err.println("Sent " + fileSizeMb + " MB in " + transferTimeSec + " seconds.");
- System.err.println("Speed: " + ((fileSizeMb / transferTimeSec) * 8.0) + " Mbits/sec, + HTTP overhead " +
- "(not taking compression into account)");
if (transferTimeSec > 0) {
- System.err.printf("Docs/sec %.3f%n\n", numSent.get() / transferTimeSec);
+ System.err.printf("Docs/sec %.3f%n", numSent.get() / transferTimeSec);
+ }
+ if (commandLineArgs.getFile() != null) {
+ double fileSizeMb = ((double) new File(commandLineArgs.getFile()).length()) / 1024.0 / 1024.0;
+ System.err.println("Sent " + fileSizeMb + " MB in " + transferTimeSec + " seconds.");
+ System.err.println("Speed: " + ((fileSizeMb / transferTimeSec) * 8.0) + " Mbits/sec, + HTTP overhead " +
+ "(not taking compression into account)");
}
}
callback.printProgress();
diff --git a/vespa-testrunner-components/src/main/java/com/yahoo/vespa/hosted/testrunner/TestRunnerHandler.java b/vespa-testrunner-components/src/main/java/com/yahoo/vespa/hosted/testrunner/TestRunnerHandler.java
index ed079442440..90f7a76b356 100644
--- a/vespa-testrunner-components/src/main/java/com/yahoo/vespa/hosted/testrunner/TestRunnerHandler.java
+++ b/vespa-testrunner-components/src/main/java/com/yahoo/vespa/hosted/testrunner/TestRunnerHandler.java
@@ -61,8 +61,7 @@ public class TestRunnerHandler extends LoggingRequestHandler {
private HttpResponse handleGET(HttpRequest request) {
String path = request.getUri().getPath();
- // TODO: Migrate to /tester/v1/log when /tester/v1/log2 is not in use anymore (and remove /tester/v1/log2)
- if (path.equals("/tester/v1/log") || path.equals("/tester/v1/log2")) {
+ if (path.equals("/tester/v1/log")) {
return new SlimeJsonResponse(logToSlime(testRunner.getLog(request.hasProperty("after")
? Long.parseLong(request.getProperty("after"))
: -1)));
@@ -91,7 +90,7 @@ public class TestRunnerHandler extends LoggingRequestHandler {
path = path.substring(0, path.length() - 1);
int lastSlash = path.lastIndexOf("/");
if (lastSlash < 0) return path;
- return path.substring(lastSlash + 1, path.length());
+ return path.substring(lastSlash + 1);
}
static Slime logToSlime(Collection<LogRecord> log) {
diff --git a/vespabase/src/rhel-prestart.sh b/vespabase/src/rhel-prestart.sh
index 3a770cdd785..d84122898a3 100755
--- a/vespabase/src/rhel-prestart.sh
+++ b/vespabase/src/rhel-prestart.sh
@@ -96,32 +96,32 @@ fixdir () {
# BEGIN directory fixups
-fixdir root wheel 1777 logs
-fixdir root wheel 1777 tmp
-fixdir root wheel 1777 var/run
-fixdir ${VESPA_USER} wheel 1777 var/crash
-fixdir ${VESPA_USER} wheel 1777 logs/vespa
-fixdir ${VESPA_USER} wheel 1777 tmp/vespa
-fixdir ${VESPA_USER} wheel 755 var
-fixdir ${VESPA_USER} wheel 755 libexec/vespa/plugins/qrs
-fixdir ${VESPA_USER} wheel 755 logs/vespa/configserver
-fixdir ${VESPA_USER} wheel 755 logs/vespa/qrs
-fixdir ${VESPA_USER} wheel 755 logs/vespa/search
-fixdir ${VESPA_USER} wheel 755 var/db/vespa
-fixdir ${VESPA_USER} wheel 755 var/db/vespa/tmp
-fixdir ${VESPA_USER} wheel 755 var/db/vespa/config_server
-fixdir ${VESPA_USER} wheel 755 var/db/vespa/config_server/serverdb
-fixdir ${VESPA_USER} wheel 755 var/db/vespa/config_server/serverdb/tenants
-fixdir ${VESPA_USER} wheel 755 var/db/vespa/filedistribution
-fixdir ${VESPA_USER} wheel 755 var/db/vespa/index
-fixdir ${VESPA_USER} wheel 755 var/db/vespa/logcontrol
-fixdir ${VESPA_USER} wheel 755 var/db/vespa/search
-fixdir ${VESPA_USER} wheel 755 var/jdisc_container
-fixdir ${VESPA_USER} wheel 755 var/vespa
-fixdir ${VESPA_USER} wheel 755 var/vespa/application
-fixdir ${VESPA_USER} wheel 755 var/vespa/bundlecache
-fixdir ${VESPA_USER} wheel 755 var/vespa/bundlecache/configserver
-fixdir ${VESPA_USER} wheel 755 var/vespa/cache/config/
+fixdir root root 1777 logs
+fixdir root root 1777 tmp
+fixdir root root 1777 var/run
+fixdir ${VESPA_USER} root 1777 var/crash
+fixdir ${VESPA_USER} root 1777 logs/vespa
+fixdir ${VESPA_USER} root 1777 tmp/vespa
+fixdir root root 755 var
+fixdir ${VESPA_USER} root 755 libexec/vespa/plugins/qrs
+fixdir ${VESPA_USER} root 755 logs/vespa/configserver
+fixdir ${VESPA_USER} root 755 logs/vespa/qrs
+fixdir ${VESPA_USER} root 755 logs/vespa/search
+fixdir ${VESPA_USER} root 755 var/db/vespa
+fixdir ${VESPA_USER} root 755 var/db/vespa/tmp
+fixdir ${VESPA_USER} root 755 var/db/vespa/config_server
+fixdir ${VESPA_USER} root 755 var/db/vespa/config_server/serverdb
+fixdir ${VESPA_USER} root 755 var/db/vespa/config_server/serverdb/tenants
+fixdir ${VESPA_USER} root 755 var/db/vespa/filedistribution
+fixdir ${VESPA_USER} root 755 var/db/vespa/index
+fixdir ${VESPA_USER} root 755 var/db/vespa/logcontrol
+fixdir ${VESPA_USER} root 755 var/db/vespa/search
+fixdir ${VESPA_USER} root 755 var/jdisc_container
+fixdir ${VESPA_USER} root 755 var/vespa
+fixdir ${VESPA_USER} root 755 var/vespa/application
+fixdir ${VESPA_USER} root 755 var/vespa/bundlecache
+fixdir ${VESPA_USER} root 755 var/vespa/bundlecache/configserver
+fixdir ${VESPA_USER} root 755 var/vespa/cache/config/
if [ "${VESPA_UNPRIVILEGED}" != yes ]; then
chown -hR ${VESPA_USER} logs/vespa
diff --git a/vespaclient/src/perl/lib/Yahoo/Vespa/Http.pm b/vespaclient/src/perl/lib/Yahoo/Vespa/Http.pm
index 2dbf475f2a7..d907e89fa54 100644
--- a/vespaclient/src/perl/lib/Yahoo/Vespa/Http.pm
+++ b/vespaclient/src/perl/lib/Yahoo/Vespa/Http.pm
@@ -100,7 +100,10 @@ sub initialize { # ()
my $tls_enabled = $ENV{'VESPA_TLS_ENABLED'};
if (defined $tls_enabled and $tls_enabled eq '1') {
$BROWSER->ssl_opts( SSL_version => 'TLSv12');
- $BROWSER->ssl_opts( verify_hostname => 0);
+ my $hostname_verification_disabled = $ENV{'VESPA_TLS_HOSTNAME_VALIDATION_DISABLED'};
+ if (defined $hostname_verification_disabled and $hostname_verification_disabled eq '1') {
+ $BROWSER->ssl_opts( verify_hostname => 0);
+ }
$BROWSER->ssl_opts( SSL_cipher_list => 'ECDHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-CHACHA20-POLY1305:TLS13-AES-128-GCM-SHA256:TLS13-AES-256-GCM-SHA384:TLS13-CHACHA20-POLY1305-SHA256' );
}
if (defined $ENV{'VESPA_TLS_CA_CERT'}) {
diff --git a/vespalib/src/tests/net/tls/openssl_impl/openssl_impl_test.cpp b/vespalib/src/tests/net/tls/openssl_impl/openssl_impl_test.cpp
index 78838ce2cd2..54c8c19fc64 100644
--- a/vespalib/src/tests/net/tls/openssl_impl/openssl_impl_test.cpp
+++ b/vespalib/src/tests/net/tls/openssl_impl/openssl_impl_test.cpp
@@ -57,11 +57,23 @@ void print_decode_result(const char* mode, const DecodeResult& res) {
decode_state_to_str(res.state));
}
+TransportSecurityOptions ts_from_pems(vespalib::stringref ca_certs_pem,
+ vespalib::stringref cert_chain_pem,
+ vespalib::stringref private_key_pem)
+{
+ auto ts_builder = TransportSecurityOptions::Params().
+ ca_certs_pem(ca_certs_pem).
+ cert_chain_pem(cert_chain_pem).
+ private_key_pem(private_key_pem).
+ authorized_peers(AuthorizedPeers::allow_all_authenticated());
+ return TransportSecurityOptions(std::move(ts_builder));
+}
+
struct Fixture {
TransportSecurityOptions tls_opts;
std::shared_ptr<TlsContext> tls_ctx;
- std::unique_ptr<CryptoCodec> client;
- std::unique_ptr<CryptoCodec> server;
+ std::unique_ptr<OpenSslCryptoCodecImpl> client;
+ std::unique_ptr<OpenSslCryptoCodecImpl> server;
SmartBuffer client_to_server;
SmartBuffer server_to_client;
@@ -77,16 +89,21 @@ struct Fixture {
static TransportSecurityOptions create_options_without_own_peer_cert() {
auto source_opts = vespalib::test::make_tls_options_for_testing();
- return TransportSecurityOptions(source_opts.ca_certs_pem(), "", "");
+ return ts_from_pems(source_opts.ca_certs_pem(), "", "");
}
- static std::unique_ptr<CryptoCodec> create_openssl_codec(
- const TransportSecurityOptions& opts, CryptoCodec::Mode mode) {
+ static std::unique_ptr<OpenSslCryptoCodecImpl> create_openssl_codec(
+ const TransportSecurityOptions& opts, CryptoCodec::Mode mode, const SocketSpec& peer_spec) {
auto ctx = TlsContext::create_default_context(opts, AuthorizationMode::Enforce);
- return create_openssl_codec(ctx, mode);
+ return create_openssl_codec(ctx, mode, peer_spec);
+ }
+
+ static std::unique_ptr<OpenSslCryptoCodecImpl> create_openssl_codec(
+ const TransportSecurityOptions& opts, CryptoCodec::Mode mode) {
+ return create_openssl_codec(opts, mode, SocketSpec::invalid);
}
- static std::unique_ptr<CryptoCodec> create_openssl_codec(
+ static std::unique_ptr<OpenSslCryptoCodecImpl> create_openssl_codec(
const TransportSecurityOptions& opts,
std::shared_ptr<CertificateVerificationCallback> cert_verify_callback,
CryptoCodec::Mode mode) {
@@ -94,21 +111,30 @@ struct Fixture {
return create_openssl_codec(ctx, mode);
}
- static std::unique_ptr<CryptoCodec> create_openssl_codec(
- const std::shared_ptr<TlsContext>& ctx, CryptoCodec::Mode mode) {
+ static std::unique_ptr<OpenSslCryptoCodecImpl> create_openssl_codec(
+ const std::shared_ptr<TlsContext>& ctx, CryptoCodec::Mode mode, const SocketSpec& peer_spec) {
auto ctx_impl = std::dynamic_pointer_cast<impl::OpenSslTlsContextImpl>(ctx);
- return std::make_unique<impl::OpenSslCryptoCodecImpl>(std::move(ctx_impl), SocketAddress(), mode);
+ if (mode == CryptoCodec::Mode::Client) {
+ return OpenSslCryptoCodecImpl::make_client_codec(std::move(ctx_impl), peer_spec, SocketAddress());
+ } else {
+ return OpenSslCryptoCodecImpl::make_server_codec(std::move(ctx_impl), SocketAddress());
+ }
}
- EncodeResult do_encode(CryptoCodec& codec, Output& buffer, vespalib::stringref plaintext) {
+ static std::unique_ptr<OpenSslCryptoCodecImpl> create_openssl_codec(
+ const std::shared_ptr<TlsContext>& ctx, CryptoCodec::Mode mode) {
+ return create_openssl_codec(ctx, mode, SocketSpec::invalid);
+ }
+
+ static EncodeResult do_encode(CryptoCodec& codec, Output& buffer, vespalib::stringref plaintext) {
auto out = buffer.reserve(codec.min_encode_buffer_size());
auto enc_res = codec.encode(plaintext.data(), plaintext.size(), out.data, out.size);
buffer.commit(enc_res.bytes_produced);
return enc_res;
}
- DecodeResult do_decode(CryptoCodec& codec, Input& buffer, vespalib::string& out,
- size_t max_bytes_produced, size_t max_bytes_consumed) {
+ static DecodeResult do_decode(CryptoCodec& codec, Input& buffer, vespalib::string& out,
+ size_t max_bytes_produced, size_t max_bytes_consumed) {
auto in = buffer.obtain();
out.resize(max_bytes_produced);
auto to_consume = std::min(in.size, max_bytes_consumed);
@@ -382,13 +408,13 @@ l9pLv1vrujrPEC78cyIQe2x55wf3pRoaDg==
-----END EC PRIVATE KEY-----)";
TEST_F("client with certificate signed by untrusted CA is rejected by server", Fixture) {
- TransportSecurityOptions client_opts(unknown_ca_pem, untrusted_host_cert_pem, untrusted_host_key_pem);
+ auto client_opts = ts_from_pems(unknown_ca_pem, untrusted_host_cert_pem, untrusted_host_key_pem);
f.client = f.create_openssl_codec(client_opts, CryptoCodec::Mode::Client);
EXPECT_FALSE(f.handshake());
}
TEST_F("server with certificate signed by untrusted CA is rejected by client", Fixture) {
- TransportSecurityOptions server_opts(unknown_ca_pem, untrusted_host_cert_pem, untrusted_host_key_pem);
+ auto server_opts = ts_from_pems(unknown_ca_pem, untrusted_host_cert_pem, untrusted_host_key_pem);
f.server = f.create_openssl_codec(server_opts, CryptoCodec::Mode::Server);
EXPECT_FALSE(f.handshake());
}
@@ -396,8 +422,8 @@ TEST_F("server with certificate signed by untrusted CA is rejected by client", F
TEST_F("Can specify multiple trusted CA certs in transport options", Fixture) {
auto& base_opts = f.tls_opts;
auto multi_ca_pem = base_opts.ca_certs_pem() + "\n" + unknown_ca_pem;
- TransportSecurityOptions multi_ca_using_ca_1(multi_ca_pem, untrusted_host_cert_pem, untrusted_host_key_pem);
- TransportSecurityOptions multi_ca_using_ca_2(multi_ca_pem, base_opts.cert_chain_pem(), base_opts.private_key_pem());
+ auto multi_ca_using_ca_1 = ts_from_pems(multi_ca_pem, untrusted_host_cert_pem, untrusted_host_key_pem);
+ auto multi_ca_using_ca_2 = ts_from_pems(multi_ca_pem, base_opts.cert_chain_pem(), base_opts.private_key_pem());
// Let client be signed by CA 1, server by CA 2. Both have the two CAs in their trust store
// so this should allow for a successful handshake.
f.client = f.create_openssl_codec(multi_ca_using_ca_1, CryptoCodec::Mode::Client);
@@ -446,7 +472,7 @@ struct CertFixture : Fixture {
return {std::move(cert), std::move(key)};
}
- static std::unique_ptr<CryptoCodec> create_openssl_codec_with_authz_mode(
+ static std::unique_ptr<OpenSslCryptoCodecImpl> create_openssl_codec_with_authz_mode(
const TransportSecurityOptions& opts,
std::shared_ptr<CertificateVerificationCallback> cert_verify_callback,
CryptoCodec::Mode codec_mode,
@@ -455,33 +481,52 @@ struct CertFixture : Fixture {
return create_openssl_codec(ctx, codec_mode);
}
+ TransportSecurityOptions::Params ts_builder_from(const CertKeyWrapper& ck) const {
+ return TransportSecurityOptions::Params().
+ ca_certs_pem(root_ca.cert->to_pem()).
+ cert_chain_pem(ck.cert->to_pem()).
+ private_key_pem(ck.key->private_to_pem());
+ }
+
void reset_client_with_cert_opts(const CertKeyWrapper& ck, AuthorizedPeers authorized) {
- TransportSecurityOptions client_opts(root_ca.cert->to_pem(), ck.cert->to_pem(),
- ck.key->private_to_pem(), std::move(authorized));
- client = create_openssl_codec(client_opts, CryptoCodec::Mode::Client);
+ auto ts_params = ts_builder_from(ck).authorized_peers(std::move(authorized));
+ client = create_openssl_codec(TransportSecurityOptions(std::move(ts_params)), CryptoCodec::Mode::Client);
}
void reset_client_with_cert_opts(const CertKeyWrapper& ck, std::shared_ptr<CertificateVerificationCallback> cert_cb) {
- TransportSecurityOptions client_opts(root_ca.cert->to_pem(), ck.cert->to_pem(), ck.key->private_to_pem());
- client = create_openssl_codec(client_opts, std::move(cert_cb), CryptoCodec::Mode::Client);
+ auto ts_params = ts_builder_from(ck).authorized_peers(AuthorizedPeers::allow_all_authenticated());
+ client = create_openssl_codec(TransportSecurityOptions(std::move(ts_params)),
+ std::move(cert_cb), CryptoCodec::Mode::Client);
}
void reset_server_with_cert_opts(const CertKeyWrapper& ck, AuthorizedPeers authorized) {
- TransportSecurityOptions server_opts(root_ca.cert->to_pem(), ck.cert->to_pem(),
- ck.key->private_to_pem(), std::move(authorized));
- server = create_openssl_codec(server_opts, CryptoCodec::Mode::Server);
+ auto ts_params = ts_builder_from(ck).authorized_peers(std::move(authorized));
+ server = create_openssl_codec(TransportSecurityOptions(std::move(ts_params)), CryptoCodec::Mode::Server);
}
void reset_server_with_cert_opts(const CertKeyWrapper& ck, std::shared_ptr<CertificateVerificationCallback> cert_cb) {
- TransportSecurityOptions server_opts(root_ca.cert->to_pem(), ck.cert->to_pem(), ck.key->private_to_pem());
- server = create_openssl_codec(server_opts, std::move(cert_cb), CryptoCodec::Mode::Server);
+ auto ts_params = ts_builder_from(ck).authorized_peers(AuthorizedPeers::allow_all_authenticated());
+ server = create_openssl_codec(TransportSecurityOptions(std::move(ts_params)),
+ std::move(cert_cb), CryptoCodec::Mode::Server);
}
void reset_server_with_cert_opts(const CertKeyWrapper& ck,
std::shared_ptr<CertificateVerificationCallback> cert_cb,
AuthorizationMode authz_mode) {
- TransportSecurityOptions server_opts(root_ca.cert->to_pem(), ck.cert->to_pem(), ck.key->private_to_pem());
- server = create_openssl_codec_with_authz_mode(server_opts, std::move(cert_cb), CryptoCodec::Mode::Server, authz_mode);
+ auto ts_params = ts_builder_from(ck).authorized_peers(AuthorizedPeers::allow_all_authenticated());
+ server = create_openssl_codec_with_authz_mode(TransportSecurityOptions(std::move(ts_params)),
+ std::move(cert_cb), CryptoCodec::Mode::Server, authz_mode);
+ }
+
+ void reset_client_with_peer_spec(const CertKeyWrapper& ck,
+ const SocketSpec& peer_spec,
+ bool disable_hostname_validation = false)
+ {
+ auto ts_params = ts_builder_from(ck).
+ authorized_peers(AuthorizedPeers::allow_all_authenticated()).
+ disable_hostname_validation(disable_hostname_validation);
+ client = create_openssl_codec(TransportSecurityOptions(std::move(ts_params)),
+ CryptoCodec::Mode::Client, peer_spec);
}
};
@@ -537,7 +582,7 @@ TEST_F("Exception during verification callback processing breaks handshake", Cer
EXPECT_FALSE(f.handshake());
}
-TEST_F("certificate verification callback observes CN and DNS SANs", CertFixture) {
+TEST_F("Certificate verification callback observes CN and DNS SANs", CertFixture) {
auto ck = f.create_ca_issued_peer_cert(
{{"rockets.wile.example.com"}},
{{"DNS:crash.wile.example.com"}, {"DNS:burn.wile.example.com"}});
@@ -556,7 +601,7 @@ TEST_F("certificate verification callback observes CN and DNS SANs", CertFixture
EXPECT_EQUAL("burn.wile.example.com", creds.dns_sans[1]);
}
-TEST_F("last occurring CN is given to verification callback if multiple CNs are present", CertFixture) {
+TEST_F("Last occurring CN is given to verification callback if multiple CNs are present", CertFixture) {
auto ck = f.create_ca_issued_peer_cert(
{{"foo.wile.example.com"}, {"bar.wile.example.com"}, {"baz.wile.example.com"}}, {});
@@ -646,6 +691,51 @@ TEST_F("Disabled insecure authorization mode ignores verification result", CertF
EXPECT_TRUE(f.handshake());
}
+void reset_peers_with_client_peer_spec(CertFixture& f,
+ const SocketSpec& peer_spec,
+ bool disable_hostname_validation = false)
+{
+ auto client_ck = f.create_ca_issued_peer_cert({"hello.world.example.com"}, {});
+ f.reset_client_with_peer_spec(client_ck, peer_spec, disable_hostname_validation);
+ // Since hostname validation is enabled by default, providing a peer spec also
+ // means that we must have a valid server name to present back (or the handshake fails).
+ auto server_ck = f.create_ca_issued_peer_cert({}, {{"DNS:*.example.com"}});
+ f.reset_server_with_cert_opts(server_ck, AuthorizedPeers::allow_all_authenticated());
+}
+
+TEST_F("Client does not send SNI extension if hostname not provided in spec", CertFixture) {
+ reset_peers_with_client_peer_spec(f, SocketSpec::invalid);
+
+ ASSERT_TRUE(f.handshake());
+ auto maybe_sni = f.server->client_provided_sni_extension();
+ EXPECT_FALSE(maybe_sni.has_value());
+}
+
+TEST_F("Client sends SNI extension with hostname provided in spec", CertFixture) {
+ reset_peers_with_client_peer_spec(f, SocketSpec::from_host_port("sni-test.example.com", 12345));
+
+ ASSERT_TRUE(f.handshake());
+ auto maybe_sni = f.server->client_provided_sni_extension();
+ ASSERT_TRUE(maybe_sni.has_value());
+ EXPECT_EQUAL("sni-test.example.com", *maybe_sni);
+}
+
+TEST_F("Client hostname validation passes handshake if server hostname matches certificate", CertFixture) {
+ reset_peers_with_client_peer_spec(f, SocketSpec::from_host_port("server-must-be-under.example.com", 12345), false);
+ EXPECT_TRUE(f.handshake());
+}
+
+TEST_F("Client hostname validation fails handshake if server hostname mismatches certificate", CertFixture) {
+ // Wildcards only apply to a single level, so this should fail as the server only has a cert for *.example.com
+ reset_peers_with_client_peer_spec(f, SocketSpec::from_host_port("nested.name.example.com", 12345), false);
+ EXPECT_FALSE(f.handshake());
+}
+
+TEST_F("Mismatching server cert vs hostname does not fail if hostname validation is disabled", CertFixture) {
+ reset_peers_with_client_peer_spec(f, SocketSpec::from_host_port("a.very.nested.name.example.com", 12345), true);
+ EXPECT_TRUE(f.handshake());
+}
+
TEST_F("Failure statistics are incremented on authorization failures", CertFixture) {
reset_peers_with_server_authz_mode(f, AuthorizationMode::Enforce);
auto server_before = ConnectionStatistics::get(true).snapshot();
diff --git a/vespalib/src/tests/net/tls/transport_options/transport_options_reading_test.cpp b/vespalib/src/tests/net/tls/transport_options/transport_options_reading_test.cpp
index a54e2f29aa1..00459a4e69c 100644
--- a/vespalib/src/tests/net/tls/transport_options/transport_options_reading_test.cpp
+++ b/vespalib/src/tests/net/tls/transport_options/transport_options_reading_test.cpp
@@ -155,6 +155,47 @@ TEST("accepted cipher list is populated if specified") {
EXPECT_EQUAL("bar", ciphers[1]);
}
+// FIXME this is temporary until we know enabling it by default won't break the world!
+TEST("hostname validation is DISABLED by default when creating options from config file") {
+ const char* json = R"({"files":{"private-key":"dummy_privkey.txt",
+ "certificates":"dummy_certs.txt",
+ "ca-certificates":"dummy_ca_certs.txt"}})";
+ EXPECT_TRUE(read_options_from_json_string(json)->disable_hostname_validation());
+}
+
+TEST("TransportSecurityOptions builder does not disable hostname validation by default") {
+ auto ts_builder = vespalib::net::tls::TransportSecurityOptions::Params().
+ ca_certs_pem("foo").
+ cert_chain_pem("bar").
+ private_key_pem("fantonald");
+ TransportSecurityOptions ts_opts(std::move(ts_builder));
+ EXPECT_FALSE(ts_opts.disable_hostname_validation());
+}
+
+TEST("hostname validation can be explicitly disabled") {
+ const char* json = R"({"files":{"private-key":"dummy_privkey.txt",
+ "certificates":"dummy_certs.txt",
+ "ca-certificates":"dummy_ca_certs.txt"},
+ "disable-hostname-validation": true})";
+ EXPECT_TRUE(read_options_from_json_string(json)->disable_hostname_validation());
+}
+
+TEST("hostname validation can be explicitly enabled") {
+ const char* json = R"({"files":{"private-key":"dummy_privkey.txt",
+ "certificates":"dummy_certs.txt",
+ "ca-certificates":"dummy_ca_certs.txt"},
+ "disable-hostname-validation": false})";
+ EXPECT_FALSE(read_options_from_json_string(json)->disable_hostname_validation());
+}
+
+TEST("unknown fields are ignored at parse-time") {
+ const char* json = R"({"files":{"private-key":"dummy_privkey.txt",
+ "certificates":"dummy_certs.txt",
+ "ca-certificates":"dummy_ca_certs.txt"},
+ "flipper-the-dolphin": "*weird dolphin noises*"})";
+ EXPECT_TRUE(read_options_from_json_string(json).get() != nullptr); // And no exception thrown.
+}
+
// TODO test parsing of multiple policies
TEST_MAIN() { TEST_RUN_ALL(); }
diff --git a/vespalib/src/vespa/vespalib/data/slime/object_value.h b/vespalib/src/vespa/vespalib/data/slime/object_value.h
index 377bf5bd37f..651f3a156d2 100644
--- a/vespalib/src/vespa/vespalib/data/slime/object_value.h
+++ b/vespalib/src/vespa/vespalib/data/slime/object_value.h
@@ -12,8 +12,7 @@
#include <vespa/vespalib/stllike/vector_map.h>
#include <vespa/vespalib/util/stash.h>
-namespace vespalib {
-namespace slime {
+namespace vespalib::slime {
/**
* Class representing a collection of unordered values that can be
@@ -32,7 +31,7 @@ private:
Cursor &setIfUnset(SymbolInserter &symbol, const ValueFactory &input) {
Value *&pos = _fields[symbol.insert()];
- if (pos != 0) {
+ if (pos != nullptr) {
return *NixValue::invalid();
}
pos = input.create(_stash);
@@ -40,7 +39,7 @@ private:
}
Value *lookup(const SymbolLookup &symbol) const {
- SymbolValueMap::const_iterator pos = _fields.find(symbol.lookup());
+ auto pos = _fields.find(symbol.lookup());
if (pos == _fields.end()) {
return NixValue::invalid();
}
@@ -81,9 +80,7 @@ public:
Cursor &setObject(Memory name) override;
Symbol resolve(Memory symbol_name) override;
- ~ObjectValue() { }
+ ~ObjectValue() override = default;
};
-} // namespace vespalib::slime
-} // namespace vespalib
-
+}
diff --git a/vespalib/src/vespa/vespalib/geo/zcurve.cpp b/vespalib/src/vespa/vespalib/geo/zcurve.cpp
index b2dc02759dc..00ce7ee18c6 100644
--- a/vespalib/src/vespa/vespalib/geo/zcurve.cpp
+++ b/vespalib/src/vespa/vespalib/geo/zcurve.cpp
@@ -4,8 +4,7 @@
#include <vespa/vespalib/util/priority_queue.h>
#include <vespa/vespalib/util/fiddle.h>
-namespace vespalib {
-namespace geo {
+namespace vespalib::geo {
namespace {
@@ -182,4 +181,3 @@ ZCurve::decodeSlow(int64_t enc, int32_t *xp, int32_t *yp)
}
}
-}
diff --git a/vespalib/src/vespa/vespalib/geo/zcurve.h b/vespalib/src/vespa/vespalib/geo/zcurve.h
index 2a05c4c7744..bd76b78ea23 100644
--- a/vespalib/src/vespa/vespalib/geo/zcurve.h
+++ b/vespalib/src/vespa/vespalib/geo/zcurve.h
@@ -6,8 +6,7 @@
#include <cassert>
#include <vector>
-namespace vespalib {
-namespace geo {
+namespace vespalib::geo {
/**
* @brief Utility methods for a Z-curve (Morton-order) encoder and decoder.
@@ -31,7 +30,7 @@ public:
public:
BoundingBox(int32_t minx, int32_t maxx, int32_t miny, int32_t maxy);
- ~BoundingBox() { }
+ ~BoundingBox() = default;
int64_t getzMinx() const { return _zMinx; }
int64_t getzMaxx() const { return _zMaxx; }
@@ -221,5 +220,3 @@ public:
};
}
-}
-
diff --git a/vespalib/src/vespa/vespalib/net/crypto_engine.h b/vespalib/src/vespa/vespalib/net/crypto_engine.h
index 4deacf9a6c7..71511b8a552 100644
--- a/vespalib/src/vespa/vespalib/net/crypto_engine.h
+++ b/vespalib/src/vespa/vespalib/net/crypto_engine.h
@@ -19,6 +19,8 @@ class SocketSpec;
**/
struct CryptoEngine {
using SP = std::shared_ptr<CryptoEngine>;
+ virtual bool use_tls_when_client() const = 0;
+ virtual bool always_use_tls_when_server() const = 0;
virtual CryptoSocket::UP create_client_crypto_socket(SocketHandle socket, const SocketSpec &spec) = 0;
virtual CryptoSocket::UP create_server_crypto_socket(SocketHandle socket) = 0;
virtual ~CryptoEngine();
@@ -29,6 +31,8 @@ struct CryptoEngine {
* Crypto engine without encryption.
**/
struct NullCryptoEngine : public CryptoEngine {
+ bool use_tls_when_client() const override { return false; }
+ bool always_use_tls_when_server() const override { return false; }
CryptoSocket::UP create_client_crypto_socket(SocketHandle socket, const SocketSpec &spec) override;
CryptoSocket::UP create_server_crypto_socket(SocketHandle socket) override;
};
@@ -39,6 +43,8 @@ struct NullCryptoEngine : public CryptoEngine {
* from TLS.
**/
struct XorCryptoEngine : public CryptoEngine {
+ bool use_tls_when_client() const override { return false; }
+ bool always_use_tls_when_server() const override { return false; }
CryptoSocket::UP create_client_crypto_socket(SocketHandle socket, const SocketSpec &spec) override;
CryptoSocket::UP create_server_crypto_socket(SocketHandle socket) override;
};
diff --git a/vespalib/src/vespa/vespalib/net/socket_spec.cpp b/vespalib/src/vespa/vespalib/net/socket_spec.cpp
index 06682086670..d1dd81a454a 100644
--- a/vespalib/src/vespa/vespalib/net/socket_spec.cpp
+++ b/vespalib/src/vespa/vespalib/net/socket_spec.cpp
@@ -41,7 +41,7 @@ SocketSpec::address(bool server) const
return SocketAddress();
}
-SocketSpec SocketSpec::invalid;
+const SocketSpec SocketSpec::invalid;
SocketSpec::SocketSpec(const vespalib::string &spec)
: SocketSpec()
diff --git a/vespalib/src/vespa/vespalib/net/socket_spec.h b/vespalib/src/vespa/vespalib/net/socket_spec.h
index 01af382d638..4e3dddf6814 100644
--- a/vespalib/src/vespa/vespalib/net/socket_spec.h
+++ b/vespalib/src/vespa/vespalib/net/socket_spec.h
@@ -24,7 +24,7 @@ private:
: _type(type), _node(node), _port(port) {}
SocketAddress address(bool server) const;
public:
- static SocketSpec invalid;
+ static const SocketSpec invalid;
explicit SocketSpec(const vespalib::string &spec);
vespalib::string spec() const;
SocketSpec replace_host(const vespalib::string &new_host) const;
diff --git a/vespalib/src/vespa/vespalib/net/tls/auto_reloading_tls_crypto_engine.cpp b/vespalib/src/vespa/vespalib/net/tls/auto_reloading_tls_crypto_engine.cpp
index c425ab75ce8..bdb2402adbc 100644
--- a/vespalib/src/vespa/vespalib/net/tls/auto_reloading_tls_crypto_engine.cpp
+++ b/vespalib/src/vespa/vespalib/net/tls/auto_reloading_tls_crypto_engine.cpp
@@ -99,6 +99,18 @@ CryptoSocket::UP AutoReloadingTlsCryptoEngine::create_server_crypto_socket(Socke
return acquire_current_engine()->create_server_crypto_socket(std::move(socket));
}
+bool
+AutoReloadingTlsCryptoEngine::use_tls_when_client() const
+{
+ return acquire_current_engine()->use_tls_when_client();
+}
+
+bool
+AutoReloadingTlsCryptoEngine::always_use_tls_when_server() const
+{
+ return acquire_current_engine()->always_use_tls_when_server();
+}
+
std::unique_ptr<TlsCryptoSocket>
AutoReloadingTlsCryptoEngine::create_tls_client_crypto_socket(SocketHandle socket, const SocketSpec &spec) {
return acquire_current_engine()->create_tls_client_crypto_socket(std::move(socket), spec);
diff --git a/vespalib/src/vespa/vespalib/net/tls/auto_reloading_tls_crypto_engine.h b/vespalib/src/vespa/vespalib/net/tls/auto_reloading_tls_crypto_engine.h
index e268cbc8f1a..1b80b782daf 100644
--- a/vespalib/src/vespa/vespalib/net/tls/auto_reloading_tls_crypto_engine.h
+++ b/vespalib/src/vespa/vespalib/net/tls/auto_reloading_tls_crypto_engine.h
@@ -47,6 +47,8 @@ public:
CryptoSocket::UP create_client_crypto_socket(SocketHandle socket, const SocketSpec &spec) override;
CryptoSocket::UP create_server_crypto_socket(SocketHandle socket) override;
+ bool use_tls_when_client() const override;
+ bool always_use_tls_when_server() const override;
std::unique_ptr<TlsCryptoSocket> create_tls_client_crypto_socket(SocketHandle socket, const SocketSpec &spec) override;
std::unique_ptr<TlsCryptoSocket> create_tls_server_crypto_socket(SocketHandle socket) override;
};
diff --git a/vespalib/src/vespa/vespalib/net/tls/crypto_codec.cpp b/vespalib/src/vespa/vespalib/net/tls/crypto_codec.cpp
index c54990b3782..d3ac975d90a 100644
--- a/vespalib/src/vespa/vespalib/net/tls/crypto_codec.cpp
+++ b/vespalib/src/vespa/vespalib/net/tls/crypto_codec.cpp
@@ -6,12 +6,23 @@
namespace vespalib::net::tls {
-std::unique_ptr<CryptoCodec> CryptoCodec::create_default_codec(
- std::shared_ptr<TlsContext> ctx, const SocketAddress& peer_address, Mode mode)
+std::unique_ptr<CryptoCodec>
+CryptoCodec::create_default_client_codec(std::shared_ptr<TlsContext> ctx,
+ const SocketSpec& peer_spec,
+ const SocketAddress& peer_address)
{
auto ctx_impl = std::dynamic_pointer_cast<impl::OpenSslTlsContextImpl>(ctx); // only takes by const ref
assert(ctx_impl);
- return std::make_unique<impl::OpenSslCryptoCodecImpl>(std::move(ctx_impl), peer_address, mode);
+ return impl::OpenSslCryptoCodecImpl::make_client_codec(std::move(ctx_impl), peer_spec, peer_address);
+}
+
+std::unique_ptr<CryptoCodec>
+CryptoCodec::create_default_server_codec(std::shared_ptr<TlsContext> ctx,
+ const SocketAddress& peer_address)
+{
+ auto ctx_impl = std::dynamic_pointer_cast<impl::OpenSslTlsContextImpl>(ctx); // only takes by const ref
+ assert(ctx_impl);
+ return impl::OpenSslCryptoCodecImpl::make_server_codec(std::move(ctx_impl), peer_address);
}
}
diff --git a/vespalib/src/vespa/vespalib/net/tls/crypto_codec.h b/vespalib/src/vespa/vespalib/net/tls/crypto_codec.h
index 5d9684461d7..787485b47be 100644
--- a/vespalib/src/vespa/vespalib/net/tls/crypto_codec.h
+++ b/vespalib/src/vespa/vespalib/net/tls/crypto_codec.h
@@ -4,6 +4,8 @@
#include <vespa/vespalib/net/socket_address.h>
#include <memory>
+namespace vespalib { class SocketSpec; }
+
namespace vespalib::net::tls {
struct HandshakeResult {
@@ -179,9 +181,13 @@ public:
*
* Throws CryptoException if resources cannot be allocated for the codec.
*/
- static std::unique_ptr<CryptoCodec> create_default_codec(std::shared_ptr<TlsContext> ctx,
- const SocketAddress& peer_address,
- Mode mode);
+ static std::unique_ptr<CryptoCodec>
+ create_default_client_codec(std::shared_ptr<TlsContext> ctx,
+ const SocketSpec& peer_spec,
+ const SocketAddress& peer_address);
+ static std::unique_ptr<CryptoCodec>
+ create_default_server_codec(std::shared_ptr<TlsContext> ctx,
+ const SocketAddress& peer_address);
};
}
diff --git a/vespalib/src/vespa/vespalib/net/tls/impl/openssl_crypto_codec_impl.cpp b/vespalib/src/vespa/vespalib/net/tls/impl/openssl_crypto_codec_impl.cpp
index 5315754d53a..6a79caa8264 100644
--- a/vespalib/src/vespa/vespalib/net/tls/impl/openssl_crypto_codec_impl.cpp
+++ b/vespalib/src/vespa/vespalib/net/tls/impl/openssl_crypto_codec_impl.cpp
@@ -172,9 +172,11 @@ void log_ssl_error(const char* source, const SocketAddress& peer_address, int ss
} // anon ns
OpenSslCryptoCodecImpl::OpenSslCryptoCodecImpl(std::shared_ptr<OpenSslTlsContextImpl> ctx,
+ const SocketSpec& peer_spec,
const SocketAddress& peer_address,
Mode mode)
: _ctx(std::move(ctx)),
+ _peer_spec(peer_spec),
_peer_address(peer_address),
_ssl(::SSL_new(_ctx->native_context())),
_mode(mode),
@@ -219,6 +221,8 @@ OpenSslCryptoCodecImpl::OpenSslCryptoCodecImpl(std::shared_ptr<OpenSslTlsContext
_output_bio = tmp_output_bio.release();
if (_mode == Mode::Client) {
::SSL_set_connect_state(_ssl.get());
+ enable_hostname_validation_if_requested();
+ set_server_name_indication_extension();
} else {
::SSL_set_accept_state(_ssl.get());
}
@@ -230,6 +234,59 @@ OpenSslCryptoCodecImpl::OpenSslCryptoCodecImpl(std::shared_ptr<OpenSslTlsContext
OpenSslCryptoCodecImpl::~OpenSslCryptoCodecImpl() = default;
+std::unique_ptr<OpenSslCryptoCodecImpl>
+OpenSslCryptoCodecImpl::make_client_codec(std::shared_ptr<OpenSslTlsContextImpl> ctx,
+ const SocketSpec& peer_spec,
+ const SocketAddress& peer_address)
+{
+ // Naked new due to private ctor
+ return std::unique_ptr<OpenSslCryptoCodecImpl>(
+ new OpenSslCryptoCodecImpl(std::move(ctx), peer_spec, peer_address, Mode::Client));
+}
+std::unique_ptr<OpenSslCryptoCodecImpl>
+OpenSslCryptoCodecImpl::make_server_codec(std::shared_ptr<OpenSslTlsContextImpl> ctx,
+ const SocketAddress& peer_address)
+{
+ // Naked new due to private ctor
+ return std::unique_ptr<OpenSslCryptoCodecImpl>(
+ new OpenSslCryptoCodecImpl(std::move(ctx), SocketSpec::invalid, peer_address, Mode::Server));
+}
+
+void OpenSslCryptoCodecImpl::enable_hostname_validation_if_requested() {
+ if (_peer_spec.valid() && !_ctx->transport_security_options().disable_hostname_validation()) {
+ auto* verify_param = SSL_get0_param(_ssl.get()); // Internal ptr, no refcount bump or alloc. We must not free.
+ LOG_ASSERT(verify_param != nullptr);
+ vespalib::string host = _peer_spec.host();
+ if (X509_VERIFY_PARAM_set1_host(verify_param, host.c_str(), host.size()) != 1) {
+ throw CryptoException("X509_VERIFY_PARAM_set1_host() failed");
+ }
+ // TODO should we set expected IP based on peer address as well?
+ }
+}
+
+void OpenSslCryptoCodecImpl::set_server_name_indication_extension() {
+ if (_peer_spec.valid()) {
+ vespalib::string host = _peer_spec.host();
+ // OpenSSL tries to cast const char* to void* in a macro, even on 1.1.1. GCC is not overly impressed,
+ // so to satiate OpenSSL's quirks we pre-cast away the constness.
+ auto* host_cstr_that_trusts_openssl_not_to_mess_up = const_cast<char*>(host.c_str());
+ if (SSL_set_tlsext_host_name(_ssl.get(), host_cstr_that_trusts_openssl_not_to_mess_up) != 1) {
+ throw CryptoException("SSL_set_tlsext_host_name() failed");
+ }
+ }
+}
+
+std::optional<vespalib::string> OpenSslCryptoCodecImpl::client_provided_sni_extension() const {
+ if ((_mode != Mode::Server) || (SSL_get_servername_type(_ssl.get()) != TLSEXT_NAMETYPE_host_name)) {
+ return {};
+ }
+ const char* sni_host_raw = SSL_get_servername(_ssl.get(), TLSEXT_NAMETYPE_host_name);
+ if (sni_host_raw == nullptr) {
+ return {};
+ }
+ return vespalib::string(sni_host_raw);
+}
+
HandshakeResult OpenSslCryptoCodecImpl::handshake(const char* from_peer, size_t from_peer_buf_size,
char* to_peer, size_t to_peer_buf_size) noexcept
{
@@ -428,3 +485,5 @@ EncodeResult OpenSslCryptoCodecImpl::half_close(char* ciphertext, size_t ciphert
// External references:
// [0] http://openssl.6102.n7.nabble.com/nonblocking-implementation-question-tp1728p1732.html
// [1] https://github.com/grpc/grpc/blob/master/src/core/tsi/ssl_transport_security.cc
+// [2] https://wiki.openssl.org/index.php/Hostname_validation
+// [3] https://wiki.openssl.org/index.php/SSL/TLS_Client
diff --git a/vespalib/src/vespa/vespalib/net/tls/impl/openssl_crypto_codec_impl.h b/vespalib/src/vespa/vespalib/net/tls/impl/openssl_crypto_codec_impl.h
index 14200de449a..ec8df853c16 100644
--- a/vespalib/src/vespa/vespalib/net/tls/impl/openssl_crypto_codec_impl.h
+++ b/vespalib/src/vespa/vespalib/net/tls/impl/openssl_crypto_codec_impl.h
@@ -3,6 +3,7 @@
#include "openssl_typedefs.h"
#include <vespa/vespalib/net/socket_address.h>
+#include <vespa/vespalib/net/socket_spec.h>
#include <vespa/vespalib/net/tls/transport_security_options.h>
#include <vespa/vespalib/net/tls/crypto_codec.h>
#include <memory>
@@ -46,17 +47,23 @@ class OpenSslCryptoCodecImpl : public CryptoCodec {
// The context maintains shared verification callback state, so it must be
// kept alive explictly for at least as long as any codecs.
std::shared_ptr<OpenSslTlsContextImpl> _ctx;
+ SocketSpec _peer_spec;
SocketAddress _peer_address;
- SslPtr _ssl;
- ::BIO* _input_bio; // Owned by _ssl
- ::BIO* _output_bio; // Owned by _ssl
- Mode _mode;
+ SslPtr _ssl;
+ ::BIO* _input_bio; // Owned by _ssl
+ ::BIO* _output_bio; // Owned by _ssl
+ Mode _mode;
std::optional<DeferredHandshakeParams> _deferred_handshake_params;
- std::optional<HandshakeResult> _deferred_handshake_result;
+ std::optional<HandshakeResult> _deferred_handshake_result;
public:
- OpenSslCryptoCodecImpl(std::shared_ptr<OpenSslTlsContextImpl> ctx, const SocketAddress& peer_address, Mode mode);
~OpenSslCryptoCodecImpl() override;
+ static std::unique_ptr<OpenSslCryptoCodecImpl> make_client_codec(std::shared_ptr<OpenSslTlsContextImpl> ctx,
+ const SocketSpec& peer_spec,
+ const SocketAddress& peer_address);
+ static std::unique_ptr<OpenSslCryptoCodecImpl> make_server_codec(std::shared_ptr<OpenSslTlsContextImpl> ctx,
+ const SocketAddress& peer_address);
+
/*
* From RFC 8449 (Record Size Limit Extension for TLS), section 1:
* "TLS versions 1.2 [RFC5246] and earlier permit senders to
@@ -89,7 +96,20 @@ public:
EncodeResult half_close(char* ciphertext, size_t ciphertext_size) noexcept override;
const SocketAddress& peer_address() const noexcept { return _peer_address; }
+ /*
+ * If a client has sent a SNI extension field as part of the handshake,
+ * returns the raw string representation of this. It only makes sense to
+ * call this for codecs in server mode.
+ */
+ std::optional<vespalib::string> client_provided_sni_extension() const;
private:
+ OpenSslCryptoCodecImpl(std::shared_ptr<OpenSslTlsContextImpl> ctx,
+ const SocketSpec& peer_spec,
+ const SocketAddress& peer_address,
+ Mode mode);
+
+ void enable_hostname_validation_if_requested();
+ void set_server_name_indication_extension();
HandshakeResult do_handshake_and_consume_peer_input_bytes() noexcept;
DecodeResult drain_and_produce_plaintext_from_ssl(char* plaintext, size_t plaintext_size) noexcept;
// Precondition: read_result < 0
diff --git a/vespalib/src/vespa/vespalib/net/tls/maybe_tls_crypto_engine.h b/vespalib/src/vespa/vespalib/net/tls/maybe_tls_crypto_engine.h
index 147a770bc8f..ece7d094c54 100644
--- a/vespalib/src/vespa/vespalib/net/tls/maybe_tls_crypto_engine.h
+++ b/vespalib/src/vespa/vespalib/net/tls/maybe_tls_crypto_engine.h
@@ -28,6 +28,8 @@ public:
: _null_engine(std::make_shared<NullCryptoEngine>()),
_tls_engine(std::move(tls_engine)),
_use_tls_when_client(use_tls_when_client) {}
+ bool use_tls_when_client() const override { return _use_tls_when_client; }
+ bool always_use_tls_when_server() const override { return false; }
CryptoSocket::UP create_client_crypto_socket(SocketHandle socket, const SocketSpec &spec) override;
CryptoSocket::UP create_server_crypto_socket(SocketHandle socket) override;
};
diff --git a/vespalib/src/vespa/vespalib/net/tls/peer_policies.h b/vespalib/src/vespa/vespalib/net/tls/peer_policies.h
index 44cbe70fd92..c558708de8f 100644
--- a/vespalib/src/vespa/vespalib/net/tls/peer_policies.h
+++ b/vespalib/src/vespa/vespalib/net/tls/peer_policies.h
@@ -64,19 +64,24 @@ public:
class AuthorizedPeers {
// A peer will be authorized iff it matches _one or more_ policies.
std::vector<PeerPolicy> _peer_policies;
- bool _allow_all_if_empty = false;
+ bool _allow_all_if_empty;
explicit AuthorizedPeers(bool allow_all_if_empty)
: _peer_policies(),
_allow_all_if_empty(allow_all_if_empty)
{}
public:
- AuthorizedPeers() = default;
+ AuthorizedPeers() : _peer_policies(), _allow_all_if_empty(false) {}
explicit AuthorizedPeers(std::vector<PeerPolicy> peer_policies_)
: _peer_policies(std::move(peer_policies_)),
_allow_all_if_empty(false)
{}
+ AuthorizedPeers(const AuthorizedPeers&) = default;
+ AuthorizedPeers& operator=(const AuthorizedPeers&) = default;
+ AuthorizedPeers(AuthorizedPeers&&) noexcept = default;
+ AuthorizedPeers& operator=(AuthorizedPeers&&) noexcept = default;
+
static AuthorizedPeers allow_all_authenticated() {
return AuthorizedPeers(true);
}
diff --git a/vespalib/src/vespa/vespalib/net/tls/tls_crypto_engine.cpp b/vespalib/src/vespa/vespalib/net/tls/tls_crypto_engine.cpp
index d0475f3e88d..99862e83dbf 100644
--- a/vespalib/src/vespa/vespalib/net/tls/tls_crypto_engine.cpp
+++ b/vespalib/src/vespa/vespalib/net/tls/tls_crypto_engine.cpp
@@ -12,18 +12,16 @@ TlsCryptoEngine::TlsCryptoEngine(net::tls::TransportSecurityOptions tls_opts, ne
}
std::unique_ptr<TlsCryptoSocket>
-TlsCryptoEngine::create_tls_client_crypto_socket(SocketHandle socket, const SocketSpec &)
+TlsCryptoEngine::create_tls_client_crypto_socket(SocketHandle socket, const SocketSpec &peer_spec)
{
- auto mode = net::tls::CryptoCodec::Mode::Client;
- auto codec = net::tls::CryptoCodec::create_default_codec(_tls_ctx, SocketAddress::peer_address(socket.get()), mode);
+ auto codec = net::tls::CryptoCodec::create_default_client_codec(_tls_ctx, peer_spec, SocketAddress::peer_address(socket.get()));
return std::make_unique<net::tls::CryptoCodecAdapter>(std::move(socket), std::move(codec));
}
std::unique_ptr<TlsCryptoSocket>
TlsCryptoEngine::create_tls_server_crypto_socket(SocketHandle socket)
{
- auto mode = net::tls::CryptoCodec::Mode::Server;
- auto codec = net::tls::CryptoCodec::create_default_codec(_tls_ctx, SocketAddress::peer_address(socket.get()), mode);
+ auto codec = net::tls::CryptoCodec::create_default_server_codec(_tls_ctx, SocketAddress::peer_address(socket.get()));
return std::make_unique<net::tls::CryptoCodecAdapter>(std::move(socket), std::move(codec));
}
diff --git a/vespalib/src/vespa/vespalib/net/tls/tls_crypto_engine.h b/vespalib/src/vespa/vespalib/net/tls/tls_crypto_engine.h
index 5e760cf5585..444a817b357 100644
--- a/vespalib/src/vespa/vespalib/net/tls/tls_crypto_engine.h
+++ b/vespalib/src/vespa/vespalib/net/tls/tls_crypto_engine.h
@@ -27,6 +27,8 @@ public:
net::tls::AuthorizationMode authz_mode = net::tls::AuthorizationMode::Enforce);
std::unique_ptr<TlsCryptoSocket> create_tls_client_crypto_socket(SocketHandle socket, const SocketSpec &spec) override;
std::unique_ptr<TlsCryptoSocket> create_tls_server_crypto_socket(SocketHandle socket) override;
+ bool use_tls_when_client() const override { return true; }
+ bool always_use_tls_when_server() const override { return true; }
CryptoSocket::UP create_client_crypto_socket(SocketHandle socket, const SocketSpec &spec) override {
return create_tls_client_crypto_socket(std::move(socket), spec);
}
diff --git a/vespalib/src/vespa/vespalib/net/tls/transport_security_options.cpp b/vespalib/src/vespa/vespalib/net/tls/transport_security_options.cpp
index c9a5fa5a6e9..47b0e1e0a43 100644
--- a/vespalib/src/vespa/vespalib/net/tls/transport_security_options.cpp
+++ b/vespalib/src/vespa/vespalib/net/tls/transport_security_options.cpp
@@ -11,41 +11,57 @@ TransportSecurityOptions::TransportSecurityOptions(Params params)
_cert_chain_pem(std::move(params._cert_chain_pem)),
_private_key_pem(std::move(params._private_key_pem)),
_authorized_peers(std::move(params._authorized_peers)),
- _accepted_ciphers(std::move(params._accepted_ciphers))
+ _accepted_ciphers(std::move(params._accepted_ciphers)),
+ _disable_hostname_validation(params._disable_hostname_validation)
{
}
TransportSecurityOptions::TransportSecurityOptions(vespalib::string ca_certs_pem,
vespalib::string cert_chain_pem,
- vespalib::string private_key_pem)
+ vespalib::string private_key_pem,
+ AuthorizedPeers authorized_peers,
+ bool disable_hostname_validation)
: _ca_certs_pem(std::move(ca_certs_pem)),
_cert_chain_pem(std::move(cert_chain_pem)),
_private_key_pem(std::move(private_key_pem)),
- _authorized_peers(AuthorizedPeers::allow_all_authenticated())
+ _authorized_peers(std::move(authorized_peers)),
+ _disable_hostname_validation(disable_hostname_validation)
{
}
-TransportSecurityOptions::TransportSecurityOptions(vespalib::string ca_certs_pem,
- vespalib::string cert_chain_pem,
- vespalib::string private_key_pem,
- AuthorizedPeers authorized_peers)
- : _ca_certs_pem(std::move(ca_certs_pem)),
- _cert_chain_pem(std::move(cert_chain_pem)),
- _private_key_pem(std::move(private_key_pem)),
- _authorized_peers(std::move(authorized_peers))
-{
+TransportSecurityOptions TransportSecurityOptions::copy_without_private_key() const {
+ return TransportSecurityOptions(_ca_certs_pem, _cert_chain_pem, "",
+ _authorized_peers, _disable_hostname_validation);
}
void secure_memzero(void* buf, size_t size) noexcept {
OPENSSL_cleanse(buf, size);
}
-TransportSecurityOptions::Params::Params() = default;
+TransportSecurityOptions::Params::Params()
+ : _ca_certs_pem(),
+ _cert_chain_pem(),
+ _private_key_pem(),
+ _authorized_peers(),
+ _accepted_ciphers(),
+ _disable_hostname_validation(false)
+{
+}
TransportSecurityOptions::Params::~Params() {
secure_memzero(&_private_key_pem[0], _private_key_pem.size());
}
+TransportSecurityOptions::Params::Params(const Params&) = default;
+
+TransportSecurityOptions::Params&
+TransportSecurityOptions::Params::operator=(const TransportSecurityOptions::Params&) = default;
+
+TransportSecurityOptions::Params::Params(Params&&) noexcept = default;
+
+TransportSecurityOptions::Params&
+TransportSecurityOptions::Params::operator=(TransportSecurityOptions::Params&&) noexcept = default;
+
TransportSecurityOptions::~TransportSecurityOptions() {
secure_memzero(&_private_key_pem[0], _private_key_pem.size());
}
diff --git a/vespalib/src/vespa/vespalib/net/tls/transport_security_options.h b/vespalib/src/vespa/vespalib/net/tls/transport_security_options.h
index 39abb1ce4de..e0a48fc6cf5 100644
--- a/vespalib/src/vespa/vespalib/net/tls/transport_security_options.h
+++ b/vespalib/src/vespa/vespalib/net/tls/transport_security_options.h
@@ -14,18 +14,22 @@ class TransportSecurityOptions {
vespalib::string _private_key_pem;
AuthorizedPeers _authorized_peers;
std::vector<vespalib::string> _accepted_ciphers;
+ bool _disable_hostname_validation;
public:
- TransportSecurityOptions() = default;
-
struct Params {
vespalib::string _ca_certs_pem;
vespalib::string _cert_chain_pem;
vespalib::string _private_key_pem;
AuthorizedPeers _authorized_peers;
std::vector<vespalib::string> _accepted_ciphers;
+ bool _disable_hostname_validation;
Params();
~Params();
+ Params(const Params&);
+ Params& operator=(const Params&);
+ Params(Params&&) noexcept;
+ Params& operator=(Params&&) noexcept;
Params& ca_certs_pem(vespalib::stringref pem) { _ca_certs_pem = pem; return *this; }
Params& cert_chain_pem(vespalib::stringref pem) { _cert_chain_pem = pem; return *this; }
@@ -35,19 +39,14 @@ public:
_accepted_ciphers = std::move(ciphers);
return *this;
}
+ Params& disable_hostname_validation(bool disable) {
+ _disable_hostname_validation = disable;
+ return *this;
+ }
};
explicit TransportSecurityOptions(Params params);
- TransportSecurityOptions(vespalib::string ca_certs_pem,
- vespalib::string cert_chain_pem,
- vespalib::string private_key_pem);
-
- TransportSecurityOptions(vespalib::string ca_certs_pem,
- vespalib::string cert_chain_pem,
- vespalib::string private_key_pem,
- AuthorizedPeers authorized_peers);
-
~TransportSecurityOptions();
const vespalib::string& ca_certs_pem() const noexcept { return _ca_certs_pem; }
@@ -55,10 +54,16 @@ public:
const vespalib::string& private_key_pem() const noexcept { return _private_key_pem; }
const AuthorizedPeers& authorized_peers() const noexcept { return _authorized_peers; }
- TransportSecurityOptions copy_without_private_key() const {
- return TransportSecurityOptions(_ca_certs_pem, _cert_chain_pem, "", _authorized_peers);
- }
+ TransportSecurityOptions copy_without_private_key() const;
const std::vector<vespalib::string>& accepted_ciphers() const noexcept { return _accepted_ciphers; }
+ bool disable_hostname_validation() const noexcept { return _disable_hostname_validation; }
+
+private:
+ TransportSecurityOptions(vespalib::string ca_certs_pem,
+ vespalib::string cert_chain_pem,
+ vespalib::string private_key_pem,
+ AuthorizedPeers authorized_peers,
+ bool disable_hostname_validation);
};
// Zeroes out `size` bytes in `buf` in a way that shall never be optimized
diff --git a/vespalib/src/vespa/vespalib/net/tls/transport_security_options_reading.cpp b/vespalib/src/vespa/vespalib/net/tls/transport_security_options_reading.cpp
index 6b6e65bef8a..80caa15e8b2 100644
--- a/vespalib/src/vespa/vespalib/net/tls/transport_security_options_reading.cpp
+++ b/vespalib/src/vespa/vespalib/net/tls/transport_security_options_reading.cpp
@@ -123,6 +123,12 @@ std::unique_ptr<TransportSecurityOptions> load_from_input(Input& input) {
auto priv_key = load_file_referenced_by_field(files, "private-key");
auto authorized_peers = parse_authorized_peers(root["authorized-peers"]);
auto accepted_ciphers = parse_accepted_ciphers(root["accepted-ciphers"]);
+ // FIXME this is temporary until we know it won't break a bunch of things!
+ // It's still possible to explicitly enable hostname validation by setting this to false.
+ bool disable_hostname_validation = true;
+ if (root["disable-hostname-validation"].valid()) {
+ disable_hostname_validation = root["disable-hostname-validation"].asBool();
+ }
auto options = std::make_unique<TransportSecurityOptions>(
TransportSecurityOptions::Params()
@@ -130,7 +136,8 @@ std::unique_ptr<TransportSecurityOptions> load_from_input(Input& input) {
.cert_chain_pem(certs)
.private_key_pem(priv_key)
.authorized_peers(std::move(authorized_peers))
- .accepted_ciphers(std::move(accepted_ciphers)));
+ .accepted_ciphers(std::move(accepted_ciphers))
+ .disable_hostname_validation(disable_hostname_validation));
secure_memzero(&priv_key[0], priv_key.size());
return options;
}
diff --git a/vespalib/src/vespa/vespalib/test/make_tls_options_for_testing.cpp b/vespalib/src/vespa/vespalib/test/make_tls_options_for_testing.cpp
index dcd2ced8036..b31f2830976 100644
--- a/vespalib/src/vespa/vespalib/test/make_tls_options_for_testing.cpp
+++ b/vespalib/src/vespa/vespalib/test/make_tls_options_for_testing.cpp
@@ -70,7 +70,13 @@ namespace vespalib::test {
SocketSpec local_spec("tcp/localhost:123");
vespalib::net::tls::TransportSecurityOptions make_tls_options_for_testing() {
- return vespalib::net::tls::TransportSecurityOptions(ca_pem, cert_pem, key_pem);
+ auto ts_builder = vespalib::net::tls::TransportSecurityOptions::Params().
+ ca_certs_pem(ca_pem).
+ cert_chain_pem(cert_pem).
+ private_key_pem(key_pem).
+ authorized_peers(vespalib::net::tls::AuthorizedPeers::allow_all_authenticated()).
+ disable_hostname_validation(true); // FIXME this is to avoid mass breakage of TLS'd networking tests.
+ return vespalib::net::tls::TransportSecurityOptions(std::move(ts_builder));
}
} // namespace vespalib::test
diff --git a/vespalib/src/vespa/vespalib/util/stash.cpp b/vespalib/src/vespa/vespalib/util/stash.cpp
index 18ed0e56e27..836654c2fb2 100644
--- a/vespalib/src/vespa/vespalib/util/stash.cpp
+++ b/vespalib/src/vespa/vespalib/util/stash.cpp
@@ -59,14 +59,14 @@ Stash::do_alloc(size_t size)
}
}
-Stash::Stash(size_t chunk_size)
+Stash::Stash(size_t chunk_size) noexcept
: _chunks(nullptr),
_cleanup(nullptr),
_chunk_size(std::max(size_t(4096), chunk_size))
{
}
-Stash::Stash(Stash &&rhs)
+Stash::Stash(Stash &&rhs) noexcept
: _chunks(rhs._chunks),
_cleanup(rhs._cleanup),
_chunk_size(rhs._chunk_size)
@@ -76,7 +76,7 @@ Stash::Stash(Stash &&rhs)
}
Stash &
-Stash::operator=(Stash &&rhs)
+Stash::operator=(Stash &&rhs) noexcept
{
stash::run_cleanup(_cleanup);
stash::free_chunks(_chunks);
diff --git a/vespalib/src/vespa/vespalib/util/stash.h b/vespalib/src/vespa/vespalib/util/stash.h
index aa1441aa0bb..c5e8631ca9e 100644
--- a/vespalib/src/vespa/vespalib/util/stash.h
+++ b/vespalib/src/vespa/vespalib/util/stash.h
@@ -14,19 +14,19 @@ struct Cleanup {
explicit Cleanup(Cleanup *next_in) noexcept : next(next_in) {}
virtual void cleanup() = 0;
protected:
- virtual ~Cleanup() {}
+ virtual ~Cleanup() = default;
};
// used as header for memory allocated outside the stash
struct DeleteMemory : public Cleanup {
explicit DeleteMemory(Cleanup *next_in) noexcept : Cleanup(next_in) {}
- virtual void cleanup() override { free((void*)this); }
+ void cleanup() override { free((void*)this); }
};
// used as prefix for objects to be destructed
template<typename T> struct DestructObject : public Cleanup {
explicit DestructObject(Cleanup *next_in) noexcept : Cleanup(next_in) {}
- virtual void cleanup() override { reinterpret_cast<T*>(this + 1)->~T(); }
+ void cleanup() override { reinterpret_cast<T*>(this + 1)->~T(); }
};
// used as prefix for arrays to be destructed
@@ -34,7 +34,7 @@ template<typename T> struct DestructArray : public Cleanup {
size_t size;
explicit DestructArray(Cleanup *next_in, size_t size_in) noexcept
: Cleanup(next_in), size(size_in) {}
- virtual void cleanup() override {
+ void cleanup() override {
T *array = reinterpret_cast<T*>(this + 1);
for (size_t i = size; i-- > 0;) {
array[i].~T();
@@ -46,7 +46,7 @@ struct Chunk {
Chunk *next;
size_t used;
Chunk(const Chunk &) = delete;
- Chunk(Chunk *next_in) : next(next_in), used(sizeof(Chunk)) {}
+ explicit Chunk(Chunk *next_in) : next(next_in), used(sizeof(Chunk)) {}
void clear() { used = sizeof(Chunk); }
char *alloc(size_t size, size_t chunk_size) {
size_t aligned_size = ((size + (sizeof(char *) - 1))
@@ -124,14 +124,14 @@ public:
};
typedef std::unique_ptr<Stash> UP;
- explicit Stash(size_t chunk_size);
- Stash() : Stash(4096) {}
- Stash(Stash &&rhs);
+ explicit Stash(size_t chunk_size) noexcept ;
+ Stash() noexcept : Stash(4096) {}
+ Stash(Stash &&rhs) noexcept;
Stash(const Stash &) = delete;
Stash & operator = (const Stash &) = delete;
~Stash();
- Stash &operator=(Stash &&rhs);
+ Stash &operator=(Stash &&rhs) noexcept;
void clear();