summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorHenning Baldersheim <balder@yahoo-inc.com>2020-02-05 09:27:01 +0100
committerGitHub <noreply@github.com>2020-02-05 09:27:01 +0100
commitcc6c05c46211b0c061f99408232bd5740a625107 (patch)
treeba148745574afa5833ec9d55e448d4bf9005b58e
parent4285a9e76ddab7761031899d52259c1fc7779c4b (diff)
parent2a7f13ab9f019586fc38275735ead7ad02afbd54 (diff)
Merge branch 'master' into balder/less-unused-header-body-references
-rw-r--r--application/src/main/java/com/yahoo/application/Application.java2
-rw-r--r--application/src/test/app-packages/model-evaluation/models/lightgbm/regression.json275
-rw-r--r--application/src/test/java/com/yahoo/application/container/ContainerModelEvaluationTest.java7
-rwxr-xr-xbootstrap-cmake.sh2
-rw-r--r--config-model-api/abi-spec.json3
-rw-r--r--config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java1
-rw-r--r--config-model/src/main/java/com/yahoo/documentmodel/NewDocumentType.java23
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/DocumentModelBuilder.java12
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/ImportedFieldsEnumerator.java33
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java3
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/SearchBuilder.java8
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/document/SDDocumentType.java9
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/ExpressionTransforms.java1
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/LightGBMFeatureConverter.java59
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/processing/ImportedFieldsResolver.java4
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeResolver.java16
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentManager.java10
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentTypes.java9
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java3
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/container/http/ssl/ConfiguredFilebasedSslProvider.java28
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/JettyConnectorBuilder.java21
-rw-r--r--config-model/src/main/resources/schema/common.rnc3
-rw-r--r--config-model/src/main/resources/schema/containercluster.rnc4
-rw-r--r--config-model/src/test/cfg/application/ml_models/models/lightgbm_regression.json275
-rw-r--r--config-model/src/test/cfg/application/ml_models/searchdefinitions/test.sd6
-rw-r--r--config-model/src/test/cfg/application/ml_serving/models/lightgbm_regression.json275
-rw-r--r--config-model/src/test/configmodel/types/references/documentmanager_multiple_imported_fields.cfg104
-rw-r--r--config-model/src/test/configmodel/types/references/documenttypes_multiple_imported_fields.cfg141
-rw-r--r--config-model/src/test/integration/lightgbm/models/regression.json275
-rw-r--r--config-model/src/test/java/com/yahoo/searchdefinition/AttributeUtils.java15
-rw-r--r--config-model/src/test/java/com/yahoo/searchdefinition/DocumentReferenceResolverTest.java8
-rw-r--r--config-model/src/test/java/com/yahoo/searchdefinition/ImportedFieldsEnumeratorTest.java66
-rw-r--r--config-model/src/test/java/com/yahoo/searchdefinition/processing/RankProfileSearchFixture.java2
-rw-r--r--config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeResolverTestCase.java2
-rw-r--r--config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithLightGBMTestCase.java88
-rw-r--r--config-model/src/test/java/com/yahoo/vespa/documentmodel/AbstractReferenceFieldTestCase.java35
-rw-r--r--config-model/src/test/java/com/yahoo/vespa/documentmodel/DocumentModelBuilderImportedFieldsTestCase.java55
-rw-r--r--config-model/src/test/java/com/yahoo/vespa/documentmodel/DocumentModelBuilderReferenceTypeTestCase.java25
-rw-r--r--config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/DocumentTypeChangeValidatorTest.java1
-rw-r--r--config-model/src/test/java/com/yahoo/vespa/model/container/xml/JettyContainerModelBuilderTest.java15
-rw-r--r--config-model/src/test/java/com/yahoo/vespa/model/ml/ImportedModelTester.java4
-rw-r--r--config-model/src/test/java/com/yahoo/vespa/model/ml/MlModelsTest.java5
-rw-r--r--config-model/src/test/java/com/yahoo/vespa/model/ml/ModelEvaluationTest.java10
-rw-r--r--config-model/src/test/schema-test-files/services.xml7
-rw-r--r--config-provisioning/src/main/java/com/yahoo/config/provision/zone/RoutingMethod.java17
-rw-r--r--config-provisioning/src/main/java/com/yahoo/config/provision/zone/ZoneFilter.java5
-rw-r--r--config/src/main/java/com/yahoo/config/subscription/ConfigInstanceSerializer.java1
-rw-r--r--config/src/main/java/com/yahoo/config/subscription/ConfigInstanceUtil.java4
-rw-r--r--config/src/main/java/com/yahoo/config/subscription/ConfigInterruptedException.java1
-rw-r--r--config/src/main/java/com/yahoo/config/subscription/ConfigSet.java2
-rw-r--r--config/src/main/java/com/yahoo/config/subscription/ConfigSource.java2
-rw-r--r--config/src/main/java/com/yahoo/config/subscription/ConfigSubscriber.java2
-rw-r--r--config/src/main/java/com/yahoo/config/subscription/DirSource.java3
-rw-r--r--config/src/main/java/com/yahoo/config/subscription/FileSource.java2
-rw-r--r--config/src/main/java/com/yahoo/config/subscription/JarSource.java3
-rw-r--r--config/src/main/java/com/yahoo/config/subscription/RawSource.java2
-rw-r--r--config/src/main/java/com/yahoo/config/subscription/impl/ConfigSetSubscription.java3
-rw-r--r--config/src/main/java/com/yahoo/config/subscription/impl/ConfigSubscription.java2
-rw-r--r--config/src/main/java/com/yahoo/config/subscription/impl/FileConfigSubscription.java2
-rw-r--r--config/src/main/java/com/yahoo/config/subscription/impl/GenericConfigHandle.java2
-rw-r--r--config/src/main/java/com/yahoo/config/subscription/impl/GenericConfigSubscriber.java2
-rw-r--r--config/src/main/java/com/yahoo/config/subscription/impl/GenericJRTConfigSubscription.java2
-rw-r--r--config/src/main/java/com/yahoo/config/subscription/impl/JRTConfigRequester.java2
-rw-r--r--config/src/main/java/com/yahoo/config/subscription/impl/JRTConfigSubscription.java23
-rw-r--r--config/src/main/java/com/yahoo/config/subscription/impl/JarConfigSubscription.java16
-rw-r--r--config/src/main/java/com/yahoo/config/subscription/impl/MockConnection.java1
-rw-r--r--config/src/main/java/com/yahoo/config/subscription/impl/RawConfigSubscription.java8
-rwxr-xr-xconfig/src/main/java/com/yahoo/vespa/config/ConfigKey.java2
-rw-r--r--config/src/main/java/com/yahoo/vespa/config/ConfigPayload.java1
-rw-r--r--config/src/main/java/com/yahoo/vespa/config/ConfigPayloadApplier.java1
-rw-r--r--config/src/main/java/com/yahoo/vespa/config/ConfigPayloadBuilder.java1
-rw-r--r--config/src/main/java/com/yahoo/vespa/config/ConfigTransformer.java1
-rw-r--r--config/src/main/java/com/yahoo/vespa/config/DefaultValueApplier.java15
-rw-r--r--config/src/main/java/com/yahoo/vespa/config/GenerationCounter.java1
-rw-r--r--config/src/main/java/com/yahoo/vespa/config/GenericConfig.java1
-rw-r--r--config/src/main/java/com/yahoo/vespa/config/SlimeUtils.java15
-rw-r--r--config/src/main/java/com/yahoo/vespa/config/buildergen/CompilationTask.java1
-rw-r--r--config/src/main/java/com/yahoo/vespa/config/buildergen/CompiledBuilder.java1
-rw-r--r--config/src/main/java/com/yahoo/vespa/config/buildergen/StringSourceObject.java1
-rw-r--r--config/src/main/java/com/yahoo/vespa/config/protocol/CompressionInfo.java1
-rw-r--r--config/src/main/java/com/yahoo/vespa/config/protocol/CompressionType.java1
-rw-r--r--config/src/main/java/com/yahoo/vespa/config/protocol/DefContent.java8
-rw-r--r--config/src/main/java/com/yahoo/vespa/config/protocol/JRTConfigRequestFactory.java1
-rw-r--r--config/src/main/java/com/yahoo/vespa/config/protocol/NoCopyByteArrayOutputStream.java1
-rw-r--r--config/src/main/java/com/yahoo/vespa/config/protocol/RequestValidation.java1
-rw-r--r--config/src/main/java/com/yahoo/vespa/config/protocol/SlimeClientConfigRequest.java20
-rw-r--r--config/src/main/java/com/yahoo/vespa/config/protocol/SlimeServerConfigRequest.java8
-rw-r--r--config/src/main/java/com/yahoo/vespa/config/protocol/SlimeTraceDeserializer.java1
-rw-r--r--config/src/main/java/com/yahoo/vespa/config/protocol/SlimeTraceSerializer.java2
-rw-r--r--config/src/main/java/com/yahoo/vespa/config/protocol/Trace.java1
-rw-r--r--config/src/main/java/com/yahoo/vespa/config/util/ConfigUtils.java29
-rw-r--r--config/src/test/java/com/yahoo/config/subscription/CfgConfigPayloadBuilderTest.java7
-rw-r--r--config/src/test/java/com/yahoo/config/subscription/ConfigInstancePayloadTest.java1
-rw-r--r--config/src/test/java/com/yahoo/config/subscription/ConfigInstanceSerializationTest.java223
-rw-r--r--config/src/test/java/com/yahoo/config/subscription/ConfigInstanceSerializerTest.java3
-rw-r--r--config/src/test/java/com/yahoo/config/subscription/ConfigInstanceTest.java2
-rw-r--r--config/src/test/java/com/yahoo/config/subscription/ConfigInstanceUtilTest.java1
-rw-r--r--config/src/test/java/com/yahoo/config/subscription/ConfigInterruptedExceptionTest.java2
-rw-r--r--config/src/test/java/com/yahoo/config/subscription/ConfigSetTest.java2
-rwxr-xr-xconfig/src/test/java/com/yahoo/config/subscription/ConfigSourceSetTest.java1
-rw-r--r--config/src/test/java/com/yahoo/config/subscription/ConfigURITest.java2
-rw-r--r--config/src/test/java/com/yahoo/config/subscription/DefaultConfigTest.java1
-rw-r--r--config/src/test/java/com/yahoo/config/subscription/GenericConfigSubscriberTest.java2
-rw-r--r--config/src/test/java/com/yahoo/config/subscription/NamespaceTest.java1
-rw-r--r--config/src/test/java/com/yahoo/config/subscription/UnicodeTest.java1
-rw-r--r--config/src/test/java/com/yahoo/config/subscription/impl/FileConfigSubscriptionTest.java1
-rw-r--r--config/src/test/java/com/yahoo/vespa/config/ConfigBuilderMergeTest.java1
-rwxr-xr-xconfig/src/test/java/com/yahoo/vespa/config/ConfigCacheKeyTest.java1
-rw-r--r--config/src/test/java/com/yahoo/vespa/config/ConfigDefinitionBuilderTest.java1
-rw-r--r--config/src/test/java/com/yahoo/vespa/config/ConfigFileFormatterTest.java1
-rw-r--r--config/src/test/java/com/yahoo/vespa/config/ConfigPayloadBuilderTest.java1
-rw-r--r--config/src/test/java/com/yahoo/vespa/config/ConfigPayloadTest.java3
-rw-r--r--config/src/test/java/com/yahoo/vespa/config/DefaultValueApplierTest.java2
-rw-r--r--config/src/test/java/com/yahoo/vespa/config/ErrorCodeTest.java2
-rw-r--r--config/src/test/java/com/yahoo/vespa/config/ErrorTypeTest.java2
-rw-r--r--config/src/test/java/com/yahoo/vespa/config/GenericConfigBuilderTest.java2
-rw-r--r--config/src/test/java/com/yahoo/vespa/config/LZ4CompressionTest.java1
-rw-r--r--config/src/test/java/com/yahoo/vespa/config/LZ4PayloadCompressorTest.java2
-rw-r--r--config/src/test/java/com/yahoo/vespa/config/RawConfigTest.java4
-rw-r--r--config/src/test/java/com/yahoo/vespa/config/SlimeUtilsTest.java3
-rw-r--r--config/src/test/java/com/yahoo/vespa/config/TimingValuesTest.java1
-rw-r--r--config/src/test/java/com/yahoo/vespa/config/protocol/JRTConfigRequestBase.java15
-rw-r--r--config/src/test/java/com/yahoo/vespa/config/protocol/JRTConfigRequestV3Test.java1
-rw-r--r--config/src/test/java/com/yahoo/vespa/config/protocol/PayloadTest.java1
-rw-r--r--config/src/test/java/com/yahoo/vespa/config/protocol/SlimeTraceSerializerTest.java2
-rw-r--r--config/src/test/java/com/yahoo/vespa/config/protocol/TraceTest.java2
-rw-r--r--configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java5
-rw-r--r--configserver/src/test/java/com/yahoo/vespa/config/server/application/OrchestratorMock.java17
-rw-r--r--container-core/abi-spec.json20
-rw-r--r--container-core/src/main/java/com/yahoo/container/handler/ClustersStatus.java21
-rw-r--r--container-core/src/main/java/com/yahoo/container/handler/VipStatus.java11
-rw-r--r--container-core/src/main/java/com/yahoo/container/jdisc/state/StateMonitor.java35
-rw-r--r--container-core/src/main/resources/configdefinitions/health-monitor.def2
-rw-r--r--container-core/src/test/java/com/yahoo/container/handler/VipStatusTestCase.java86
-rw-r--r--container-core/src/test/java/com/yahoo/container/jdisc/state/StateHandlerTestBase.java3
-rw-r--r--container-disc/abi-spec.json2
-rw-r--r--container-disc/src/main/java/com/yahoo/container/jdisc/AthenzIdentityProviderProvider.java10
-rw-r--r--container-disc/src/main/java/com/yahoo/container/jdisc/athenz/AthenzIdentityProvider.java2
-rw-r--r--container-search/abi-spec.json8
-rw-r--r--container-search/src/main/java/com/yahoo/prelude/Index.java13
-rw-r--r--container-search/src/main/java/com/yahoo/prelude/query/BlockItem.java6
-rw-r--r--container-search/src/main/java/com/yahoo/prelude/querytransform/NormalizingSearcher.java20
-rw-r--r--container-search/src/main/java/com/yahoo/prelude/querytransform/QueryRewrite.java20
-rw-r--r--container-search/src/main/java/com/yahoo/prelude/querytransform/StemmingSearcher.java54
-rw-r--r--container-search/src/main/java/com/yahoo/search/Query.java2
-rw-r--r--container-search/src/main/java/com/yahoo/search/cluster/ClusterMonitor.java11
-rw-r--r--container-search/src/main/java/com/yahoo/search/cluster/ClusterSearcher.java10
-rw-r--r--container-search/src/main/java/com/yahoo/search/cluster/MonitorConfiguration.java2
-rw-r--r--container-search/src/main/java/com/yahoo/search/cluster/NodeManager.java16
-rw-r--r--container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java47
-rw-r--r--container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcInvokerFactory.java9
-rw-r--r--container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcPing.java58
-rw-r--r--container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcPingFactory.java18
-rw-r--r--container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/Node.java14
-rw-r--r--container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/PingFactory.java4
-rw-r--r--container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/Pinger.java12
-rw-r--r--container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/PongHandler.java12
-rw-r--r--container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java88
-rw-r--r--container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileProperties.java2
-rw-r--r--container-search/src/main/java/com/yahoo/search/query/properties/PropertyMap.java2
-rw-r--r--container-search/src/main/java/com/yahoo/search/query/properties/QueryProperties.java2
-rw-r--r--container-search/src/main/java/com/yahoo/search/query/properties/RequestContextProperties.java8
-rw-r--r--container-search/src/main/java/com/yahoo/search/querytransform/VespaLowercasingSearcher.java1
-rw-r--r--container-search/src/test/java/com/yahoo/prelude/fastsearch/test/MockDispatcher.java6
-rw-r--r--container-search/src/test/java/com/yahoo/prelude/querytransform/test/QueryRewriteTestCase.java18
-rw-r--r--container-search/src/test/java/com/yahoo/search/dispatch/DispatcherTest.java19
-rw-r--r--container-search/src/test/java/com/yahoo/search/dispatch/LoadBalancerTest.java8
-rw-r--r--container-search/src/test/java/com/yahoo/search/dispatch/MockSearchCluster.java2
-rw-r--r--container-search/src/test/java/com/yahoo/search/dispatch/searchcluster/SearchClusterTest.java93
-rw-r--r--container-search/src/test/java/com/yahoo/search/query/profile/test/QueryProfileTestCase.java6
-rw-r--r--controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/ServiceRegistry.java4
-rw-r--r--controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/ZtsClientMock.java11
-rw-r--r--controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/ApplicationCertificate.java43
-rw-r--r--controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/EndpointCertificateMock.java (renamed from controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/ApplicationCertificateMock.java)11
-rw-r--r--controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/EndpointCertificateProvider.java (renamed from controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/ApplicationCertificateProvider.java)6
-rw-r--r--controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServer.java13
-rw-r--r--controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/TesterCloud.java9
-rw-r--r--controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/MemoryNameService.java5
-rw-r--r--controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/routing/RoutingGeneratorMock.java16
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java9
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java9
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java35
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java4
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java29
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java3
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/endpointcertificates/EndpointCertificateManager.java79
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java34
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java11
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java21
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java15
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/HtmlResponse.java30
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java8
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/routing/RoutingApiHandler.java29
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java4
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java26
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java16
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java45
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/endpointcertificates/EndpointCertificateManagerTest.java94
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java24
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/SecretStoreMock.java5
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ServiceRegistryMock.java17
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneFilterMock.java25
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneRegistryMock.java64
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OutstandingChangeDeployerTest.java3
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java2
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/RoutingApiTest.java21
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/multi-status-in.json4
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/multi-status-initial.json4
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/multi-status-out.json4
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/policy/deployment-status-in.json2
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/policy/deployment-status-initial.json2
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/policy/deployment-status-out.json2
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/policy/zone-status-in.json2
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/policy/zone-status-initial.json2
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/policy/zone-status-out.json2
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/rotation/deployment-status-in.json2
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/rotation/deployment-status-initial.json2
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/rotation/deployment-status-out.json2
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/rotation/zone-status-in.json2
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/rotation/zone-status-initial.json2
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/rotation/zone-status-out.json2
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java6
-rw-r--r--default_build_settings.cmake16
-rw-r--r--dist/vespa.spec18
-rw-r--r--docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/ContainerResources.java4
-rw-r--r--document/abi-spec.json42
-rwxr-xr-xdocument/src/main/java/com/yahoo/document/DocumentType.java30
-rw-r--r--document/src/main/java/com/yahoo/document/DocumentTypeManagerConfigurer.java7
-rwxr-xr-xdocument/src/main/java/com/yahoo/document/FieldPath.java2
-rw-r--r--document/src/main/java/com/yahoo/document/select/rule/AttributeNode.java32
-rw-r--r--document/src/test/document/documentmanager.importedfields.cfg65
-rw-r--r--document/src/test/java/com/yahoo/document/DocumentTypeManagerTestCase.java30
-rw-r--r--document/src/test/java/com/yahoo/document/select/DocumentSelectorTestCase.java30
-rw-r--r--document/src/tests/documentselectparsertest.cpp28
-rw-r--r--document/src/tests/repo/documenttyperepo_test.cpp41
-rw-r--r--document/src/vespa/document/config/documentmanager.def3
-rw-r--r--document/src/vespa/document/config/documenttypes.def3
-rw-r--r--document/src/vespa/document/datatype/documenttype.cpp71
-rw-r--r--document/src/vespa/document/datatype/documenttype.h51
-rw-r--r--document/src/vespa/document/repo/configbuilder.h18
-rw-r--r--document/src/vespa/document/repo/documenttyperepo.cpp15
-rw-r--r--document/src/vespa/document/select/valuenodes.cpp49
-rw-r--r--eval/src/tests/tensor/dense_xw_product_function/dense_xw_product_function_test.cpp6
-rw-r--r--eval/src/vespa/eval/eval/test/eval_fixture.cpp14
-rw-r--r--eval/src/vespa/eval/eval/test/eval_fixture.h2
-rw-r--r--eval/src/vespa/eval/tensor/dense/dense_dot_product_function.cpp67
-rw-r--r--eval/src/vespa/eval/tensor/dense/dense_dot_product_function.h2
-rw-r--r--eval/src/vespa/eval/tensor/dense/dense_matmul_function.cpp2
-rw-r--r--eval/src/vespa/eval/tensor/dense/dense_xw_product_function.cpp181
-rw-r--r--eval/src/vespa/eval/tensor/dense/dense_xw_product_function.h36
-rw-r--r--flags/src/main/java/com/yahoo/vespa/flags/Flags.java12
-rw-r--r--fnet/src/vespa/fnet/connection.cpp5
-rw-r--r--fnet/src/vespa/fnet/connection.h10
-rw-r--r--jdisc-security-filters/src/main/java/com/yahoo/jdisc/http/filter/security/athenz/RequestResourceMapper.java8
-rw-r--r--jdisc-security-filters/src/main/java/com/yahoo/jdisc/http/filter/security/base/JsonSecurityRequestFilterBase.java9
-rw-r--r--jdisc_http_service/abi-spec.json15
-rw-r--r--jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/impl/ConfiguredSslContextFactoryProvider.java69
-rw-r--r--jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/impl/SslContextFactoryUtils.java32
-rw-r--r--jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/impl/TlsContextBasedProvider.java24
-rw-r--r--jdisc_http_service/src/main/resources/configdefinitions/jdisc.http.connector.def6
-rw-r--r--model-evaluation/abi-spec.json1
-rw-r--r--model-evaluation/src/main/java/ai/vespa/models/evaluation/FunctionEvaluator.java16
-rw-r--r--model-evaluation/src/main/java/ai/vespa/models/handler/ModelsEvaluationHandler.java10
-rw-r--r--model-evaluation/src/test/java/ai/vespa/models/evaluation/MlModelsImportingTest.java19
-rw-r--r--model-evaluation/src/test/java/ai/vespa/models/handler/ModelsEvaluationHandlerTest.java35
-rw-r--r--model-evaluation/src/test/resources/config/models/rank-profiles.cfg3
-rw-r--r--model-integration/src/main/config/model-integration.xml3
-rw-r--r--model-integration/src/main/java/ai/vespa/rankingexpression/importer/lightgbm/LightGBMImporter.java54
-rw-r--r--model-integration/src/main/java/ai/vespa/rankingexpression/importer/lightgbm/LightGBMNode.java67
-rw-r--r--model-integration/src/main/java/ai/vespa/rankingexpression/importer/lightgbm/LightGBMParser.java146
-rw-r--r--model-integration/src/main/java/ai/vespa/rankingexpression/importer/lightgbm/package-info.java5
-rw-r--r--model-integration/src/test/java/ai/vespa/rankingexpression/importer/lightgbm/LightGBMImportEvaluationTestCase.java49
-rw-r--r--model-integration/src/test/java/ai/vespa/rankingexpression/importer/lightgbm/LightGBMTestBase.java42
-rw-r--r--model-integration/src/test/models/lightgbm/classification.json275
-rw-r--r--model-integration/src/test/models/lightgbm/regression.json275
-rwxr-xr-xmodel-integration/src/test/models/lightgbm/train_lightgbm_classification.py54
-rwxr-xr-xmodel-integration/src/test/models/lightgbm/train_lightgbm_regression.py53
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java77
-rw-r--r--node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerTester.java9
-rw-r--r--node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java105
-rw-r--r--node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporter.java22
-rw-r--r--node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java2
-rw-r--r--node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesResponse.java4
-rw-r--r--node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/ContainerConfig.java3
-rw-r--r--node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/OrchestratorMock.java17
-rw-r--r--node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporterTest.java4
-rw-r--r--orchestrator/src/main/java/com/yahoo/vespa/orchestrator/Orchestrator.java9
-rw-r--r--orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorImpl.java13
-rw-r--r--orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ApplicationApiImpl.java19
-rw-r--r--orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ClusterApiImpl.java13
-rw-r--r--orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/InstanceResource.java7
-rw-r--r--orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/InstanceStatusResponse.java1
-rw-r--r--orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/HostInfos.java13
-rw-r--r--orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/HostInfosCache.java11
-rw-r--r--orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/HostStatus.java1
-rw-r--r--orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/MutableStatusRegistry.java14
-rw-r--r--orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/StatusService.java8
-rw-r--r--orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/ZookeeperStatusService.java13
-rw-r--r--orchestrator/src/test/java/com/yahoo/vespa/orchestrator/model/ClusterApiImplTest.java9
-rw-r--r--orchestrator/src/test/java/com/yahoo/vespa/orchestrator/model/ModelTestUtils.java23
-rw-r--r--searchcore/src/tests/proton/common/selectpruner_test.cpp46
-rw-r--r--searchcore/src/vespa/searchcore/proton/common/attributefieldvaluenode.cpp2
-rw-r--r--searchcore/src/vespa/searchcore/proton/common/selectcontext.cpp6
-rw-r--r--searchcore/src/vespa/searchcore/proton/common/selectcontext.h4
-rw-r--r--searchcore/src/vespa/searchcore/proton/common/selectpruner.cpp56
-rw-r--r--searchcore/src/vespa/searchcore/proton/common/selectpruner.h1
-rw-r--r--searchlib/CMakeLists.txt1
-rw-r--r--searchlib/src/tests/attribute/imported_attribute_vector/imported_attribute_vector_test.cpp14
-rw-r--r--searchlib/src/tests/tensor/hnsw_index/CMakeLists.txt9
-rw-r--r--searchlib/src/tests/tensor/hnsw_index/hnsw_index_test.cpp94
-rw-r--r--searchlib/src/vespa/searchlib/attribute/imported_attribute_vector_read_guard.cpp4
-rw-r--r--searchlib/src/vespa/searchlib/attribute/imported_attribute_vector_read_guard.h1
-rw-r--r--searchlib/src/vespa/searchlib/docstore/chunk.cpp2
-rw-r--r--searchlib/src/vespa/searchlib/docstore/writeablefilechunk.cpp16
-rw-r--r--searchlib/src/vespa/searchlib/tensor/CMakeLists.txt2
-rw-r--r--searchlib/src/vespa/searchlib/tensor/doc_vector_access.h21
-rw-r--r--searchlib/src/vespa/searchlib/tensor/hnsw_index.cpp102
-rw-r--r--searchlib/src/vespa/searchlib/tensor/hnsw_index.h45
-rw-r--r--searchlib/src/vespa/searchlib/tensor/hnsw_index_base.cpp135
-rw-r--r--searchlib/src/vespa/searchlib/tensor/hnsw_index_base.h105
-rw-r--r--searchlib/src/vespa/searchlib/tensor/hnsw_index_utils.h47
-rw-r--r--searchlib/src/vespa/searchlib/tensor/hnsw_node.h34
-rw-r--r--searchlib/src/vespa/searchlib/tensor/nearest_neighbor_index.h19
-rw-r--r--security-utils/src/main/java/com/yahoo/security/tls/TlsContext.java2
-rw-r--r--storage/src/vespa/storage/storageserver/CMakeLists.txt3
-rw-r--r--storage/src/vespa/storage/storageserver/fnet_metrics_wrapper.cpp21
-rw-r--r--storage/src/vespa/storage/storageserver/fnet_metrics_wrapper.h22
-rw-r--r--storage/src/vespa/storage/storageserver/storagemetricsset.cpp4
-rw-r--r--storage/src/vespa/storage/storageserver/storagemetricsset.h2
-rw-r--r--vespa-athenz/src/main/java/com/yahoo/vespa/athenz/api/AthenzAccessToken.java8
-rw-r--r--vespa-athenz/src/main/java/com/yahoo/vespa/athenz/api/AthenzRole.java19
-rw-r--r--vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/DefaultZtsClient.java31
-rw-r--r--vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/ZtsClient.java17
-rw-r--r--vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/bindings/AccessTokenResponseEntity.java49
-rw-r--r--vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzIdentityProviderImpl.java27
-rw-r--r--vespa-athenz/src/main/java/com/yahoo/vespa/athenz/tls/AthenzX509CertificateUtils.java5
-rw-r--r--zookeeper-server/zookeeper-server-3.5/src/test/java/com/yahoo/vespa/zookeeper/VespaZooKeeperServerImplTest.java4
337 files changed, 6133 insertions, 1503 deletions
diff --git a/application/src/main/java/com/yahoo/application/Application.java b/application/src/main/java/com/yahoo/application/Application.java
index 5f9b1f51863..d4b1735e4c1 100644
--- a/application/src/main/java/com/yahoo/application/Application.java
+++ b/application/src/main/java/com/yahoo/application/Application.java
@@ -2,6 +2,7 @@
package com.yahoo.application;
import ai.vespa.rankingexpression.importer.configmodelview.MlModelImporter;
+import ai.vespa.rankingexpression.importer.lightgbm.LightGBMImporter;
import ai.vespa.rankingexpression.importer.onnx.OnnxImporter;
import ai.vespa.rankingexpression.importer.tensorflow.TensorFlowImporter;
import ai.vespa.rankingexpression.importer.vespa.VespaImporter;
@@ -117,6 +118,7 @@ public final class Application implements AutoCloseable {
List<MlModelImporter> modelImporters = List.of(new VespaImporter(),
new TensorFlowImporter(),
new OnnxImporter(),
+ new LightGBMImporter(),
new XGBoostImporter());
DeployState deployState = new DeployState.Builder()
.applicationPackage(FilesApplicationPackage.fromFile(path.toFile(), true))
diff --git a/application/src/test/app-packages/model-evaluation/models/lightgbm/regression.json b/application/src/test/app-packages/model-evaluation/models/lightgbm/regression.json
new file mode 100644
index 00000000000..cf0488ecd8b
--- /dev/null
+++ b/application/src/test/app-packages/model-evaluation/models/lightgbm/regression.json
@@ -0,0 +1,275 @@
+{
+ "name": "tree",
+ "version": "v3",
+ "num_class": 1,
+ "num_tree_per_iteration": 1,
+ "label_index": 0,
+ "max_feature_idx": 3,
+ "average_output": false,
+ "objective": "regression",
+ "feature_names": [
+ "numerical_1",
+ "numerical_2",
+ "categorical_1",
+ "categorical_2"
+ ],
+ "monotone_constraints": [],
+ "tree_info": [
+ {
+ "tree_index": 0,
+ "num_leaves": 3,
+ "num_cat": 1,
+ "shrinkage": 1,
+ "tree_structure": {
+ "split_index": 0,
+ "split_feature": 1,
+ "split_gain": 68.5353012084961,
+ "threshold": 0.46643291586559305,
+ "decision_type": "<=",
+ "default_left": true,
+ "missing_type": "NaN",
+ "internal_value": 0,
+ "internal_weight": 0,
+ "internal_count": 1000,
+ "left_child": {
+ "leaf_index": 0,
+ "leaf_value": 2.1594397038037663,
+ "leaf_weight": 469,
+ "leaf_count": 469
+ },
+ "right_child": {
+ "split_index": 1,
+ "split_feature": 3,
+ "split_gain": 41.27640151977539,
+ "threshold": "2||3||4",
+ "decision_type": "==",
+ "default_left": false,
+ "missing_type": "NaN",
+ "internal_value": 0.246035,
+ "internal_weight": 531,
+ "internal_count": 531,
+ "left_child": {
+ "leaf_index": 1,
+ "leaf_value": 2.235297305276056,
+ "leaf_weight": 302,
+ "leaf_count": 302
+ },
+ "right_child": {
+ "leaf_index": 2,
+ "leaf_value": 2.1792953471546546,
+ "leaf_weight": 229,
+ "leaf_count": 229
+ }
+ }
+ }
+ },
+ {
+ "tree_index": 1,
+ "num_leaves": 3,
+ "num_cat": 1,
+ "shrinkage": 0.1,
+ "tree_structure": {
+ "split_index": 0,
+ "split_feature": 2,
+ "split_gain": 64.22250366210938,
+ "threshold": "3||4",
+ "decision_type": "==",
+ "default_left": false,
+ "missing_type": "NaN",
+ "internal_value": 0,
+ "internal_weight": 0,
+ "internal_count": 1000,
+ "left_child": {
+ "leaf_index": 0,
+ "leaf_value": 0.03070842919354316,
+ "leaf_weight": 399,
+ "leaf_count": 399
+ },
+ "right_child": {
+ "split_index": 1,
+ "split_feature": 0,
+ "split_gain": 36.74250030517578,
+ "threshold": 0.5102250691730842,
+ "decision_type": "<=",
+ "default_left": true,
+ "missing_type": "NaN",
+ "internal_value": -0.204906,
+ "internal_weight": 601,
+ "internal_count": 601,
+ "left_child": {
+ "leaf_index": 1,
+ "leaf_value": -0.04439151147520909,
+ "leaf_weight": 315,
+ "leaf_count": 315
+ },
+ "right_child": {
+ "leaf_index": 2,
+ "leaf_value": 0.005117411709368601,
+ "leaf_weight": 286,
+ "leaf_count": 286
+ }
+ }
+ }
+ },
+ {
+ "tree_index": 2,
+ "num_leaves": 3,
+ "num_cat": 0,
+ "shrinkage": 0.1,
+ "tree_structure": {
+ "split_index": 0,
+ "split_feature": 1,
+ "split_gain": 57.1327018737793,
+ "threshold": 0.668665477622446,
+ "decision_type": "<=",
+ "default_left": true,
+ "missing_type": "NaN",
+ "internal_value": 0,
+ "internal_weight": 0,
+ "internal_count": 1000,
+ "left_child": {
+ "split_index": 1,
+ "split_feature": 1,
+ "split_gain": 40.859100341796875,
+ "threshold": 0.008118820676863816,
+ "decision_type": "<=",
+ "default_left": true,
+ "missing_type": "NaN",
+ "internal_value": -0.162926,
+ "internal_weight": 681,
+ "internal_count": 681,
+ "left_child": {
+ "leaf_index": 0,
+ "leaf_value": -0.15361238490967524,
+ "leaf_weight": 21,
+ "leaf_count": 21
+ },
+ "right_child": {
+ "leaf_index": 2,
+ "leaf_value": -0.01192330846157292,
+ "leaf_weight": 660,
+ "leaf_count": 660
+ }
+ },
+ "right_child": {
+ "leaf_index": 1,
+ "leaf_value": 0.03499044894987518,
+ "leaf_weight": 319,
+ "leaf_count": 319
+ }
+ }
+ },
+ {
+ "tree_index": 3,
+ "num_leaves": 3,
+ "num_cat": 1,
+ "shrinkage": 0.1,
+ "tree_structure": {
+ "split_index": 0,
+ "split_feature": 0,
+ "split_gain": 54.77090072631836,
+ "threshold": 0.5201391072644542,
+ "decision_type": "<=",
+ "default_left": true,
+ "missing_type": "NaN",
+ "internal_value": 0,
+ "internal_weight": 0,
+ "internal_count": 1000,
+ "left_child": {
+ "leaf_index": 0,
+ "leaf_value": -0.02141000620783247,
+ "leaf_weight": 543,
+ "leaf_count": 543
+ },
+ "right_child": {
+ "split_index": 1,
+ "split_feature": 2,
+ "split_gain": 27.200700759887695,
+ "threshold": "0||1",
+ "decision_type": "==",
+ "default_left": false,
+ "missing_type": "NaN",
+ "internal_value": 0.255704,
+ "internal_weight": 457,
+ "internal_count": 457,
+ "left_child": {
+ "leaf_index": 1,
+ "leaf_value": -0.004121485787596721,
+ "leaf_weight": 191,
+ "leaf_count": 191
+ },
+ "right_child": {
+ "leaf_index": 2,
+ "leaf_value": 0.04534090904886873,
+ "leaf_weight": 266,
+ "leaf_count": 266
+ }
+ }
+ }
+ },
+ {
+ "tree_index": 4,
+ "num_leaves": 3,
+ "num_cat": 1,
+ "shrinkage": 0.1,
+ "tree_structure": {
+ "split_index": 0,
+ "split_feature": 3,
+ "split_gain": 51.84349822998047,
+ "threshold": "2||3||4",
+ "decision_type": "==",
+ "default_left": false,
+ "missing_type": "NaN",
+ "internal_value": 0,
+ "internal_weight": 0,
+ "internal_count": 1000,
+ "left_child": {
+ "split_index": 1,
+ "split_feature": 1,
+ "split_gain": 39.352699279785156,
+ "threshold": 0.27283279016959255,
+ "decision_type": "<=",
+ "default_left": true,
+ "missing_type": "NaN",
+ "internal_value": 0.188414,
+ "internal_weight": 593,
+ "internal_count": 593,
+ "left_child": {
+ "leaf_index": 0,
+ "leaf_value": -0.01924803254356527,
+ "leaf_weight": 184,
+ "leaf_count": 184
+ },
+ "right_child": {
+ "leaf_index": 2,
+ "leaf_value": 0.03643772842347651,
+ "leaf_weight": 409,
+ "leaf_count": 409
+ }
+ },
+ "right_child": {
+ "leaf_index": 1,
+ "leaf_value": -0.02701711918923075,
+ "leaf_weight": 407,
+ "leaf_count": 407
+ }
+ }
+ }
+ ],
+ "pandas_categorical": [
+ [
+ "a",
+ "b",
+ "c",
+ "d",
+ "e"
+ ],
+ [
+ "i",
+ "j",
+ "k",
+ "l",
+ "m"
+ ]
+ ]
+} \ No newline at end of file
diff --git a/application/src/test/java/com/yahoo/application/container/ContainerModelEvaluationTest.java b/application/src/test/java/com/yahoo/application/container/ContainerModelEvaluationTest.java
index 79510375414..3d7eed1e729 100644
--- a/application/src/test/java/com/yahoo/application/container/ContainerModelEvaluationTest.java
+++ b/application/src/test/java/com/yahoo/application/container/ContainerModelEvaluationTest.java
@@ -45,7 +45,7 @@ public class ContainerModelEvaluationTest {
}
private void assertLoadedModels(JDisc jdisc) {
{
- String expected = "{\"xgboost_xgboost_2_2\":\"http://localhost/model-evaluation/v1/xgboost_xgboost_2_2\",\"onnx_mnist_softmax\":\"http://localhost/model-evaluation/v1/onnx_mnist_softmax\",\"tensorflow_mnist_softmax_saved\":\"http://localhost/model-evaluation/v1/tensorflow_mnist_softmax_saved\",\"tensorflow_mnist_saved\":\"http://localhost/model-evaluation/v1/tensorflow_mnist_saved\",\"vespa_example\":\"http://localhost/model-evaluation/v1/vespa_example\"}";
+ String expected = "{\"xgboost_xgboost_2_2\":\"http://localhost/model-evaluation/v1/xgboost_xgboost_2_2\",\"onnx_mnist_softmax\":\"http://localhost/model-evaluation/v1/onnx_mnist_softmax\",\"tensorflow_mnist_softmax_saved\":\"http://localhost/model-evaluation/v1/tensorflow_mnist_softmax_saved\",\"tensorflow_mnist_saved\":\"http://localhost/model-evaluation/v1/tensorflow_mnist_saved\",\"vespa_example\":\"http://localhost/model-evaluation/v1/vespa_example\",\"lightgbm_regression\":\"http://localhost/model-evaluation/v1/lightgbm_regression\"}";
assertResponse("http://localhost/model-evaluation/v1", expected, jdisc);
}
@@ -55,6 +55,11 @@ public class ContainerModelEvaluationTest {
}
{
+ String expected = "{\"cells\":[{\"address\":{},\"value\":1.9130086820218188}]}";
+ assertResponse("http://localhost/model-evaluation/v1/lightgbm_regression/eval", expected, jdisc);
+ }
+
+ {
// Note: The specific response value here has not been verified
String expected = "{\"cells\":[{\"address\":{\"d0\":\"0\",\"d1\":\"0\"},\"value\":-0.5066885003407351},{\"address\":{\"d0\":\"0\",\"d1\":\"1\"},\"value\":0.3912837743150205},{\"address\":{\"d0\":\"0\",\"d1\":\"2\"},\"value\":-0.12401806321703948},{\"address\":{\"d0\":\"0\",\"d1\":\"3\"},\"value\":-0.7019029168606575},{\"address\":{\"d0\":\"0\",\"d1\":\"4\"},\"value\":0.13120114146441697},{\"address\":{\"d0\":\"0\",\"d1\":\"5\"},\"value\":0.6611923203384626},{\"address\":{\"d0\":\"0\",\"d1\":\"6\"},\"value\":-0.22365810810026446},{\"address\":{\"d0\":\"0\",\"d1\":\"7\"},\"value\":-0.0740018307465809},{\"address\":{\"d0\":\"0\",\"d1\":\"8\"},\"value\":0.056492490256153896},{\"address\":{\"d0\":\"0\",\"d1\":\"9\"},\"value\":-0.18422015072393733}]}";
assertResponse("http://localhost/model-evaluation/v1/tensorflow_mnist_saved/serving_default.y/eval?input=" + inputTensor(), expected, jdisc);
diff --git a/bootstrap-cmake.sh b/bootstrap-cmake.sh
index 5e3e2a42ed5..27c7ed6c21b 100755
--- a/bootstrap-cmake.sh
+++ b/bootstrap-cmake.sh
@@ -55,7 +55,7 @@ cmake3 \
-DJAVA_HOME=${JAVA_HOME:-/usr/lib/jvm/java-openjdk} \
-DCMAKE_PREFIX_PATH="/opt/vespa-deps" \
-DEXTRA_LINK_DIRECTORY="/opt/vespa-deps/lib64;/usr/lib64/llvm$VESPA_LLVM_VERSION/lib" \
- -DEXTRA_INCLUDE_DIRECTORY="/opt/vespa-deps/include;/usr/include/llvm$VESPA_LLVM_VERSION" \
+ -DEXTRA_INCLUDE_DIRECTORY="/opt/vespa-deps/include;/usr/include/llvm$VESPA_LLVM_VERSION;/usr/include/openblas" \
-DCMAKE_INSTALL_RPATH="${VESPA_INSTALL_PREFIX}/lib64;/opt/vespa-deps/lib64;/usr/lib/jvm/java-1.8.0/jre/lib/amd64/server;/usr/lib64/llvm$VESPA_LLVM_VERSION/lib" \
${UNPRIVILEGED_ARGS} \
${EXTRA_CMAKE_ARGS} \
diff --git a/config-model-api/abi-spec.json b/config-model-api/abi-spec.json
index 0f5a5e6271d..43527335802 100644
--- a/config-model-api/abi-spec.json
+++ b/config-model-api/abi-spec.json
@@ -879,7 +879,8 @@
"public java.util.Optional tlsSecrets()",
"public java.util.Optional endpointCertificateSecrets()",
"public abstract double defaultTermwiseLimit()",
- "public abstract boolean useBucketSpaceMetric()"
+ "public abstract boolean useBucketSpaceMetric()",
+ "public boolean useNewAthenzFilter()"
],
"fields": []
},
diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java
index 81ac02a5400..2410de55f86 100644
--- a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java
+++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java
@@ -59,6 +59,7 @@ public interface ModelContext {
default Optional<EndpointCertificateSecrets> endpointCertificateSecrets() { return Optional.empty(); }
double defaultTermwiseLimit();
boolean useBucketSpaceMetric();
+ default boolean useNewAthenzFilter() { return false; }
}
}
diff --git a/config-model/src/main/java/com/yahoo/documentmodel/NewDocumentType.java b/config-model/src/main/java/com/yahoo/documentmodel/NewDocumentType.java
index fc42864f1d0..41a30c4553d 100644
--- a/config-model/src/main/java/com/yahoo/documentmodel/NewDocumentType.java
+++ b/config-model/src/main/java/com/yahoo/documentmodel/NewDocumentType.java
@@ -69,25 +69,37 @@ public final class NewDocumentType extends StructuredDataType implements DataTyp
private final StructDataType body;
private final Set<FieldSet> fieldSets = new LinkedHashSet<>();
private final Set<Name> documentReferences;
+ // Imported fields are virtual and therefore exist outside of the SD's document field definition
+ // block itself. But for features like imported fields in a non-search context (e.g. GC selections)
+ // it is necessary to know that certain identifiers refer to imported fields instead of being unknown
+ // document fields. To achieve this, we track the names of imported fields as part of the document
+ // config itself.
+ private final Set<String> importedFieldNames;
public NewDocumentType(Name name) {
this(name, emptySet());
}
- public NewDocumentType(Name name, Set<Name> documentReferences) {
+ public NewDocumentType(Name name, Set<Name> documentReferences, Set<String> importedFieldNames) {
this(
name,
new StructDataType(name.getName() + ".header"),
new StructDataType(name.getName() + ".body"),
new FieldSets(),
- documentReferences);
+ documentReferences,
+ importedFieldNames);
+ }
+
+ public NewDocumentType(Name name, Set<Name> documentReferences) {
+ this(name, documentReferences, emptySet());
}
public NewDocumentType(Name name,
StructDataType header,
StructDataType body,
FieldSets fs,
- Set<Name> documentReferences) {
+ Set<Name> documentReferences,
+ Set<String> importedFieldNames) {
super(name.getName());
this.name = name;
this.header = header;
@@ -102,6 +114,7 @@ public final class NewDocumentType extends StructuredDataType implements DataTyp
}
}
this.documentReferences = documentReferences;
+ this.importedFieldNames = importedFieldNames;
}
public Name getFullName() {
@@ -389,4 +402,8 @@ public final class NewDocumentType extends StructuredDataType implements DataTyp
return documentReferences;
}
+ public Set<String> getImportedFieldNames() {
+ return importedFieldNames;
+ }
+
}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/DocumentModelBuilder.java b/config-model/src/main/java/com/yahoo/searchdefinition/DocumentModelBuilder.java
index f0b1b427531..d3a78321106 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/DocumentModelBuilder.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/DocumentModelBuilder.java
@@ -18,6 +18,7 @@ import com.yahoo.documentmodel.VespaDocumentType;
import com.yahoo.searchdefinition.document.Attribute;
import com.yahoo.searchdefinition.document.SDDocumentType;
import com.yahoo.searchdefinition.document.SDField;
+import com.yahoo.searchdefinition.document.TemporaryImportedFields;
import com.yahoo.searchdefinition.document.annotation.SDAnnotationType;
import com.yahoo.searchdefinition.document.annotation.TemporaryAnnotationReferenceDataType;
import com.yahoo.vespa.documentmodel.DocumentModel;
@@ -27,6 +28,7 @@ import com.yahoo.vespa.documentmodel.SearchField;
import java.util.ArrayList;
import java.util.Collection;
+import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedList;
@@ -338,7 +340,8 @@ public class DocumentModelBuilder {
sdoc.getDocumentType().contentStruct(),
sdoc.getDocumentType().getBodyType(),
sdoc.getFieldSets(),
- convertDocumentReferencesToNames(sdoc.getDocumentReferences()));
+ convertDocumentReferencesToNames(sdoc.getDocumentReferences()),
+ convertTemporaryImportedFieldsToNames(sdoc.getTemporaryImportedFields()));
for (SDDocumentType n : sdoc.getInheritedTypes()) {
NewDocumentType.Name name = new NewDocumentType.Name(n.getName());
NewDocumentType inherited = model.getDocumentManager().getDocumentType(name);
@@ -404,6 +407,13 @@ public class DocumentModelBuilder {
.collect(toSet());
}
+ private static Set<String> convertTemporaryImportedFieldsToNames(TemporaryImportedFields importedFields) {
+ if (importedFields == null) {
+ return emptySet();
+ }
+ return Collections.unmodifiableSet(importedFields.fields().keySet());
+ }
+
private static void extractDataTypesFromFields(NewDocumentType dt, Collection<Field> fields) {
for (Field f : fields) {
DataType type = f.getDataType();
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/ImportedFieldsEnumerator.java b/config-model/src/main/java/com/yahoo/searchdefinition/ImportedFieldsEnumerator.java
new file mode 100644
index 00000000000..ee4fc41f2f9
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/ImportedFieldsEnumerator.java
@@ -0,0 +1,33 @@
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition;
+
+import com.yahoo.searchdefinition.document.SDDocumentType;
+import com.yahoo.searchdefinition.document.TemporaryImportedFields;
+
+import java.util.Collections;
+import java.util.List;
+
+/**
+ * Enumerates and emplaces a set of all imported fields into a SDDocumentType from
+ * its corresponding Search instance.
+ */
+public class ImportedFieldsEnumerator {
+
+ private final List<Search> searchDefinitions;
+
+ public ImportedFieldsEnumerator(List<Search> searchDefinitions) {
+ this.searchDefinitions = searchDefinitions;
+ }
+
+ public void enumerateImportedFields(SDDocumentType documentType) {
+ var search = this.searchDefinitions.stream()
+ .filter(s -> s.getDocument() != null)
+ .filter(s -> s.getDocument().getName().equals(documentType.getName()))
+ .findFirst();
+ if (search.isEmpty()) {
+ return; // No imported fields present.
+ }
+ search.get().temporaryImportedFields().ifPresent(documentType::setTemporaryImportedFields);
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java
index fa196cd3bbf..23eb814de81 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java
@@ -788,8 +788,7 @@ public class RankProfile implements Cloneable {
type = existingType.dimensionwiseGeneralizationWith(type).orElseThrow( () ->
new IllegalArgumentException(queryProfileType + " contains query feature " + feature.get() +
" with type " + field.getType().asTensorType() +
- ", but this is already defined " +
- "in another query profile with type " +
+ ", but this is already defined in another query profile with type " +
context.getType(feature.get())));
context.setType(feature.get(), type);
}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/SearchBuilder.java b/config-model/src/main/java/com/yahoo/searchdefinition/SearchBuilder.java
index 949539ff99f..eb68e6af203 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/SearchBuilder.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/SearchBuilder.java
@@ -230,20 +230,22 @@ public class SearchBuilder {
sdocs.add(search.getDocument());
}
}
- SDDocumentTypeOrderer orderer = new SDDocumentTypeOrderer(sdocs, deployLogger);
+ var orderer = new SDDocumentTypeOrderer(sdocs, deployLogger);
orderer.process();
for (SDDocumentType sdoc : orderer.getOrdered()) {
new FieldOperationApplierForStructs().process(sdoc);
new FieldOperationApplier().process(sdoc);
}
- DocumentReferenceResolver resolver = new DocumentReferenceResolver(searchList);
+ var resolver = new DocumentReferenceResolver(searchList);
sdocs.forEach(resolver::resolveReferences);
+ var importedFieldsEnumerator = new ImportedFieldsEnumerator(searchList);
+ sdocs.forEach(importedFieldsEnumerator::enumerateImportedFields);
if (validate)
new DocumentGraphValidator().validateDocumentGraph(sdocs);
- DocumentModelBuilder builder = new DocumentModelBuilder(model);
+ var builder = new DocumentModelBuilder(model);
for (Search search : new SearchOrderer().order(searchList)) {
new FieldOperationApplierForSearch().process(search); // TODO: Why is this not in the regular list?
process(search, deployLogger, new QueryProfiles(queryProfileRegistry, deployLogger), validate);
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/SDDocumentType.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/SDDocumentType.java
index 414a605c621..b3f98cc6f26 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/document/SDDocumentType.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/SDDocumentType.java
@@ -47,6 +47,7 @@ public class SDDocumentType implements Cloneable, Serializable {
private FieldSets fieldSets;
// Document references
private Optional<DocumentReferences> documentReferences = Optional.empty();
+ private TemporaryImportedFields temporaryImportedFields;
static {
VESPA_DOCUMENT = new SDDocumentType(VespaDocumentType.INSTANCE.getFullName().getName());
@@ -58,6 +59,7 @@ public class SDDocumentType implements Cloneable, Serializable {
type.docType = docType.clone();
type.inheritedTypes.putAll(inheritedTypes);
type.structType = structType;
+ // TODO this isn't complete; should it be..?!
return type;
}
@@ -334,4 +336,11 @@ public class SDDocumentType implements Cloneable, Serializable {
this.documentReferences = Optional.of(documentReferences);
}
+ public TemporaryImportedFields getTemporaryImportedFields() {
+ return temporaryImportedFields;
+ }
+
+ public void setTemporaryImportedFields(TemporaryImportedFields temporaryImportedFields) {
+ this.temporaryImportedFields = temporaryImportedFields;
+ }
}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/ExpressionTransforms.java b/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/ExpressionTransforms.java
index 6fdf448a39b..a6707ec7ac0 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/ExpressionTransforms.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/ExpressionTransforms.java
@@ -27,6 +27,7 @@ public class ExpressionTransforms {
ImmutableList.of(new TensorFlowFeatureConverter(),
new OnnxFeatureConverter(),
new XgboostFeatureConverter(),
+ new LightGBMFeatureConverter(),
new ConstantDereferencer(),
new ConstantTensorTransformer(),
new FunctionInliner(),
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/LightGBMFeatureConverter.java b/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/LightGBMFeatureConverter.java
new file mode 100644
index 00000000000..5bde627dc0a
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/LightGBMFeatureConverter.java
@@ -0,0 +1,59 @@
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.expressiontransforms;
+
+import com.yahoo.path.Path;
+import com.yahoo.searchlib.rankingexpression.rule.Arguments;
+import com.yahoo.searchlib.rankingexpression.rule.CompositeNode;
+import com.yahoo.searchlib.rankingexpression.rule.ExpressionNode;
+import com.yahoo.searchlib.rankingexpression.rule.ReferenceNode;
+import com.yahoo.searchlib.rankingexpression.transform.ExpressionTransformer;
+import com.yahoo.vespa.model.ml.ConvertedModel;
+import com.yahoo.vespa.model.ml.FeatureArguments;
+
+import java.io.UncheckedIOException;
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * Replaces instances of the lightgbm(model-path) pseudofeature with the
+ * native Vespa ranking expression implementing the same computation.
+ *
+ * @author lesters
+ */
+public class LightGBMFeatureConverter extends ExpressionTransformer<RankProfileTransformContext> {
+
+ /** A cache of imported models indexed by model path. This avoids importing the same model multiple times. */
+ private final Map<Path, ConvertedModel> convertedLightGBMModels = new HashMap<>();
+
+ @Override
+ public ExpressionNode transform(ExpressionNode node, RankProfileTransformContext context) {
+ if (node instanceof ReferenceNode)
+ return transformFeature((ReferenceNode) node, context);
+ else if (node instanceof CompositeNode)
+ return super.transformChildren((CompositeNode) node, context);
+ else
+ return node;
+ }
+
+ private ExpressionNode transformFeature(ReferenceNode feature, RankProfileTransformContext context) {
+ if ( ! feature.getName().equals("lightgbm")) return feature;
+
+ try {
+ FeatureArguments arguments = asFeatureArguments(feature.getArguments());
+ ConvertedModel convertedModel =
+ convertedLightGBMModels.computeIfAbsent(arguments.path(),
+ path -> ConvertedModel.fromSourceOrStore(path, true, context));
+ return convertedModel.expression(arguments, context);
+ } catch (IllegalArgumentException | UncheckedIOException e) {
+ throw new IllegalArgumentException("Could not use LightGBM model from " + feature, e);
+ }
+ }
+
+ private FeatureArguments asFeatureArguments(Arguments arguments) {
+ if (arguments.size() != 1)
+ throw new IllegalArgumentException("A lightgbm node must take a single argument pointing to " +
+ "the LightGBM model file under [application]/models");
+ return new FeatureArguments(arguments);
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImportedFieldsResolver.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImportedFieldsResolver.java
index d6c334ee80b..6d3de23238d 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImportedFieldsResolver.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImportedFieldsResolver.java
@@ -64,7 +64,7 @@ public class ImportedFieldsResolver extends Processor {
}
private void resolveImportedPositionField(TemporaryImportedField importedField, DocumentReference reference,
- ImmutableSDField targetField, boolean validate) {
+ ImmutableSDField targetField, boolean validate) {
TemporaryImportedField importedZCurveField = new TemporaryImportedField(PositionDataType.getZCurveFieldName(importedField.fieldName()),
reference.referenceField().getName(), PositionDataType.getZCurveFieldName(targetField.getName()));
ImmutableSDField targetZCurveField = getTargetField(importedZCurveField, reference);
@@ -175,7 +175,7 @@ public class ImportedFieldsResolver extends Processor {
}
private void validateTargetField(TemporaryImportedField importedField,
- ImmutableSDField targetField, DocumentReference reference) {
+ ImmutableSDField targetField, DocumentReference reference) {
if (!targetField.doesAttributing()) {
fail(importedField, targetFieldAsString(targetField.getName(), reference) +
": Is not an attribute field. Only attribute fields supported");
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeResolver.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeResolver.java
index 9a9e9bbba5f..89b8889b4ae 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeResolver.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeResolver.java
@@ -15,7 +15,10 @@ import com.yahoo.tensor.TensorType;
import com.yahoo.tensor.evaluation.TypeContext;
import com.yahoo.vespa.model.container.search.QueryProfiles;
+import java.util.HashMap;
+import java.util.HashSet;
import java.util.Map;
+import java.util.Set;
import java.util.logging.Level;
/**
@@ -45,9 +48,10 @@ public class RankingExpressionTypeResolver extends Processor {
public void process(boolean validate, boolean documentsOnly) {
if (documentsOnly) return;
+ Set<Reference> warnedAbout = new HashSet<>();
for (RankProfile profile : rankProfileRegistry.rankProfilesOf(search)) {
try {
- resolveTypesIn(profile, validate);
+ resolveTypesIn(profile, validate, warnedAbout);
}
catch (IllegalArgumentException e) {
throw new IllegalArgumentException("In " + (search != null ? search + ", " : "") + profile, e);
@@ -60,7 +64,7 @@ public class RankingExpressionTypeResolver extends Processor {
*
* @throws IllegalArgumentException if validate is true and the given rank profile does not produce valid types
*/
- private void resolveTypesIn(RankProfile profile, boolean validate) {
+ private void resolveTypesIn(RankProfile profile, boolean validate, Set<Reference> warnedAbout) {
MapEvaluationTypeContext context = profile.typeContext(queryProfiles);
for (Map.Entry<String, RankProfile.RankingExpressionFunction> function : profile.getFunctions().entrySet()) {
ExpressionFunction expressionFunction = function.getValue().function();
@@ -83,10 +87,14 @@ public class RankingExpressionTypeResolver extends Processor {
profile.getSummaryFeatures().forEach(f -> resolveType(f, "summary feature " + f, context));
ensureValidDouble(profile.getFirstPhaseRanking(), "first-phase expression", context);
ensureValidDouble(profile.getSecondPhaseRanking(), "second-phase expression", context);
- if ( context.tensorsAreUsed() && ! context.queryFeaturesNotDeclared().isEmpty()) {
- deployLogger.log(Level.WARNING, "The following query features are not declared in query profile " +
+ if ( context.tensorsAreUsed() &&
+ ! context.queryFeaturesNotDeclared().isEmpty() &&
+ ! warnedAbout.containsAll(context.queryFeaturesNotDeclared())) {
+ deployLogger.log(Level.WARNING, "The following query features used in '" + profile.getName() +
+ "' are not declared in query profile " +
"types and will be interpreted as scalars, not tensors: " +
context.queryFeaturesNotDeclared());
+ warnedAbout.addAll(context.queryFeaturesNotDeclared());
}
}
}
diff --git a/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentManager.java b/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentManager.java
index 02d500931d7..e0d015cc8b2 100644
--- a/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentManager.java
+++ b/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentManager.java
@@ -62,6 +62,7 @@ public class DocumentManager {
}
}
}
+
private void buildConfig(Collection<AnnotationType> types, DocumentmanagerConfig.Builder builder) {
for (AnnotationType type : types) {
DocumentmanagerConfig.Annotationtype.Builder atb = new DocumentmanagerConfig.Annotationtype.Builder();
@@ -110,6 +111,7 @@ public class DocumentManager {
doc.inherits(new Datatype.Documenttype.Inherits.Builder().name(inherited.getName()));
}
buildConfig(dt.getFieldSets(), doc);
+ buildImportedFieldsConfig(dt.getImportedFieldNames(), doc);
} else if (type instanceof TemporaryStructuredDataType) {
//Ignored
} else if (type instanceof StructDataType) {
@@ -164,4 +166,12 @@ public class DocumentManager {
doc.fieldsets(fs.getName(), new Datatype.Documenttype.Fieldsets.Builder().fields(fs.getFieldNames()));
}
+ private void buildImportedFieldsConfig(Collection<String> fieldNames, Datatype.Documenttype.Builder builder) {
+ for (String fieldName : fieldNames) {
+ var ib = new DocumentmanagerConfig.Datatype.Documenttype.Importedfield.Builder();
+ ib.name(fieldName);
+ builder.importedfield(ib);
+ }
+ }
+
}
diff --git a/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentTypes.java b/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentTypes.java
index d4228b52746..e6bf826dccc 100644
--- a/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentTypes.java
+++ b/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentTypes.java
@@ -58,6 +58,7 @@ public class DocumentTypes {
buildConfig(annotation, atb);
}
buildConfig(documentType.getFieldSets(), db);
+ buildImportedFieldsConfig(documentType.getImportedFieldNames(), db);
builder.documenttype(db);
}
@@ -120,6 +121,14 @@ public class DocumentTypes {
}
}
+ private void buildImportedFieldsConfig(Collection<String> fieldNames, DocumenttypesConfig.Documenttype.Builder builder) {
+ for (String fieldName : fieldNames) {
+ var ib = new DocumenttypesConfig.Documenttype.Importedfield.Builder();
+ ib.name(fieldName);
+ builder.importedfield(ib);
+ }
+ }
+
private void buildConfig(StructDataType type,
DocumenttypesConfig.Documenttype.Datatype.Builder dataTypeBuilder,
DocumenttypesConfig.Documenttype.Builder documentBuilder,
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java
index 57d938f8a71..62662bde2ad 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java
@@ -72,6 +72,9 @@ public class VespaMetricSet {
metrics.add(new Metric("vds.server.network.tls-connections-broken"));
metrics.add(new Metric("vds.server.network.failed-tls-config-reloads"));
+ // C++ Fnet metrics
+ metrics.add(new Metric("vds.server.fnet.num-connections"));
+
return metrics;
}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/http/ssl/ConfiguredFilebasedSslProvider.java b/config-model/src/main/java/com/yahoo/vespa/model/container/http/ssl/ConfiguredFilebasedSslProvider.java
index 4f84a01ff94..4a331718985 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/container/http/ssl/ConfiguredFilebasedSslProvider.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/http/ssl/ConfiguredFilebasedSslProvider.java
@@ -8,6 +8,7 @@ import com.yahoo.jdisc.http.ssl.impl.ConfiguredSslContextFactoryProvider;
import com.yahoo.osgi.provider.model.ComponentModel;
import com.yahoo.vespa.model.container.component.SimpleComponent;
+import java.util.List;
import java.util.Optional;
import static com.yahoo.component.ComponentSpecification.fromString;
@@ -16,6 +17,7 @@ import static com.yahoo.component.ComponentSpecification.fromString;
* Configure SSL using file references
*
* @author mortent
+ * @author bjorncs
*/
public class ConfiguredFilebasedSslProvider extends SimpleComponent implements ConnectorConfig.Producer {
public static final String COMPONENT_ID_PREFIX = "configured-ssl-provider@";
@@ -26,8 +28,16 @@ public class ConfiguredFilebasedSslProvider extends SimpleComponent implements C
private final String certificatePath;
private final String caCertificatePath;
private final ConnectorConfig.Ssl.ClientAuth.Enum clientAuthentication;
+ private final List<String> cipherSuites;
+ private final List<String> protocolVersions;
- public ConfiguredFilebasedSslProvider(String servername, String privateKeyPath, String certificatePath, String caCertificatePath, String clientAuthentication) {
+ public ConfiguredFilebasedSslProvider(String servername,
+ String privateKeyPath,
+ String certificatePath,
+ String caCertificatePath,
+ String clientAuthentication,
+ List<String> cipherSuites,
+ List<String> protocolVersions) {
super(new ComponentModel(
new BundleInstantiationSpecification(new ComponentId(COMPONENT_ID_PREFIX+servername),
fromString(COMPONENT_CLASS),
@@ -36,15 +46,21 @@ public class ConfiguredFilebasedSslProvider extends SimpleComponent implements C
this.certificatePath = certificatePath;
this.caCertificatePath = caCertificatePath;
this.clientAuthentication = mapToConfigEnum(clientAuthentication);
+ this.cipherSuites = cipherSuites;
+ this.protocolVersions = protocolVersions;
}
@Override
public void getConfig(ConnectorConfig.Builder builder) {
- builder.ssl.enabled(true);
- builder.ssl.privateKeyFile(privateKeyPath);
- builder.ssl.certificateFile(certificatePath);
- builder.ssl.caCertificateFile(Optional.ofNullable(caCertificatePath).orElse(""));
- builder.ssl.clientAuth(clientAuthentication);
+ builder.ssl(
+ new ConnectorConfig.Ssl.Builder()
+ .enabled(true)
+ .privateKeyFile(privateKeyPath)
+ .certificateFile(certificatePath)
+ .caCertificateFile(Optional.ofNullable(caCertificatePath).orElse(""))
+ .clientAuth(clientAuthentication)
+ .enabledCipherSuites(cipherSuites)
+ .enabledProtocols(protocolVersions));
}
public SimpleComponent getComponent() {
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/JettyConnectorBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/JettyConnectorBuilder.java
index db831a1ec2f..562026ab4dd 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/JettyConnectorBuilder.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/JettyConnectorBuilder.java
@@ -9,13 +9,17 @@ import com.yahoo.vespa.model.builder.xml.dom.ModelElement;
import com.yahoo.vespa.model.builder.xml.dom.VespaDomBuilder;
import com.yahoo.vespa.model.container.component.SimpleComponent;
import com.yahoo.vespa.model.container.http.ConnectorFactory;
-import com.yahoo.vespa.model.container.http.ssl.CustomSslProvider;
import com.yahoo.vespa.model.container.http.ssl.ConfiguredFilebasedSslProvider;
+import com.yahoo.vespa.model.container.http.ssl.CustomSslProvider;
import com.yahoo.vespa.model.container.http.ssl.DefaultSslProvider;
import org.w3c.dom.Element;
+import java.util.Arrays;
+import java.util.List;
import java.util.Optional;
+import static java.util.stream.Collectors.toList;
+
/**
* @author Einar M R Rosenvinge
* @author mortent
@@ -40,12 +44,16 @@ public class JettyConnectorBuilder extends VespaDomBuilder.DomConfigProducerBuil
String certificateFile = XML.getValue(XML.getChild(sslConfigurator, "certificate-file"));
Optional<String> caCertificateFile = XmlHelper.getOptionalChildValue(sslConfigurator, "ca-certificates-file");
Optional<String> clientAuthentication = XmlHelper.getOptionalChildValue(sslConfigurator, "client-authentication");
+ List<String> cipherSuites = extractOptionalCommaSeparatedList(sslConfigurator, "cipher-suites");
+ List<String> protocols = extractOptionalCommaSeparatedList(sslConfigurator, "protocols");
return new ConfiguredFilebasedSslProvider(
serverName,
privateKeyFile,
certificateFile,
caCertificateFile.orElse(null),
- clientAuthentication.orElse(null));
+ clientAuthentication.orElse(null),
+ cipherSuites,
+ protocols);
} else if (sslProviderConfigurator != null) {
String className = sslProviderConfigurator.getAttribute("class");
String bundle = sslProviderConfigurator.getAttribute("bundle");
@@ -55,4 +63,13 @@ public class JettyConnectorBuilder extends VespaDomBuilder.DomConfigProducerBuil
}
}
+ private static List<String> extractOptionalCommaSeparatedList(Element sslElement, String listElementName) {
+ return XmlHelper.getOptionalChildValue(sslElement, listElementName)
+ .map(element ->
+ Arrays.stream(element.split(","))
+ .filter(listEntry -> !listEntry.isBlank())
+ .map(String::trim)
+ .collect(toList()))
+ .orElse(List.of());
+ }
}
diff --git a/config-model/src/main/resources/schema/common.rnc b/config-model/src/main/resources/schema/common.rnc
index e3ad942e7b3..c47983adc12 100644
--- a/config-model/src/main/resources/schema/common.rnc
+++ b/config-model/src/main/resources/schema/common.rnc
@@ -37,7 +37,8 @@ OptionalDedicatedNodes = element nodes {
attribute required { xsd:boolean }? &
attribute docker-image { xsd:string }? &
attribute dedicated { xsd:boolean }? &
- attribute exclusive { xsd:boolean }?
+ attribute exclusive { xsd:boolean }? &
+ Resources?
}
GenericConfig = element config {
diff --git a/config-model/src/main/resources/schema/containercluster.rnc b/config-model/src/main/resources/schema/containercluster.rnc
index 6e4346d96ee..726fa849c00 100644
--- a/config-model/src/main/resources/schema/containercluster.rnc
+++ b/config-model/src/main/resources/schema/containercluster.rnc
@@ -96,7 +96,9 @@ Ssl = element ssl {
element private-key-file { string } &
element certificate-file { string } &
element ca-certificates-file { string }? &
- element client-authentication { string "disabled" | string "want" | string "need" }?
+ element client-authentication { string "disabled" | string "want" | string "need" }? &
+ element cipher-suites { string }? &
+ element protocols { string }?
}
SslProvider = element ssl-provider {
diff --git a/config-model/src/test/cfg/application/ml_models/models/lightgbm_regression.json b/config-model/src/test/cfg/application/ml_models/models/lightgbm_regression.json
new file mode 100644
index 00000000000..cf0488ecd8b
--- /dev/null
+++ b/config-model/src/test/cfg/application/ml_models/models/lightgbm_regression.json
@@ -0,0 +1,275 @@
+{
+ "name": "tree",
+ "version": "v3",
+ "num_class": 1,
+ "num_tree_per_iteration": 1,
+ "label_index": 0,
+ "max_feature_idx": 3,
+ "average_output": false,
+ "objective": "regression",
+ "feature_names": [
+ "numerical_1",
+ "numerical_2",
+ "categorical_1",
+ "categorical_2"
+ ],
+ "monotone_constraints": [],
+ "tree_info": [
+ {
+ "tree_index": 0,
+ "num_leaves": 3,
+ "num_cat": 1,
+ "shrinkage": 1,
+ "tree_structure": {
+ "split_index": 0,
+ "split_feature": 1,
+ "split_gain": 68.5353012084961,
+ "threshold": 0.46643291586559305,
+ "decision_type": "<=",
+ "default_left": true,
+ "missing_type": "NaN",
+ "internal_value": 0,
+ "internal_weight": 0,
+ "internal_count": 1000,
+ "left_child": {
+ "leaf_index": 0,
+ "leaf_value": 2.1594397038037663,
+ "leaf_weight": 469,
+ "leaf_count": 469
+ },
+ "right_child": {
+ "split_index": 1,
+ "split_feature": 3,
+ "split_gain": 41.27640151977539,
+ "threshold": "2||3||4",
+ "decision_type": "==",
+ "default_left": false,
+ "missing_type": "NaN",
+ "internal_value": 0.246035,
+ "internal_weight": 531,
+ "internal_count": 531,
+ "left_child": {
+ "leaf_index": 1,
+ "leaf_value": 2.235297305276056,
+ "leaf_weight": 302,
+ "leaf_count": 302
+ },
+ "right_child": {
+ "leaf_index": 2,
+ "leaf_value": 2.1792953471546546,
+ "leaf_weight": 229,
+ "leaf_count": 229
+ }
+ }
+ }
+ },
+ {
+ "tree_index": 1,
+ "num_leaves": 3,
+ "num_cat": 1,
+ "shrinkage": 0.1,
+ "tree_structure": {
+ "split_index": 0,
+ "split_feature": 2,
+ "split_gain": 64.22250366210938,
+ "threshold": "3||4",
+ "decision_type": "==",
+ "default_left": false,
+ "missing_type": "NaN",
+ "internal_value": 0,
+ "internal_weight": 0,
+ "internal_count": 1000,
+ "left_child": {
+ "leaf_index": 0,
+ "leaf_value": 0.03070842919354316,
+ "leaf_weight": 399,
+ "leaf_count": 399
+ },
+ "right_child": {
+ "split_index": 1,
+ "split_feature": 0,
+ "split_gain": 36.74250030517578,
+ "threshold": 0.5102250691730842,
+ "decision_type": "<=",
+ "default_left": true,
+ "missing_type": "NaN",
+ "internal_value": -0.204906,
+ "internal_weight": 601,
+ "internal_count": 601,
+ "left_child": {
+ "leaf_index": 1,
+ "leaf_value": -0.04439151147520909,
+ "leaf_weight": 315,
+ "leaf_count": 315
+ },
+ "right_child": {
+ "leaf_index": 2,
+ "leaf_value": 0.005117411709368601,
+ "leaf_weight": 286,
+ "leaf_count": 286
+ }
+ }
+ }
+ },
+ {
+ "tree_index": 2,
+ "num_leaves": 3,
+ "num_cat": 0,
+ "shrinkage": 0.1,
+ "tree_structure": {
+ "split_index": 0,
+ "split_feature": 1,
+ "split_gain": 57.1327018737793,
+ "threshold": 0.668665477622446,
+ "decision_type": "<=",
+ "default_left": true,
+ "missing_type": "NaN",
+ "internal_value": 0,
+ "internal_weight": 0,
+ "internal_count": 1000,
+ "left_child": {
+ "split_index": 1,
+ "split_feature": 1,
+ "split_gain": 40.859100341796875,
+ "threshold": 0.008118820676863816,
+ "decision_type": "<=",
+ "default_left": true,
+ "missing_type": "NaN",
+ "internal_value": -0.162926,
+ "internal_weight": 681,
+ "internal_count": 681,
+ "left_child": {
+ "leaf_index": 0,
+ "leaf_value": -0.15361238490967524,
+ "leaf_weight": 21,
+ "leaf_count": 21
+ },
+ "right_child": {
+ "leaf_index": 2,
+ "leaf_value": -0.01192330846157292,
+ "leaf_weight": 660,
+ "leaf_count": 660
+ }
+ },
+ "right_child": {
+ "leaf_index": 1,
+ "leaf_value": 0.03499044894987518,
+ "leaf_weight": 319,
+ "leaf_count": 319
+ }
+ }
+ },
+ {
+ "tree_index": 3,
+ "num_leaves": 3,
+ "num_cat": 1,
+ "shrinkage": 0.1,
+ "tree_structure": {
+ "split_index": 0,
+ "split_feature": 0,
+ "split_gain": 54.77090072631836,
+ "threshold": 0.5201391072644542,
+ "decision_type": "<=",
+ "default_left": true,
+ "missing_type": "NaN",
+ "internal_value": 0,
+ "internal_weight": 0,
+ "internal_count": 1000,
+ "left_child": {
+ "leaf_index": 0,
+ "leaf_value": -0.02141000620783247,
+ "leaf_weight": 543,
+ "leaf_count": 543
+ },
+ "right_child": {
+ "split_index": 1,
+ "split_feature": 2,
+ "split_gain": 27.200700759887695,
+ "threshold": "0||1",
+ "decision_type": "==",
+ "default_left": false,
+ "missing_type": "NaN",
+ "internal_value": 0.255704,
+ "internal_weight": 457,
+ "internal_count": 457,
+ "left_child": {
+ "leaf_index": 1,
+ "leaf_value": -0.004121485787596721,
+ "leaf_weight": 191,
+ "leaf_count": 191
+ },
+ "right_child": {
+ "leaf_index": 2,
+ "leaf_value": 0.04534090904886873,
+ "leaf_weight": 266,
+ "leaf_count": 266
+ }
+ }
+ }
+ },
+ {
+ "tree_index": 4,
+ "num_leaves": 3,
+ "num_cat": 1,
+ "shrinkage": 0.1,
+ "tree_structure": {
+ "split_index": 0,
+ "split_feature": 3,
+ "split_gain": 51.84349822998047,
+ "threshold": "2||3||4",
+ "decision_type": "==",
+ "default_left": false,
+ "missing_type": "NaN",
+ "internal_value": 0,
+ "internal_weight": 0,
+ "internal_count": 1000,
+ "left_child": {
+ "split_index": 1,
+ "split_feature": 1,
+ "split_gain": 39.352699279785156,
+ "threshold": 0.27283279016959255,
+ "decision_type": "<=",
+ "default_left": true,
+ "missing_type": "NaN",
+ "internal_value": 0.188414,
+ "internal_weight": 593,
+ "internal_count": 593,
+ "left_child": {
+ "leaf_index": 0,
+ "leaf_value": -0.01924803254356527,
+ "leaf_weight": 184,
+ "leaf_count": 184
+ },
+ "right_child": {
+ "leaf_index": 2,
+ "leaf_value": 0.03643772842347651,
+ "leaf_weight": 409,
+ "leaf_count": 409
+ }
+ },
+ "right_child": {
+ "leaf_index": 1,
+ "leaf_value": -0.02701711918923075,
+ "leaf_weight": 407,
+ "leaf_count": 407
+ }
+ }
+ }
+ ],
+ "pandas_categorical": [
+ [
+ "a",
+ "b",
+ "c",
+ "d",
+ "e"
+ ],
+ [
+ "i",
+ "j",
+ "k",
+ "l",
+ "m"
+ ]
+ ]
+} \ No newline at end of file
diff --git a/config-model/src/test/cfg/application/ml_models/searchdefinitions/test.sd b/config-model/src/test/cfg/application/ml_models/searchdefinitions/test.sd
index ab5e42f983d..247df8a0241 100644
--- a/config-model/src/test/cfg/application/ml_models/searchdefinitions/test.sd
+++ b/config-model/src/test/cfg/application/ml_models/searchdefinitions/test.sd
@@ -33,8 +33,12 @@ search test {
expression: xgboost("xgboost_2_2")
}
+ function my_lightgbm() {
+ expression: lightgbm("lightgbm_regression")
+ }
+
first-phase {
- expression: mnist_tensorflow + mnist_softmax_tensorflow + mnist_softmax_onnx + my_xgboost
+ expression: mnist_tensorflow + mnist_softmax_tensorflow + mnist_softmax_onnx + my_xgboost + my_lightgbm
}
}
diff --git a/config-model/src/test/cfg/application/ml_serving/models/lightgbm_regression.json b/config-model/src/test/cfg/application/ml_serving/models/lightgbm_regression.json
new file mode 100644
index 00000000000..cf0488ecd8b
--- /dev/null
+++ b/config-model/src/test/cfg/application/ml_serving/models/lightgbm_regression.json
@@ -0,0 +1,275 @@
+{
+ "name": "tree",
+ "version": "v3",
+ "num_class": 1,
+ "num_tree_per_iteration": 1,
+ "label_index": 0,
+ "max_feature_idx": 3,
+ "average_output": false,
+ "objective": "regression",
+ "feature_names": [
+ "numerical_1",
+ "numerical_2",
+ "categorical_1",
+ "categorical_2"
+ ],
+ "monotone_constraints": [],
+ "tree_info": [
+ {
+ "tree_index": 0,
+ "num_leaves": 3,
+ "num_cat": 1,
+ "shrinkage": 1,
+ "tree_structure": {
+ "split_index": 0,
+ "split_feature": 1,
+ "split_gain": 68.5353012084961,
+ "threshold": 0.46643291586559305,
+ "decision_type": "<=",
+ "default_left": true,
+ "missing_type": "NaN",
+ "internal_value": 0,
+ "internal_weight": 0,
+ "internal_count": 1000,
+ "left_child": {
+ "leaf_index": 0,
+ "leaf_value": 2.1594397038037663,
+ "leaf_weight": 469,
+ "leaf_count": 469
+ },
+ "right_child": {
+ "split_index": 1,
+ "split_feature": 3,
+ "split_gain": 41.27640151977539,
+ "threshold": "2||3||4",
+ "decision_type": "==",
+ "default_left": false,
+ "missing_type": "NaN",
+ "internal_value": 0.246035,
+ "internal_weight": 531,
+ "internal_count": 531,
+ "left_child": {
+ "leaf_index": 1,
+ "leaf_value": 2.235297305276056,
+ "leaf_weight": 302,
+ "leaf_count": 302
+ },
+ "right_child": {
+ "leaf_index": 2,
+ "leaf_value": 2.1792953471546546,
+ "leaf_weight": 229,
+ "leaf_count": 229
+ }
+ }
+ }
+ },
+ {
+ "tree_index": 1,
+ "num_leaves": 3,
+ "num_cat": 1,
+ "shrinkage": 0.1,
+ "tree_structure": {
+ "split_index": 0,
+ "split_feature": 2,
+ "split_gain": 64.22250366210938,
+ "threshold": "3||4",
+ "decision_type": "==",
+ "default_left": false,
+ "missing_type": "NaN",
+ "internal_value": 0,
+ "internal_weight": 0,
+ "internal_count": 1000,
+ "left_child": {
+ "leaf_index": 0,
+ "leaf_value": 0.03070842919354316,
+ "leaf_weight": 399,
+ "leaf_count": 399
+ },
+ "right_child": {
+ "split_index": 1,
+ "split_feature": 0,
+ "split_gain": 36.74250030517578,
+ "threshold": 0.5102250691730842,
+ "decision_type": "<=",
+ "default_left": true,
+ "missing_type": "NaN",
+ "internal_value": -0.204906,
+ "internal_weight": 601,
+ "internal_count": 601,
+ "left_child": {
+ "leaf_index": 1,
+ "leaf_value": -0.04439151147520909,
+ "leaf_weight": 315,
+ "leaf_count": 315
+ },
+ "right_child": {
+ "leaf_index": 2,
+ "leaf_value": 0.005117411709368601,
+ "leaf_weight": 286,
+ "leaf_count": 286
+ }
+ }
+ }
+ },
+ {
+ "tree_index": 2,
+ "num_leaves": 3,
+ "num_cat": 0,
+ "shrinkage": 0.1,
+ "tree_structure": {
+ "split_index": 0,
+ "split_feature": 1,
+ "split_gain": 57.1327018737793,
+ "threshold": 0.668665477622446,
+ "decision_type": "<=",
+ "default_left": true,
+ "missing_type": "NaN",
+ "internal_value": 0,
+ "internal_weight": 0,
+ "internal_count": 1000,
+ "left_child": {
+ "split_index": 1,
+ "split_feature": 1,
+ "split_gain": 40.859100341796875,
+ "threshold": 0.008118820676863816,
+ "decision_type": "<=",
+ "default_left": true,
+ "missing_type": "NaN",
+ "internal_value": -0.162926,
+ "internal_weight": 681,
+ "internal_count": 681,
+ "left_child": {
+ "leaf_index": 0,
+ "leaf_value": -0.15361238490967524,
+ "leaf_weight": 21,
+ "leaf_count": 21
+ },
+ "right_child": {
+ "leaf_index": 2,
+ "leaf_value": -0.01192330846157292,
+ "leaf_weight": 660,
+ "leaf_count": 660
+ }
+ },
+ "right_child": {
+ "leaf_index": 1,
+ "leaf_value": 0.03499044894987518,
+ "leaf_weight": 319,
+ "leaf_count": 319
+ }
+ }
+ },
+ {
+ "tree_index": 3,
+ "num_leaves": 3,
+ "num_cat": 1,
+ "shrinkage": 0.1,
+ "tree_structure": {
+ "split_index": 0,
+ "split_feature": 0,
+ "split_gain": 54.77090072631836,
+ "threshold": 0.5201391072644542,
+ "decision_type": "<=",
+ "default_left": true,
+ "missing_type": "NaN",
+ "internal_value": 0,
+ "internal_weight": 0,
+ "internal_count": 1000,
+ "left_child": {
+ "leaf_index": 0,
+ "leaf_value": -0.02141000620783247,
+ "leaf_weight": 543,
+ "leaf_count": 543
+ },
+ "right_child": {
+ "split_index": 1,
+ "split_feature": 2,
+ "split_gain": 27.200700759887695,
+ "threshold": "0||1",
+ "decision_type": "==",
+ "default_left": false,
+ "missing_type": "NaN",
+ "internal_value": 0.255704,
+ "internal_weight": 457,
+ "internal_count": 457,
+ "left_child": {
+ "leaf_index": 1,
+ "leaf_value": -0.004121485787596721,
+ "leaf_weight": 191,
+ "leaf_count": 191
+ },
+ "right_child": {
+ "leaf_index": 2,
+ "leaf_value": 0.04534090904886873,
+ "leaf_weight": 266,
+ "leaf_count": 266
+ }
+ }
+ }
+ },
+ {
+ "tree_index": 4,
+ "num_leaves": 3,
+ "num_cat": 1,
+ "shrinkage": 0.1,
+ "tree_structure": {
+ "split_index": 0,
+ "split_feature": 3,
+ "split_gain": 51.84349822998047,
+ "threshold": "2||3||4",
+ "decision_type": "==",
+ "default_left": false,
+ "missing_type": "NaN",
+ "internal_value": 0,
+ "internal_weight": 0,
+ "internal_count": 1000,
+ "left_child": {
+ "split_index": 1,
+ "split_feature": 1,
+ "split_gain": 39.352699279785156,
+ "threshold": 0.27283279016959255,
+ "decision_type": "<=",
+ "default_left": true,
+ "missing_type": "NaN",
+ "internal_value": 0.188414,
+ "internal_weight": 593,
+ "internal_count": 593,
+ "left_child": {
+ "leaf_index": 0,
+ "leaf_value": -0.01924803254356527,
+ "leaf_weight": 184,
+ "leaf_count": 184
+ },
+ "right_child": {
+ "leaf_index": 2,
+ "leaf_value": 0.03643772842347651,
+ "leaf_weight": 409,
+ "leaf_count": 409
+ }
+ },
+ "right_child": {
+ "leaf_index": 1,
+ "leaf_value": -0.02701711918923075,
+ "leaf_weight": 407,
+ "leaf_count": 407
+ }
+ }
+ }
+ ],
+ "pandas_categorical": [
+ [
+ "a",
+ "b",
+ "c",
+ "d",
+ "e"
+ ],
+ [
+ "i",
+ "j",
+ "k",
+ "l",
+ "m"
+ ]
+ ]
+} \ No newline at end of file
diff --git a/config-model/src/test/configmodel/types/references/documentmanager_multiple_imported_fields.cfg b/config-model/src/test/configmodel/types/references/documentmanager_multiple_imported_fields.cfg
new file mode 100644
index 00000000000..7b50176625d
--- /dev/null
+++ b/config-model/src/test/configmodel/types/references/documentmanager_multiple_imported_fields.cfg
@@ -0,0 +1,104 @@
+enablecompression false
+datatype[0].id 1381038251
+datatype[0].structtype[0].name "position"
+datatype[0].structtype[0].version 0
+datatype[0].structtype[0].compresstype NONE
+datatype[0].structtype[0].compresslevel 0
+datatype[0].structtype[0].compressthreshold 95
+datatype[0].structtype[0].compressminsize 800
+datatype[0].structtype[0].field[0].name "x"
+datatype[0].structtype[0].field[0].datatype 0
+datatype[0].structtype[0].field[0].detailedtype ""
+datatype[0].structtype[0].field[1].name "y"
+datatype[0].structtype[0].field[1].datatype 0
+datatype[0].structtype[0].field[1].detailedtype ""
+datatype[1].id 595216861
+datatype[1].referencetype[0].target_type_id -1318255918
+datatype[2].id 542332920
+datatype[2].referencetype[0].target_type_id 443162583
+datatype[3].id 959075962
+datatype[3].structtype[0].name "ad.header"
+datatype[3].structtype[0].version 0
+datatype[3].structtype[0].compresstype NONE
+datatype[3].structtype[0].compresslevel 0
+datatype[3].structtype[0].compressthreshold 95
+datatype[3].structtype[0].compressminsize 800
+datatype[3].structtype[0].field[0].name "campaign_ref"
+datatype[3].structtype[0].field[0].datatype 595216861
+datatype[3].structtype[0].field[0].detailedtype ""
+datatype[3].structtype[0].field[1].name "person_ref"
+datatype[3].structtype[0].field[1].datatype 542332920
+datatype[3].structtype[0].field[1].detailedtype ""
+datatype[4].id -255288561
+datatype[4].structtype[0].name "ad.body"
+datatype[4].structtype[0].version 0
+datatype[4].structtype[0].compresstype NONE
+datatype[4].structtype[0].compresslevel 0
+datatype[4].structtype[0].compressthreshold 95
+datatype[4].structtype[0].compressminsize 800
+datatype[5].id 2987301
+datatype[5].documenttype[0].name "ad"
+datatype[5].documenttype[0].version 0
+datatype[5].documenttype[0].inherits[0].name "document"
+datatype[5].documenttype[0].inherits[0].version 0
+datatype[5].documenttype[0].headerstruct 959075962
+datatype[5].documenttype[0].bodystruct -255288561
+datatype[5].documenttype[0].fieldsets{[document]}.fields[0] "campaign_ref"
+datatype[5].documenttype[0].fieldsets{[document]}.fields[1] "person_ref"
+datatype[5].documenttype[0].importedfield[0].name "my_cool_field"
+datatype[5].documenttype[0].importedfield[1].name "my_swag_field"
+datatype[5].documenttype[0].importedfield[2].name "my_name"
+datatype[6].id -2041471955
+datatype[6].structtype[0].name "campaign.header"
+datatype[6].structtype[0].version 0
+datatype[6].structtype[0].compresstype NONE
+datatype[6].structtype[0].compresslevel 0
+datatype[6].structtype[0].compressthreshold 95
+datatype[6].structtype[0].compressminsize 800
+datatype[6].structtype[0].field[0].name "cool_field"
+datatype[6].structtype[0].field[0].datatype 2
+datatype[6].structtype[0].field[0].detailedtype ""
+datatype[6].structtype[0].field[1].name "swag_field"
+datatype[6].structtype[0].field[1].datatype 4
+datatype[6].structtype[0].field[1].detailedtype ""
+datatype[7].id 1448849794
+datatype[7].structtype[0].name "campaign.body"
+datatype[7].structtype[0].version 0
+datatype[7].structtype[0].compresstype NONE
+datatype[7].structtype[0].compresslevel 0
+datatype[7].structtype[0].compressthreshold 95
+datatype[7].structtype[0].compressminsize 800
+datatype[8].id -1318255918
+datatype[8].documenttype[0].name "campaign"
+datatype[8].documenttype[0].version 0
+datatype[8].documenttype[0].inherits[0].name "document"
+datatype[8].documenttype[0].inherits[0].version 0
+datatype[8].documenttype[0].headerstruct -2041471955
+datatype[8].documenttype[0].bodystruct 1448849794
+datatype[8].documenttype[0].fieldsets{[document]}.fields[0] "cool_field"
+datatype[8].documenttype[0].fieldsets{[document]}.fields[1] "swag_field"
+datatype[9].id 3129224
+datatype[9].structtype[0].name "person.header"
+datatype[9].structtype[0].version 0
+datatype[9].structtype[0].compresstype NONE
+datatype[9].structtype[0].compresslevel 0
+datatype[9].structtype[0].compressthreshold 95
+datatype[9].structtype[0].compressminsize 800
+datatype[9].structtype[0].field[0].name "name"
+datatype[9].structtype[0].field[0].datatype 2
+datatype[9].structtype[0].field[0].detailedtype ""
+datatype[10].id -2003767395
+datatype[10].structtype[0].name "person.body"
+datatype[10].structtype[0].version 0
+datatype[10].structtype[0].compresstype NONE
+datatype[10].structtype[0].compresslevel 0
+datatype[10].structtype[0].compressthreshold 95
+datatype[10].structtype[0].compressminsize 800
+datatype[11].id 443162583
+datatype[11].documenttype[0].name "person"
+datatype[11].documenttype[0].version 0
+datatype[11].documenttype[0].inherits[0].name "document"
+datatype[11].documenttype[0].inherits[0].version 0
+datatype[11].documenttype[0].headerstruct 3129224
+datatype[11].documenttype[0].bodystruct -2003767395
+datatype[11].documenttype[0].fieldsets{[document]}.fields[0] "name" \ No newline at end of file
diff --git a/config-model/src/test/configmodel/types/references/documenttypes_multiple_imported_fields.cfg b/config-model/src/test/configmodel/types/references/documenttypes_multiple_imported_fields.cfg
new file mode 100644
index 00000000000..7859703ffe0
--- /dev/null
+++ b/config-model/src/test/configmodel/types/references/documenttypes_multiple_imported_fields.cfg
@@ -0,0 +1,141 @@
+enablecompression false
+documenttype[0].id 2987301
+documenttype[0].name "ad"
+documenttype[0].version 0
+documenttype[0].headerstruct 959075962
+documenttype[0].bodystruct -255288561
+documenttype[0].inherits[0].id 8
+documenttype[0].datatype[0].id 959075962
+documenttype[0].datatype[0].type STRUCT
+documenttype[0].datatype[0].array.element.id 0
+documenttype[0].datatype[0].map.key.id 0
+documenttype[0].datatype[0].map.value.id 0
+documenttype[0].datatype[0].wset.key.id 0
+documenttype[0].datatype[0].wset.createifnonexistent false
+documenttype[0].datatype[0].wset.removeifzero false
+documenttype[0].datatype[0].annotationref.annotation.id 0
+documenttype[0].datatype[0].sstruct.name "ad.header"
+documenttype[0].datatype[0].sstruct.version 0
+documenttype[0].datatype[0].sstruct.compression.type NONE
+documenttype[0].datatype[0].sstruct.compression.level 0
+documenttype[0].datatype[0].sstruct.compression.threshold 95
+documenttype[0].datatype[0].sstruct.compression.minsize 200
+documenttype[0].datatype[0].sstruct.field[0].name "campaign_ref"
+documenttype[0].datatype[0].sstruct.field[0].id 23963250
+documenttype[0].datatype[0].sstruct.field[0].datatype 595216861
+documenttype[0].datatype[0].sstruct.field[0].detailedtype ""
+documenttype[0].datatype[0].sstruct.field[1].name "person_ref"
+documenttype[0].datatype[0].sstruct.field[1].id 100779805
+documenttype[0].datatype[0].sstruct.field[1].datatype 542332920
+documenttype[0].datatype[0].sstruct.field[1].detailedtype ""
+documenttype[0].datatype[1].id -255288561
+documenttype[0].datatype[1].type STRUCT
+documenttype[0].datatype[1].array.element.id 0
+documenttype[0].datatype[1].map.key.id 0
+documenttype[0].datatype[1].map.value.id 0
+documenttype[0].datatype[1].wset.key.id 0
+documenttype[0].datatype[1].wset.createifnonexistent false
+documenttype[0].datatype[1].wset.removeifzero false
+documenttype[0].datatype[1].annotationref.annotation.id 0
+documenttype[0].datatype[1].sstruct.name "ad.body"
+documenttype[0].datatype[1].sstruct.version 0
+documenttype[0].datatype[1].sstruct.compression.type NONE
+documenttype[0].datatype[1].sstruct.compression.level 0
+documenttype[0].datatype[1].sstruct.compression.threshold 95
+documenttype[0].datatype[1].sstruct.compression.minsize 200
+documenttype[0].fieldsets{[document]}.fields[0] "campaign_ref"
+documenttype[0].fieldsets{[document]}.fields[1] "person_ref"
+documenttype[0].referencetype[0].id 595216861
+documenttype[0].referencetype[0].target_type_id -1318255918
+documenttype[0].referencetype[1].id 542332920
+documenttype[0].referencetype[1].target_type_id 443162583
+documenttype[0].importedfield[0].name "my_cool_field"
+documenttype[0].importedfield[1].name "my_swag_field"
+documenttype[0].importedfield[2].name "my_name"
+documenttype[1].id -1318255918
+documenttype[1].name "campaign"
+documenttype[1].version 0
+documenttype[1].headerstruct -2041471955
+documenttype[1].bodystruct 1448849794
+documenttype[1].inherits[0].id 8
+documenttype[1].datatype[0].id -2041471955
+documenttype[1].datatype[0].type STRUCT
+documenttype[1].datatype[0].array.element.id 0
+documenttype[1].datatype[0].map.key.id 0
+documenttype[1].datatype[0].map.value.id 0
+documenttype[1].datatype[0].wset.key.id 0
+documenttype[1].datatype[0].wset.createifnonexistent false
+documenttype[1].datatype[0].wset.removeifzero false
+documenttype[1].datatype[0].annotationref.annotation.id 0
+documenttype[1].datatype[0].sstruct.name "campaign.header"
+documenttype[1].datatype[0].sstruct.version 0
+documenttype[1].datatype[0].sstruct.compression.type NONE
+documenttype[1].datatype[0].sstruct.compression.level 0
+documenttype[1].datatype[0].sstruct.compression.threshold 95
+documenttype[1].datatype[0].sstruct.compression.minsize 200
+documenttype[1].datatype[0].sstruct.field[0].name "cool_field"
+documenttype[1].datatype[0].sstruct.field[0].id 1588702436
+documenttype[1].datatype[0].sstruct.field[0].datatype 2
+documenttype[1].datatype[0].sstruct.field[0].detailedtype ""
+documenttype[1].datatype[0].sstruct.field[1].name "swag_field"
+documenttype[1].datatype[0].sstruct.field[1].id 1691224741
+documenttype[1].datatype[0].sstruct.field[1].datatype 4
+documenttype[1].datatype[0].sstruct.field[1].detailedtype ""
+documenttype[1].datatype[1].id 1448849794
+documenttype[1].datatype[1].type STRUCT
+documenttype[1].datatype[1].array.element.id 0
+documenttype[1].datatype[1].map.key.id 0
+documenttype[1].datatype[1].map.value.id 0
+documenttype[1].datatype[1].wset.key.id 0
+documenttype[1].datatype[1].wset.createifnonexistent false
+documenttype[1].datatype[1].wset.removeifzero false
+documenttype[1].datatype[1].annotationref.annotation.id 0
+documenttype[1].datatype[1].sstruct.name "campaign.body"
+documenttype[1].datatype[1].sstruct.version 0
+documenttype[1].datatype[1].sstruct.compression.type NONE
+documenttype[1].datatype[1].sstruct.compression.level 0
+documenttype[1].datatype[1].sstruct.compression.threshold 95
+documenttype[1].datatype[1].sstruct.compression.minsize 200
+documenttype[1].fieldsets{[document]}.fields[0] "cool_field"
+documenttype[1].fieldsets{[document]}.fields[1] "swag_field"
+documenttype[2].id 443162583
+documenttype[2].name "person"
+documenttype[2].version 0
+documenttype[2].headerstruct 3129224
+documenttype[2].bodystruct -2003767395
+documenttype[2].inherits[0].id 8
+documenttype[2].datatype[0].id 3129224
+documenttype[2].datatype[0].type STRUCT
+documenttype[2].datatype[0].array.element.id 0
+documenttype[2].datatype[0].map.key.id 0
+documenttype[2].datatype[0].map.value.id 0
+documenttype[2].datatype[0].wset.key.id 0
+documenttype[2].datatype[0].wset.createifnonexistent false
+documenttype[2].datatype[0].wset.removeifzero false
+documenttype[2].datatype[0].annotationref.annotation.id 0
+documenttype[2].datatype[0].sstruct.name "person.header"
+documenttype[2].datatype[0].sstruct.version 0
+documenttype[2].datatype[0].sstruct.compression.type NONE
+documenttype[2].datatype[0].sstruct.compression.level 0
+documenttype[2].datatype[0].sstruct.compression.threshold 95
+documenttype[2].datatype[0].sstruct.compression.minsize 200
+documenttype[2].datatype[0].sstruct.field[0].name "name"
+documenttype[2].datatype[0].sstruct.field[0].id 1160796772
+documenttype[2].datatype[0].sstruct.field[0].datatype 2
+documenttype[2].datatype[0].sstruct.field[0].detailedtype ""
+documenttype[2].datatype[1].id -2003767395
+documenttype[2].datatype[1].type STRUCT
+documenttype[2].datatype[1].array.element.id 0
+documenttype[2].datatype[1].map.key.id 0
+documenttype[2].datatype[1].map.value.id 0
+documenttype[2].datatype[1].wset.key.id 0
+documenttype[2].datatype[1].wset.createifnonexistent false
+documenttype[2].datatype[1].wset.removeifzero false
+documenttype[2].datatype[1].annotationref.annotation.id 0
+documenttype[2].datatype[1].sstruct.name "person.body"
+documenttype[2].datatype[1].sstruct.version 0
+documenttype[2].datatype[1].sstruct.compression.type NONE
+documenttype[2].datatype[1].sstruct.compression.level 0
+documenttype[2].datatype[1].sstruct.compression.threshold 95
+documenttype[2].datatype[1].sstruct.compression.minsize 200
+documenttype[2].fieldsets{[document]}.fields[0] "name" \ No newline at end of file
diff --git a/config-model/src/test/integration/lightgbm/models/regression.json b/config-model/src/test/integration/lightgbm/models/regression.json
new file mode 100644
index 00000000000..cf0488ecd8b
--- /dev/null
+++ b/config-model/src/test/integration/lightgbm/models/regression.json
@@ -0,0 +1,275 @@
+{
+ "name": "tree",
+ "version": "v3",
+ "num_class": 1,
+ "num_tree_per_iteration": 1,
+ "label_index": 0,
+ "max_feature_idx": 3,
+ "average_output": false,
+ "objective": "regression",
+ "feature_names": [
+ "numerical_1",
+ "numerical_2",
+ "categorical_1",
+ "categorical_2"
+ ],
+ "monotone_constraints": [],
+ "tree_info": [
+ {
+ "tree_index": 0,
+ "num_leaves": 3,
+ "num_cat": 1,
+ "shrinkage": 1,
+ "tree_structure": {
+ "split_index": 0,
+ "split_feature": 1,
+ "split_gain": 68.5353012084961,
+ "threshold": 0.46643291586559305,
+ "decision_type": "<=",
+ "default_left": true,
+ "missing_type": "NaN",
+ "internal_value": 0,
+ "internal_weight": 0,
+ "internal_count": 1000,
+ "left_child": {
+ "leaf_index": 0,
+ "leaf_value": 2.1594397038037663,
+ "leaf_weight": 469,
+ "leaf_count": 469
+ },
+ "right_child": {
+ "split_index": 1,
+ "split_feature": 3,
+ "split_gain": 41.27640151977539,
+ "threshold": "2||3||4",
+ "decision_type": "==",
+ "default_left": false,
+ "missing_type": "NaN",
+ "internal_value": 0.246035,
+ "internal_weight": 531,
+ "internal_count": 531,
+ "left_child": {
+ "leaf_index": 1,
+ "leaf_value": 2.235297305276056,
+ "leaf_weight": 302,
+ "leaf_count": 302
+ },
+ "right_child": {
+ "leaf_index": 2,
+ "leaf_value": 2.1792953471546546,
+ "leaf_weight": 229,
+ "leaf_count": 229
+ }
+ }
+ }
+ },
+ {
+ "tree_index": 1,
+ "num_leaves": 3,
+ "num_cat": 1,
+ "shrinkage": 0.1,
+ "tree_structure": {
+ "split_index": 0,
+ "split_feature": 2,
+ "split_gain": 64.22250366210938,
+ "threshold": "3||4",
+ "decision_type": "==",
+ "default_left": false,
+ "missing_type": "NaN",
+ "internal_value": 0,
+ "internal_weight": 0,
+ "internal_count": 1000,
+ "left_child": {
+ "leaf_index": 0,
+ "leaf_value": 0.03070842919354316,
+ "leaf_weight": 399,
+ "leaf_count": 399
+ },
+ "right_child": {
+ "split_index": 1,
+ "split_feature": 0,
+ "split_gain": 36.74250030517578,
+ "threshold": 0.5102250691730842,
+ "decision_type": "<=",
+ "default_left": true,
+ "missing_type": "NaN",
+ "internal_value": -0.204906,
+ "internal_weight": 601,
+ "internal_count": 601,
+ "left_child": {
+ "leaf_index": 1,
+ "leaf_value": -0.04439151147520909,
+ "leaf_weight": 315,
+ "leaf_count": 315
+ },
+ "right_child": {
+ "leaf_index": 2,
+ "leaf_value": 0.005117411709368601,
+ "leaf_weight": 286,
+ "leaf_count": 286
+ }
+ }
+ }
+ },
+ {
+ "tree_index": 2,
+ "num_leaves": 3,
+ "num_cat": 0,
+ "shrinkage": 0.1,
+ "tree_structure": {
+ "split_index": 0,
+ "split_feature": 1,
+ "split_gain": 57.1327018737793,
+ "threshold": 0.668665477622446,
+ "decision_type": "<=",
+ "default_left": true,
+ "missing_type": "NaN",
+ "internal_value": 0,
+ "internal_weight": 0,
+ "internal_count": 1000,
+ "left_child": {
+ "split_index": 1,
+ "split_feature": 1,
+ "split_gain": 40.859100341796875,
+ "threshold": 0.008118820676863816,
+ "decision_type": "<=",
+ "default_left": true,
+ "missing_type": "NaN",
+ "internal_value": -0.162926,
+ "internal_weight": 681,
+ "internal_count": 681,
+ "left_child": {
+ "leaf_index": 0,
+ "leaf_value": -0.15361238490967524,
+ "leaf_weight": 21,
+ "leaf_count": 21
+ },
+ "right_child": {
+ "leaf_index": 2,
+ "leaf_value": -0.01192330846157292,
+ "leaf_weight": 660,
+ "leaf_count": 660
+ }
+ },
+ "right_child": {
+ "leaf_index": 1,
+ "leaf_value": 0.03499044894987518,
+ "leaf_weight": 319,
+ "leaf_count": 319
+ }
+ }
+ },
+ {
+ "tree_index": 3,
+ "num_leaves": 3,
+ "num_cat": 1,
+ "shrinkage": 0.1,
+ "tree_structure": {
+ "split_index": 0,
+ "split_feature": 0,
+ "split_gain": 54.77090072631836,
+ "threshold": 0.5201391072644542,
+ "decision_type": "<=",
+ "default_left": true,
+ "missing_type": "NaN",
+ "internal_value": 0,
+ "internal_weight": 0,
+ "internal_count": 1000,
+ "left_child": {
+ "leaf_index": 0,
+ "leaf_value": -0.02141000620783247,
+ "leaf_weight": 543,
+ "leaf_count": 543
+ },
+ "right_child": {
+ "split_index": 1,
+ "split_feature": 2,
+ "split_gain": 27.200700759887695,
+ "threshold": "0||1",
+ "decision_type": "==",
+ "default_left": false,
+ "missing_type": "NaN",
+ "internal_value": 0.255704,
+ "internal_weight": 457,
+ "internal_count": 457,
+ "left_child": {
+ "leaf_index": 1,
+ "leaf_value": -0.004121485787596721,
+ "leaf_weight": 191,
+ "leaf_count": 191
+ },
+ "right_child": {
+ "leaf_index": 2,
+ "leaf_value": 0.04534090904886873,
+ "leaf_weight": 266,
+ "leaf_count": 266
+ }
+ }
+ }
+ },
+ {
+ "tree_index": 4,
+ "num_leaves": 3,
+ "num_cat": 1,
+ "shrinkage": 0.1,
+ "tree_structure": {
+ "split_index": 0,
+ "split_feature": 3,
+ "split_gain": 51.84349822998047,
+ "threshold": "2||3||4",
+ "decision_type": "==",
+ "default_left": false,
+ "missing_type": "NaN",
+ "internal_value": 0,
+ "internal_weight": 0,
+ "internal_count": 1000,
+ "left_child": {
+ "split_index": 1,
+ "split_feature": 1,
+ "split_gain": 39.352699279785156,
+ "threshold": 0.27283279016959255,
+ "decision_type": "<=",
+ "default_left": true,
+ "missing_type": "NaN",
+ "internal_value": 0.188414,
+ "internal_weight": 593,
+ "internal_count": 593,
+ "left_child": {
+ "leaf_index": 0,
+ "leaf_value": -0.01924803254356527,
+ "leaf_weight": 184,
+ "leaf_count": 184
+ },
+ "right_child": {
+ "leaf_index": 2,
+ "leaf_value": 0.03643772842347651,
+ "leaf_weight": 409,
+ "leaf_count": 409
+ }
+ },
+ "right_child": {
+ "leaf_index": 1,
+ "leaf_value": -0.02701711918923075,
+ "leaf_weight": 407,
+ "leaf_count": 407
+ }
+ }
+ }
+ ],
+ "pandas_categorical": [
+ [
+ "a",
+ "b",
+ "c",
+ "d",
+ "e"
+ ],
+ [
+ "i",
+ "j",
+ "k",
+ "l",
+ "m"
+ ]
+ ]
+} \ No newline at end of file
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/AttributeUtils.java b/config-model/src/test/java/com/yahoo/searchdefinition/AttributeUtils.java
new file mode 100644
index 00000000000..2c13427760f
--- /dev/null
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/AttributeUtils.java
@@ -0,0 +1,15 @@
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition;
+
+import com.yahoo.searchdefinition.document.SDField;
+
+/**
+ * Convenience class for tests that need to set attribute properties on fields.
+ */
+public class AttributeUtils {
+
+ public static void addAttributeAspect(SDField field) {
+ field.parseIndexingScript("{ attribute }");
+ }
+
+}
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/DocumentReferenceResolverTest.java b/config-model/src/test/java/com/yahoo/searchdefinition/DocumentReferenceResolverTest.java
index 8378ec811a5..e46208c770d 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/DocumentReferenceResolverTest.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/DocumentReferenceResolverTest.java
@@ -36,7 +36,7 @@ public class DocumentReferenceResolverTest {
// Create foo document with document reference to bar and add another field
SDField fooRefToBarField = new SDField
("bar_ref", ReferenceDataType.createWithInferredId(barDocument.getDocumentType()));
- addAttributeAspect(fooRefToBarField);
+ AttributeUtils.addAttributeAspect(fooRefToBarField);
SDField irrelevantField = new SDField("irrelevant_stuff", DataType.INT);
Search fooSearch = new Search();
SDDocumentType fooDocument = new SDDocumentType("foo", fooSearch);
@@ -59,7 +59,7 @@ public class DocumentReferenceResolverTest {
// Create foo document with document reference to non-existing document bar
SDField fooRefToBarField = new SDField(
"bar_ref", ReferenceDataType.createWithInferredId(TemporaryStructuredDataType.create("bar")));
- addAttributeAspect(fooRefToBarField);
+ AttributeUtils.addAttributeAspect(fooRefToBarField);
Search fooSearch = new Search();
SDDocumentType fooDocument = new SDDocumentType("foo", fooSearch);
fooDocument.addField(fooRefToBarField);
@@ -95,8 +95,4 @@ public class DocumentReferenceResolverTest {
resolver.resolveReferences(fooDocument);
}
- private static void addAttributeAspect(SDField fooRefToBarField) {
- fooRefToBarField.parseIndexingScript("{ attribute }");
- }
-
}
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/ImportedFieldsEnumeratorTest.java b/config-model/src/test/java/com/yahoo/searchdefinition/ImportedFieldsEnumeratorTest.java
new file mode 100644
index 00000000000..fcbb89b5c42
--- /dev/null
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/ImportedFieldsEnumeratorTest.java
@@ -0,0 +1,66 @@
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition;
+
+import com.yahoo.document.DataType;
+import com.yahoo.document.ReferenceDataType;
+import com.yahoo.searchdefinition.document.SDDocumentType;
+import com.yahoo.searchdefinition.document.SDField;
+import com.yahoo.searchdefinition.document.TemporaryImportedField;
+import org.junit.Test;
+
+import java.util.HashSet;
+import java.util.List;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+
+public class ImportedFieldsEnumeratorTest {
+
+ @Test
+ public void imported_fields_are_enumerated_and_copied_from_correct_search_instance() {
+ Search parentSearch = new Search();
+ SDDocumentType parentDocument = new SDDocumentType("parent", parentSearch);
+ var parentField = new SDField("their_field", DataType.INT);
+ AttributeUtils.addAttributeAspect(parentField);
+ parentDocument.addField(parentField);
+ parentSearch.addDocument(parentDocument);
+
+ Search fooSearch = new Search();
+ SDField fooRefToParent = new SDField(
+ "foo_ref", ReferenceDataType.createWithInferredId(parentDocument.getDocumentType()));
+ AttributeUtils.addAttributeAspect(fooRefToParent);
+ var fooImports = fooSearch.temporaryImportedFields().get();
+ fooImports.add(new TemporaryImportedField("my_first_import", "foo_ref", "their_field"));
+ fooImports.add(new TemporaryImportedField("my_second_import", "foo_ref", "their_field"));
+ SDDocumentType fooDocument = new SDDocumentType("foo", fooSearch);
+ fooSearch.addDocument(fooDocument);
+
+ Search barSearch = new Search();
+ SDField barRefToParent = new SDField(
+ "bar_ref", ReferenceDataType.createWithInferredId(parentDocument.getDocumentType()));
+ AttributeUtils.addAttributeAspect(barRefToParent);
+ var barImports = barSearch.temporaryImportedFields().get();
+ barImports.add(new TemporaryImportedField("my_cool_import", "my_ref", "their_field"));
+ SDDocumentType barDocument = new SDDocumentType("bar", barSearch);
+ barSearch.addDocument(barDocument);
+
+ var enumerator = new ImportedFieldsEnumerator(List.of(parentSearch, fooSearch, barSearch));
+
+ enumerator.enumerateImportedFields(parentDocument);
+ assertImportedFieldsAre(parentDocument, List.of()); // No imported fields in parent
+
+ enumerator.enumerateImportedFields(fooDocument);
+ assertImportedFieldsAre(fooDocument, List.of("my_first_import", "my_second_import"));
+
+ enumerator.enumerateImportedFields(barDocument);
+ assertImportedFieldsAre(barDocument, List.of("my_cool_import"));
+ }
+
+ private void assertImportedFieldsAre(SDDocumentType documentType, List<String> expectedNames) {
+ assertNotNull(documentType.getTemporaryImportedFields());
+ var actualNames = documentType.getTemporaryImportedFields().fields().keySet();
+ var expectedNameSet = new HashSet<>(expectedNames);
+ assertEquals(expectedNameSet, actualNames);
+ }
+
+}
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankProfileSearchFixture.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankProfileSearchFixture.java
index 08dd5148b29..0cd6674751e 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankProfileSearchFixture.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankProfileSearchFixture.java
@@ -15,6 +15,7 @@ import com.yahoo.searchdefinition.parser.ParseException;
import ai.vespa.rankingexpression.importer.configmodelview.ImportedMlModels;
import ai.vespa.rankingexpression.importer.onnx.OnnxImporter;
import ai.vespa.rankingexpression.importer.tensorflow.TensorFlowImporter;
+import ai.vespa.rankingexpression.importer.lightgbm.LightGBMImporter;
import ai.vespa.rankingexpression.importer.xgboost.XGBoostImporter;
import java.util.HashMap;
@@ -33,6 +34,7 @@ class RankProfileSearchFixture {
private final ImmutableList<MlModelImporter> importers = ImmutableList.of(new TensorFlowImporter(),
new OnnxImporter(),
+ new LightGBMImporter(),
new XGBoostImporter());
private RankProfileRegistry rankProfileRegistry = new RankProfileRegistry();
private final QueryProfileRegistry queryProfileRegistry;
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeResolverTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeResolverTestCase.java
index 9e9ef2589be..a306e0f2c90 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeResolverTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeResolverTestCase.java
@@ -275,7 +275,7 @@ public class RankingExpressionTypeResolverTestCase {
builder.build(true, logger);
String message = logger.findMessage("The following query features");
assertNotNull(message);
- assertEquals("WARNING: The following query features are not declared in query profile types and " +
+ assertEquals("WARNING: The following query features used in 'my_rank_profile' are not declared in query profile types and " +
"will be interpreted as scalars, not tensors: [query(bar), query(baz), query(foo)]",
message);
}
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithLightGBMTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithLightGBMTestCase.java
new file mode 100644
index 00000000000..79d19371f1c
--- /dev/null
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithLightGBMTestCase.java
@@ -0,0 +1,88 @@
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.processing;
+
+import com.yahoo.config.application.api.ApplicationPackage;
+import com.yahoo.io.IOUtils;
+import com.yahoo.path.Path;
+import com.yahoo.searchdefinition.parser.ParseException;
+import org.junit.After;
+import org.junit.Test;
+
+import java.io.IOException;
+
+/**
+ * @author lesters
+ */
+public class RankingExpressionWithLightGBMTestCase {
+
+ private final Path applicationDir = Path.fromString("src/test/integration/lightgbm/");
+
+ private final static String lightGBMExpression =
+ "if (!(numerical_2 >= 0.46643291586559305), 2.1594397038037663, if (categorical_2 in [\"k\", \"l\", \"m\"], 2.235297305276056, 2.1792953471546546)) + if (categorical_1 in [\"d\", \"e\"], 0.03070842919354316, if (!(numerical_1 >= 0.5102250691730842), -0.04439151147520909, 0.005117411709368601)) + if (!(numerical_2 >= 0.668665477622446), if (!(numerical_2 >= 0.008118820676863816), -0.15361238490967524, -0.01192330846157292), 0.03499044894987518) + if (!(numerical_1 >= 0.5201391072644542), -0.02141000620783247, if (categorical_1 in [\"a\", \"b\"], -0.004121485787596721, 0.04534090904886873)) + if (categorical_2 in [\"k\", \"l\", \"m\"], if (!(numerical_2 >= 0.27283279016959255), -0.01924803254356527, 0.03643772842347651), -0.02701711918923075)";
+
+ @After
+ public void removeGeneratedModelFiles() {
+ IOUtils.recursiveDeleteDir(applicationDir.append(ApplicationPackage.MODELS_GENERATED_DIR).toFile());
+ }
+
+ @Test
+ public void testLightGBMReference() {
+ RankProfileSearchFixture search = fixtureWith("lightgbm('regression.json')");
+ search.assertFirstPhaseExpression(lightGBMExpression, "my_profile");
+ }
+
+ @Test
+ public void testNestedLightGBMReference() {
+ RankProfileSearchFixture search = fixtureWith("5 + sum(lightgbm('regression.json'))");
+ search.assertFirstPhaseExpression("5 + reduce(" + lightGBMExpression + ", sum)", "my_profile");
+ }
+
+ @Test
+ public void testImportingFromStoredExpressions() throws IOException {
+ RankProfileSearchFixture search = fixtureWith("lightgbm('regression.json')");
+ search.assertFirstPhaseExpression(lightGBMExpression, "my_profile");
+
+ // At this point the expression is stored - copy application to another location which do not have a models dir
+ Path storedApplicationDirectory = applicationDir.getParentPath().append("copy");
+ try {
+ storedApplicationDirectory.toFile().mkdirs();
+ IOUtils.copyDirectory(applicationDir.append(ApplicationPackage.MODELS_GENERATED_DIR).toFile(),
+ storedApplicationDirectory.append(ApplicationPackage.MODELS_GENERATED_DIR).toFile());
+ RankingExpressionWithTensorFlowTestCase.StoringApplicationPackage storedApplication = new RankingExpressionWithTensorFlowTestCase.StoringApplicationPackage(storedApplicationDirectory);
+ RankProfileSearchFixture searchFromStored = fixtureWith("lightgbm('regression.json')");
+ searchFromStored.assertFirstPhaseExpression(lightGBMExpression, "my_profile");
+ }
+ finally {
+ IOUtils.recursiveDeleteDir(storedApplicationDirectory.toFile());
+ }
+ }
+
+ private RankProfileSearchFixture fixtureWith(String firstPhaseExpression) {
+ return fixtureWith(firstPhaseExpression, null, null,
+ new RankingExpressionWithTensorFlowTestCase.StoringApplicationPackage(applicationDir));
+ }
+
+ private RankProfileSearchFixture fixtureWith(String firstPhaseExpression,
+ String constant,
+ String field,
+ RankingExpressionWithTensorFlowTestCase.StoringApplicationPackage application) {
+ try {
+ RankProfileSearchFixture fixture = new RankProfileSearchFixture(
+ application,
+ application.getQueryProfiles(),
+ " rank-profile my_profile {\n" +
+ " first-phase {\n" +
+ " expression: " + firstPhaseExpression +
+ " }\n" +
+ " }",
+ constant,
+ field);
+ fixture.compileRankProfile("my_profile", applicationDir.append("models"));
+ return fixture;
+ } catch (ParseException e) {
+ throw new IllegalArgumentException(e);
+ }
+ }
+
+}
+
diff --git a/config-model/src/test/java/com/yahoo/vespa/documentmodel/AbstractReferenceFieldTestCase.java b/config-model/src/test/java/com/yahoo/vespa/documentmodel/AbstractReferenceFieldTestCase.java
new file mode 100644
index 00000000000..d0ee0523489
--- /dev/null
+++ b/config-model/src/test/java/com/yahoo/vespa/documentmodel/AbstractReferenceFieldTestCase.java
@@ -0,0 +1,35 @@
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.documentmodel;
+
+import com.yahoo.document.DocumenttypesConfig;
+import com.yahoo.document.config.DocumentmanagerConfig;
+import com.yahoo.searchdefinition.SearchDefinitionTestCase;
+import com.yahoo.vespa.configmodel.producers.DocumentManager;
+import com.yahoo.vespa.configmodel.producers.DocumentTypes;
+
+import java.io.IOException;
+
+/**
+ * Utility functions for testing generated configs for reference/imported fields.
+ */
+public abstract class AbstractReferenceFieldTestCase extends SearchDefinitionTestCase {
+
+ private static String TEST_FOLDER = "src/test/configmodel/types/references/";
+
+ protected void assertDocumentConfigs(DocumentModel model,
+ String cfgFileSpec) throws IOException {
+ assertDocumentmanagerCfg(model, "documentmanager_" + cfgFileSpec + ".cfg");
+ assertDocumenttypesCfg(model , "documenttypes_" + cfgFileSpec + ".cfg");
+ }
+
+ protected void assertDocumentmanagerCfg(DocumentModel model, String documentmanagerCfgFile) throws IOException {
+ DocumentmanagerConfig.Builder documentmanagerCfg = new DocumentManager().produce(model, new DocumentmanagerConfig.Builder());
+ assertConfigFile(TEST_FOLDER + documentmanagerCfgFile, new DocumentmanagerConfig(documentmanagerCfg).toString());
+ }
+
+ protected void assertDocumenttypesCfg(DocumentModel model, String documenttypesCfgFile) throws IOException {
+ DocumenttypesConfig.Builder documenttypesCfg = new DocumentTypes().produce(model, new DocumenttypesConfig.Builder());
+ assertConfigFile(TEST_FOLDER + documenttypesCfgFile, new DocumenttypesConfig(documenttypesCfg).toString());
+ }
+
+}
diff --git a/config-model/src/test/java/com/yahoo/vespa/documentmodel/DocumentModelBuilderImportedFieldsTestCase.java b/config-model/src/test/java/com/yahoo/vespa/documentmodel/DocumentModelBuilderImportedFieldsTestCase.java
new file mode 100644
index 00000000000..599ae77a456
--- /dev/null
+++ b/config-model/src/test/java/com/yahoo/vespa/documentmodel/DocumentModelBuilderImportedFieldsTestCase.java
@@ -0,0 +1,55 @@
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.documentmodel;
+
+import com.yahoo.searchdefinition.SearchBuilder;
+import com.yahoo.searchdefinition.parser.ParseException;
+import org.junit.Test;
+
+import java.io.IOException;
+
+import static com.yahoo.config.model.test.TestUtil.joinLines;
+
+public class DocumentModelBuilderImportedFieldsTestCase extends AbstractReferenceFieldTestCase {
+
+ @Test
+ public void imported_fields_are_included_in_generated_document_configs() throws ParseException, IOException {
+ assertDocumentConfigs(new TestDocumentModelBuilder().addCampaign().addPerson().build(joinLines(
+ "search ad {",
+ " document ad {",
+ " field campaign_ref type reference<campaign> { indexing: attribute }",
+ " field person_ref type reference<person> { indexing: attribute }",
+ " }",
+ " import field campaign_ref.cool_field as my_cool_field {}",
+ " import field campaign_ref.swag_field as my_swag_field {}",
+ " import field person_ref.name as my_name {}",
+ "}")),
+ "multiple_imported_fields");
+ }
+
+ private static class TestDocumentModelBuilder {
+ private final SearchBuilder builder = new SearchBuilder();
+ public TestDocumentModelBuilder addCampaign() throws ParseException {
+ builder.importString(joinLines("search campaign {",
+ " document campaign {",
+ " field cool_field type string { indexing: attribute }",
+ " field swag_field type long { indexing: attribute }",
+ " }",
+ "}"));
+ return this;
+ }
+ public TestDocumentModelBuilder addPerson() throws ParseException {
+ builder.importString(joinLines("search person {",
+ " document person {",
+ " field name type string { indexing: attribute }",
+ " }",
+ "}"));
+ return this;
+ }
+ public DocumentModel build(String adSdContent) throws ParseException {
+ builder.importString(adSdContent);
+ builder.build();
+ return builder.getModel();
+ }
+ }
+
+}
diff --git a/config-model/src/test/java/com/yahoo/vespa/documentmodel/DocumentModelBuilderReferenceTypeTestCase.java b/config-model/src/test/java/com/yahoo/vespa/documentmodel/DocumentModelBuilderReferenceTypeTestCase.java
index e9a7a6ed33e..55980ee5fea 100644
--- a/config-model/src/test/java/com/yahoo/vespa/documentmodel/DocumentModelBuilderReferenceTypeTestCase.java
+++ b/config-model/src/test/java/com/yahoo/vespa/documentmodel/DocumentModelBuilderReferenceTypeTestCase.java
@@ -1,15 +1,10 @@
// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.documentmodel;
-import com.yahoo.document.DocumenttypesConfig;
import com.yahoo.document.ReferenceDataType;
-import com.yahoo.document.config.DocumentmanagerConfig;
import com.yahoo.documentmodel.NewDocumentType;
import com.yahoo.searchdefinition.SearchBuilder;
-import com.yahoo.searchdefinition.SearchDefinitionTestCase;
import com.yahoo.searchdefinition.parser.ParseException;
-import com.yahoo.vespa.configmodel.producers.DocumentManager;
-import com.yahoo.vespa.configmodel.producers.DocumentTypes;
import org.junit.Test;
import java.io.IOException;
@@ -20,7 +15,7 @@ import static org.junit.Assert.assertEquals;
/**
* @author geirst
*/
-public class DocumentModelBuilderReferenceTypeTestCase extends SearchDefinitionTestCase {
+public class DocumentModelBuilderReferenceTypeTestCase extends AbstractReferenceFieldTestCase {
@Test
public void reference_fields_can_reference_other_document_types() throws ParseException, IOException {
@@ -60,24 +55,6 @@ public class DocumentModelBuilderReferenceTypeTestCase extends SearchDefinitionT
assertEquals(campaignRefType.getTargetType(), campaignType);
}
- private static String TEST_FOLDER = "src/test/configmodel/types/references/";
-
- private void assertDocumentConfigs(DocumentModel model,
- String cfgFileSpec) throws IOException {
- assertDocumentmanagerCfg(model, "documentmanager_" + cfgFileSpec + ".cfg");
- assertDocumenttypesCfg(model , "documenttypes_" + cfgFileSpec + ".cfg");
- }
-
- private void assertDocumentmanagerCfg(DocumentModel model, String documentmanagerCfgFile) throws IOException {
- DocumentmanagerConfig.Builder documentmanagerCfg = new DocumentManager().produce(model, new DocumentmanagerConfig.Builder());
- assertConfigFile(TEST_FOLDER + documentmanagerCfgFile, new DocumentmanagerConfig(documentmanagerCfg).toString());
- }
-
- private void assertDocumenttypesCfg(DocumentModel model, String documenttypesCfgFile) throws IOException {
- DocumenttypesConfig.Builder documenttypesCfg = new DocumentTypes().produce(model, new DocumenttypesConfig.Builder());
- assertConfigFile(TEST_FOLDER + documenttypesCfgFile, new DocumenttypesConfig(documenttypesCfg).toString());
- }
-
private static class TestDocumentModelBuilder {
private final SearchBuilder builder = new SearchBuilder();
public TestDocumentModelBuilder addCampaign() throws ParseException {
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/DocumentTypeChangeValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/DocumentTypeChangeValidatorTest.java
index 4064e53dfb7..3dfcef70aba 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/DocumentTypeChangeValidatorTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/DocumentTypeChangeValidatorTest.java
@@ -203,6 +203,7 @@ public class DocumentTypeChangeValidatorTest {
headerfields,
new StructDataType("bodyfields"),
new FieldSets(),
+ Collections.emptySet(),
Collections.emptySet());
}
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/JettyContainerModelBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/JettyContainerModelBuilderTest.java
index 68f507c810d..0f9bd506310 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/JettyContainerModelBuilderTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/JettyContainerModelBuilderTest.java
@@ -152,6 +152,14 @@ public class JettyContainerModelBuilderTest extends ContainerModelBuilderTestBas
" <client-authentication>need</client-authentication>",
" </ssl>",
" </server>",
+ " <server port='9003' id='with-ciphers-and-protocols'>",
+ " <ssl>",
+ " <private-key-file>/foo/key</private-key-file>",
+ " <certificate-file>/foo/cert</certificate-file>",
+ " <cipher-suites>TLS_AES_128_GCM_SHA256,TLS_AES_256_GCM_SHA384</cipher-suites>",
+ " <protocols>TLSv1.3</protocols>",
+ " </ssl>",
+ " </server>",
" </http>",
nodesXml,
"",
@@ -179,6 +187,13 @@ public class JettyContainerModelBuilderTest extends ContainerModelBuilderTestBas
assertThat(needClientAuth.ssl().caCertificateFile(), is(equalTo("")));
assertThat(needClientAuth.ssl().clientAuth(), is(equalTo(ConnectorConfig.Ssl.ClientAuth.Enum.NEED_AUTH)));
+ ConnectorConfig withCiphersAndProtocols = root.getConfig(ConnectorConfig.class, "default/http/jdisc-jetty/with-ciphers-and-protocols/configured-ssl-provider@with-ciphers-and-protocols");
+ assertTrue(withCiphersAndProtocols.ssl().enabled());
+ assertThat(withCiphersAndProtocols.ssl().privateKeyFile(), is(equalTo("/foo/key")));
+ assertThat(withCiphersAndProtocols.ssl().certificateFile(), is(equalTo("/foo/cert")));
+ assertThat(withCiphersAndProtocols.ssl().enabledCipherSuites(), is(equalTo(List.of("TLS_AES_128_GCM_SHA256", "TLS_AES_256_GCM_SHA384"))));
+ assertThat(withCiphersAndProtocols.ssl().enabledProtocols(), is(equalTo(List.of("TLSv1.3"))));
+
ContainerCluster cluster = (ContainerCluster) root.getChildren().get("default");
List<ConnectorFactory> connectorFactories = cluster.getChildrenByTypeRecursive(ConnectorFactory.class);
connectorFactories.forEach(connectorFactory -> assertChildComponentExists(connectorFactory, ConfiguredFilebasedSslProvider.COMPONENT_CLASS));
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/ml/ImportedModelTester.java b/config-model/src/test/java/com/yahoo/vespa/model/ml/ImportedModelTester.java
index ce36ecc4a1c..7a3b76db7f8 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/ml/ImportedModelTester.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/ml/ImportedModelTester.java
@@ -1,7 +1,6 @@
// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.model.ml;
-import ai.vespa.rankingexpression.importer.vespa.VespaImporter;
import com.google.common.collect.ImmutableList;
import com.yahoo.config.model.ApplicationPackageTester;
import ai.vespa.rankingexpression.importer.configmodelview.MlModelImporter;
@@ -10,8 +9,10 @@ import com.yahoo.io.GrowableByteBuffer;
import com.yahoo.io.IOUtils;
import com.yahoo.path.Path;
import com.yahoo.searchdefinition.RankingConstant;
+import ai.vespa.rankingexpression.importer.lightgbm.LightGBMImporter;
import ai.vespa.rankingexpression.importer.onnx.OnnxImporter;
import ai.vespa.rankingexpression.importer.tensorflow.TensorFlowImporter;
+import ai.vespa.rankingexpression.importer.vespa.VespaImporter;
import ai.vespa.rankingexpression.importer.xgboost.XGBoostImporter;
import com.yahoo.tensor.Tensor;
import com.yahoo.tensor.serialization.TypedBinaryFormat;
@@ -35,6 +36,7 @@ public class ImportedModelTester {
private final ImmutableList<MlModelImporter> importers = ImmutableList.of(new TensorFlowImporter(),
new OnnxImporter(),
+ new LightGBMImporter(),
new XGBoostImporter(),
new VespaImporter());
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/ml/MlModelsTest.java b/config-model/src/test/java/com/yahoo/vespa/model/ml/MlModelsTest.java
index c5c475360a3..ced7243adf5 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/ml/MlModelsTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/ml/MlModelsTest.java
@@ -45,7 +45,7 @@ public class MlModelsTest {
private void verify(VespaModel model) {
assertEquals("Global models are created (although not used directly here",
- 4, model.rankProfileList().getRankProfiles().size());
+ 5, model.rankProfileList().getRankProfiles().size());
RankProfilesConfig.Builder builder = new RankProfilesConfig.Builder();
model.getSearchClusters().get(0).getConfig(builder);
@@ -71,8 +71,9 @@ public class MlModelsTest {
"rankingExpression(mnist_softmax_tensorflow).rankingScript: join(reduce(join(rename(rankingExpression(Placeholder), (d0, d1), (d0, d2)), constant(mnist_softmax_saved_layer_Variable_read), f(a,b)(a * b)), sum, d2), constant(mnist_softmax_saved_layer_Variable_1_read), f(a,b)(a + b))\n" +
"rankingExpression(mnist_softmax_onnx).rankingScript: join(reduce(join(rename(rankingExpression(Placeholder), (d0, d1), (d0, d2)), constant(mnist_softmax_Variable), f(a,b)(a * b)), sum, d2), constant(mnist_softmax_Variable_1), f(a,b)(a + b))\n" +
"rankingExpression(my_xgboost).rankingScript: if (f29 < -0.1234567, if (!(f56 >= -0.242398), 1.71218, -1.70044), if (f109 < 0.8723473, -1.94071, 1.85965)) + if (!(f60 >= -0.482947), if (f29 < -4.2387498, 0.784718, -0.96853), -6.23624)\n" +
+ "rankingExpression(my_lightgbm).rankingScript: if (!(numerical_2 >= 0.46643291586559305), 2.1594397038037663, if (categorical_2 in [\"k\", \"l\", \"m\"], 2.235297305276056, 2.1792953471546546)) + if (categorical_1 in [\"d\", \"e\"], 0.03070842919354316, if (!(numerical_1 >= 0.5102250691730842), -0.04439151147520909, 0.005117411709368601)) + if (!(numerical_2 >= 0.668665477622446), if (!(numerical_2 >= 0.008118820676863816), -0.15361238490967524, -0.01192330846157292), 0.03499044894987518) + if (!(numerical_1 >= 0.5201391072644542), -0.02141000620783247, if (categorical_1 in [\"a\", \"b\"], -0.004121485787596721, 0.04534090904886873)) + if (categorical_2 in [\"k\", \"l\", \"m\"], if (!(numerical_2 >= 0.27283279016959255), -0.01924803254356527, 0.03643772842347651), -0.02701711918923075)\n" +
"vespa.rank.firstphase: rankingExpression(firstphase)\n" +
- "rankingExpression(firstphase).rankingScript: rankingExpression(mnist_tensorflow) + rankingExpression(mnist_softmax_tensorflow) + rankingExpression(mnist_softmax_onnx) + rankingExpression(my_xgboost)\n" +
+ "rankingExpression(firstphase).rankingScript: rankingExpression(mnist_tensorflow) + rankingExpression(mnist_softmax_tensorflow) + rankingExpression(mnist_softmax_onnx) + rankingExpression(my_xgboost) + rankingExpression(my_lightgbm)\n" +
"vespa.type.attribute.argument: tensor<float>(d0[],d1[784])\n";
}
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/ml/ModelEvaluationTest.java b/config-model/src/test/java/com/yahoo/vespa/model/ml/ModelEvaluationTest.java
index 3d4ac7f2eeb..2d3ddc33afb 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/ml/ModelEvaluationTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/ml/ModelEvaluationTest.java
@@ -96,9 +96,10 @@ public class ModelEvaluationTest {
cluster.getConfig(cb);
RankingConstantsConfig constantsConfig = new RankingConstantsConfig(cb);
- assertEquals(4, config.rankprofile().size());
+ assertEquals(5, config.rankprofile().size());
Set<String> modelNames = config.rankprofile().stream().map(v -> v.name()).collect(Collectors.toSet());
assertTrue(modelNames.contains("xgboost_2_2"));
+ assertTrue(modelNames.contains("lightgbm_regression"));
assertTrue(modelNames.contains("mnist_saved"));
assertTrue(modelNames.contains("mnist_softmax"));
assertTrue(modelNames.contains("mnist_softmax_saved"));
@@ -112,13 +113,18 @@ public class ModelEvaluationTest {
ModelsEvaluator evaluator = new ModelsEvaluator(new ToleratingMissingConstantFilesRankProfilesConfigImporter(MockFileAcquirer.returnFile(null))
.importFrom(config, constantsConfig));
- assertEquals(4, evaluator.models().size());
+ assertEquals(5, evaluator.models().size());
Model xgboost = evaluator.models().get("xgboost_2_2");
assertNotNull(xgboost);
assertNotNull(xgboost.evaluatorOf());
assertNotNull(xgboost.evaluatorOf("xgboost_2_2"));
+ Model lightgbm = evaluator.models().get("lightgbm_regression");
+ assertNotNull(lightgbm);
+ assertNotNull(lightgbm.evaluatorOf());
+ assertNotNull(lightgbm.evaluatorOf("lightgbm_regression"));
+
Model tensorflow_mnist = evaluator.models().get("mnist_saved");
assertNotNull(tensorflow_mnist);
assertEquals(1, tensorflow_mnist.functions().size());
diff --git a/config-model/src/test/schema-test-files/services.xml b/config-model/src/test/schema-test-files/services.xml
index 2bbd98f72ac..1bf42650123 100644
--- a/config-model/src/test/schema-test-files/services.xml
+++ b/config-model/src/test/schema-test-files/services.xml
@@ -119,6 +119,13 @@
<certificate-file>/foo/cert</certificate-file>
<ca-certificates-file>/foo/cacerts</ca-certificates-file>
<client-authentication>want</client-authentication>
+ <cipher-suites>
+ TLS_AES_128_GCM_SHA256,
+ TLS_AES_256_GCM_SHA384,
+ TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,
+ TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384
+ </cipher-suites>
+ <protocols>TLSv1.2,TLSv1.3</protocols>
</ssl>
</server>
<server port="4083" id="sslProvider">
diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/zone/RoutingMethod.java b/config-provisioning/src/main/java/com/yahoo/config/provision/zone/RoutingMethod.java
new file mode 100644
index 00000000000..892ac639198
--- /dev/null
+++ b/config-provisioning/src/main/java/com/yahoo/config/provision/zone/RoutingMethod.java
@@ -0,0 +1,17 @@
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.config.provision.zone;
+
+/**
+ * The routing methods supported by a zone.
+ *
+ * @author mpolden
+ */
+public enum RoutingMethod {
+
+ /** Routing happens through shared routing layer */
+ shared,
+
+ /** Routing happens through a dedicated layer 4 load balancer */
+ exclusive,
+
+}
diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/zone/ZoneFilter.java b/config-provisioning/src/main/java/com/yahoo/config/provision/zone/ZoneFilter.java
index 46efe7a440d..5ef23c80eac 100644
--- a/config-provisioning/src/main/java/com/yahoo/config/provision/zone/ZoneFilter.java
+++ b/config-provisioning/src/main/java/com/yahoo/config/provision/zone/ZoneFilter.java
@@ -1,4 +1,4 @@
-// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.config.provision.zone;
import com.yahoo.config.provision.CloudName;
@@ -22,6 +22,9 @@ public interface ZoneFilter {
/** Zones which support direct routing through exclusive load balancers. */
ZoneList directlyRouted();
+ /** Zones where traffic is routed using given method */
+ ZoneList routingMethod(RoutingMethod method);
+
/** Zones where config servers are up and running. */
ZoneList reachable();
diff --git a/config/src/main/java/com/yahoo/config/subscription/ConfigInstanceSerializer.java b/config/src/main/java/com/yahoo/config/subscription/ConfigInstanceSerializer.java
index 1069f1cdd53..7c042656bda 100644
--- a/config/src/main/java/com/yahoo/config/subscription/ConfigInstanceSerializer.java
+++ b/config/src/main/java/com/yahoo/config/subscription/ConfigInstanceSerializer.java
@@ -9,7 +9,6 @@ import com.yahoo.slime.Slime;
* Implements a config instance serializer, serializing a config instance to a slime object.
*
* @author Ulf Lilleengen
- * @since 5.1.14
*/
public class ConfigInstanceSerializer implements Serializer {
private final Slime slime;
diff --git a/config/src/main/java/com/yahoo/config/subscription/ConfigInstanceUtil.java b/config/src/main/java/com/yahoo/config/subscription/ConfigInstanceUtil.java
index fb2a3acbfdc..9710ee607eb 100644
--- a/config/src/main/java/com/yahoo/config/subscription/ConfigInstanceUtil.java
+++ b/config/src/main/java/com/yahoo/config/subscription/ConfigInstanceUtil.java
@@ -38,9 +38,7 @@ public class ConfigInstanceUtil {
}
}
- public static <T extends ConfigInstance> T getNewInstance(Class<T> type,
- String configId,
- ConfigPayload payload) {
+ public static <T extends ConfigInstance> T getNewInstance(Class<T> type, String configId, ConfigPayload payload) {
T instance;
try {
ConfigTransformer<?> transformer = new ConfigTransformer<>(type);
diff --git a/config/src/main/java/com/yahoo/config/subscription/ConfigInterruptedException.java b/config/src/main/java/com/yahoo/config/subscription/ConfigInterruptedException.java
index 1eaada5ae9d..121dba1e555 100644
--- a/config/src/main/java/com/yahoo/config/subscription/ConfigInterruptedException.java
+++ b/config/src/main/java/com/yahoo/config/subscription/ConfigInterruptedException.java
@@ -4,7 +4,6 @@ package com.yahoo.config.subscription;
/**
* This exception is thrown when any blocking call within the Config API is interrupted.
* @author Ulf Lilleengen
- * @since 5.1
*/
@SuppressWarnings("serial")
public class ConfigInterruptedException extends RuntimeException {
diff --git a/config/src/main/java/com/yahoo/config/subscription/ConfigSet.java b/config/src/main/java/com/yahoo/config/subscription/ConfigSet.java
index 57f4ad863f2..c3ceb81d3db 100644
--- a/config/src/main/java/com/yahoo/config/subscription/ConfigSet.java
+++ b/config/src/main/java/com/yahoo/config/subscription/ConfigSet.java
@@ -10,7 +10,7 @@ import com.yahoo.vespa.config.ConfigKey;
/**
* Config source as a programmatically built set of {@link com.yahoo.config.ConfigInstance}s
*
- * @author vegardh
+ * @author Vegard Havdal
*/
public class ConfigSet implements ConfigSource {
private final Map<ConfigKey<?>, ConfigInstance.Builder> configs = new ConcurrentHashMap<>();
diff --git a/config/src/main/java/com/yahoo/config/subscription/ConfigSource.java b/config/src/main/java/com/yahoo/config/subscription/ConfigSource.java
index d0eda9d27cc..0aac060abf6 100644
--- a/config/src/main/java/com/yahoo/config/subscription/ConfigSource.java
+++ b/config/src/main/java/com/yahoo/config/subscription/ConfigSource.java
@@ -4,7 +4,7 @@ package com.yahoo.config.subscription;
/**
* A type of source of config
*
- * @author vegardh
+ * @author Vegard Havdal
*/
public interface ConfigSource {
diff --git a/config/src/main/java/com/yahoo/config/subscription/ConfigSubscriber.java b/config/src/main/java/com/yahoo/config/subscription/ConfigSubscriber.java
index ae6f14ad59e..5c0b932dcce 100644
--- a/config/src/main/java/com/yahoo/config/subscription/ConfigSubscriber.java
+++ b/config/src/main/java/com/yahoo/config/subscription/ConfigSubscriber.java
@@ -25,7 +25,7 @@ import static java.util.stream.Collectors.toList;
* {@link #subscribe(Class, String)} on the configs needed, call {@link #nextConfig(long)} and get the config from the
* {@link ConfigHandle} which {@link #subscribe(Class, String)} returned.
*
- * @author vegardh
+ * @author Vegard Havdal
*/
public class ConfigSubscriber implements AutoCloseable {
diff --git a/config/src/main/java/com/yahoo/config/subscription/DirSource.java b/config/src/main/java/com/yahoo/config/subscription/DirSource.java
index d90a0493838..6e12a38faa3 100644
--- a/config/src/main/java/com/yahoo/config/subscription/DirSource.java
+++ b/config/src/main/java/com/yahoo/config/subscription/DirSource.java
@@ -5,8 +5,7 @@ import java.io.File;
/**
* Source specifying config from a local directory
- * @author vegardh
- * @since 5.1
+ * @author Vegard Havdal
*
*/
public class DirSource implements ConfigSource {
diff --git a/config/src/main/java/com/yahoo/config/subscription/FileSource.java b/config/src/main/java/com/yahoo/config/subscription/FileSource.java
index 27129853eae..9b6fb6442e8 100644
--- a/config/src/main/java/com/yahoo/config/subscription/FileSource.java
+++ b/config/src/main/java/com/yahoo/config/subscription/FileSource.java
@@ -6,7 +6,7 @@ import java.io.File;
/**
* Source specifying config from one local file
*
- * @author vegardh
+ * @author Vegard Havdal
*/
public class FileSource implements ConfigSource {
diff --git a/config/src/main/java/com/yahoo/config/subscription/JarSource.java b/config/src/main/java/com/yahoo/config/subscription/JarSource.java
index 6275befe18c..a098dec7f21 100644
--- a/config/src/main/java/com/yahoo/config/subscription/JarSource.java
+++ b/config/src/main/java/com/yahoo/config/subscription/JarSource.java
@@ -5,8 +5,7 @@ import java.util.jar.JarFile;
/**
* Source specifying config as a jar file entry
- * @author vegardh
- * @since 5.1
+ * @author Vegard Havdal
*
*/
public class JarSource implements ConfigSource {
diff --git a/config/src/main/java/com/yahoo/config/subscription/RawSource.java b/config/src/main/java/com/yahoo/config/subscription/RawSource.java
index d68b503eb74..099a6bdc334 100644
--- a/config/src/main/java/com/yahoo/config/subscription/RawSource.java
+++ b/config/src/main/java/com/yahoo/config/subscription/RawSource.java
@@ -4,7 +4,7 @@ package com.yahoo.config.subscription;
/**
* Source specifying raw config, where payload is given programmatically
*
- * @author vegardh
+ * @author Vegard Havdal
*/
public class RawSource implements ConfigSource {
diff --git a/config/src/main/java/com/yahoo/config/subscription/impl/ConfigSetSubscription.java b/config/src/main/java/com/yahoo/config/subscription/impl/ConfigSetSubscription.java
index aa80cc75ef0..d3a1bfb50a9 100644
--- a/config/src/main/java/com/yahoo/config/subscription/impl/ConfigSetSubscription.java
+++ b/config/src/main/java/com/yahoo/config/subscription/impl/ConfigSetSubscription.java
@@ -11,8 +11,7 @@ import java.lang.reflect.Constructor;
/**
* Subscription on a programmatically built set of configs
- * @author vegardh
- * @since 5.1
+ * @author Vegard Havdal
*/
public class ConfigSetSubscription<T extends ConfigInstance> extends ConfigSubscription<T> {
diff --git a/config/src/main/java/com/yahoo/config/subscription/impl/ConfigSubscription.java b/config/src/main/java/com/yahoo/config/subscription/impl/ConfigSubscription.java
index c02301a0c17..3bf6093e872 100644
--- a/config/src/main/java/com/yahoo/config/subscription/impl/ConfigSubscription.java
+++ b/config/src/main/java/com/yahoo/config/subscription/impl/ConfigSubscription.java
@@ -21,7 +21,7 @@ import com.yahoo.vespa.config.protocol.DefContent;
/**
* Represents one active subscription to one config
*
- * @author vegardh
+ * @author Vegard Havdal
*/
public abstract class ConfigSubscription<T extends ConfigInstance> {
diff --git a/config/src/main/java/com/yahoo/config/subscription/impl/FileConfigSubscription.java b/config/src/main/java/com/yahoo/config/subscription/impl/FileConfigSubscription.java
index bcee06cd667..60c920d1dfa 100644
--- a/config/src/main/java/com/yahoo/config/subscription/impl/FileConfigSubscription.java
+++ b/config/src/main/java/com/yahoo/config/subscription/impl/FileConfigSubscription.java
@@ -18,7 +18,7 @@ import com.yahoo.log.LogLevel;
/**
* Subscription used when config id is file:...
*
- * @author vegardh
+ * @author Vegard Havdal
*/
public class FileConfigSubscription<T extends ConfigInstance> extends ConfigSubscription<T> {
diff --git a/config/src/main/java/com/yahoo/config/subscription/impl/GenericConfigHandle.java b/config/src/main/java/com/yahoo/config/subscription/impl/GenericConfigHandle.java
index bf247dffb7c..a40da85c88b 100644
--- a/config/src/main/java/com/yahoo/config/subscription/impl/GenericConfigHandle.java
+++ b/config/src/main/java/com/yahoo/config/subscription/impl/GenericConfigHandle.java
@@ -7,7 +7,7 @@ import com.yahoo.vespa.config.RawConfig;
/**
* A config handle which does not use the config class, but payload instead. Used in config proxy.
*
- * @author vegardh
+ * @author Vegard Havdal
*/
@SuppressWarnings({"rawtypes", "unchecked"})
public class GenericConfigHandle extends ConfigHandle {
diff --git a/config/src/main/java/com/yahoo/config/subscription/impl/GenericConfigSubscriber.java b/config/src/main/java/com/yahoo/config/subscription/impl/GenericConfigSubscriber.java
index 82d4708f53b..324546230d9 100644
--- a/config/src/main/java/com/yahoo/config/subscription/impl/GenericConfigSubscriber.java
+++ b/config/src/main/java/com/yahoo/config/subscription/impl/GenericConfigSubscriber.java
@@ -16,7 +16,7 @@ import com.yahoo.vespa.config.TimingValues;
/**
* A subscriber that can subscribe without the class. Used by configproxy.
*
- * @author vegardh
+ * @author Vegard Havdal
*/
public class GenericConfigSubscriber extends ConfigSubscriber {
diff --git a/config/src/main/java/com/yahoo/config/subscription/impl/GenericJRTConfigSubscription.java b/config/src/main/java/com/yahoo/config/subscription/impl/GenericJRTConfigSubscription.java
index c1f9ce02650..0887ed9aad5 100644
--- a/config/src/main/java/com/yahoo/config/subscription/impl/GenericJRTConfigSubscription.java
+++ b/config/src/main/java/com/yahoo/config/subscription/impl/GenericJRTConfigSubscription.java
@@ -16,7 +16,7 @@ import com.yahoo.vespa.config.protocol.JRTClientConfigRequest;
* A JRT subscription which does not use the config class, but {@link com.yahoo.vespa.config.RawConfig} instead.
* Used by config proxy.
*
- * @author vegardh
+ * @author Vegard Havdal
*
*/
public class GenericJRTConfigSubscription extends JRTConfigSubscription<RawConfig> {
diff --git a/config/src/main/java/com/yahoo/config/subscription/impl/JRTConfigRequester.java b/config/src/main/java/com/yahoo/config/subscription/impl/JRTConfigRequester.java
index 06c772ac456..6390be7b59c 100644
--- a/config/src/main/java/com/yahoo/config/subscription/impl/JRTConfigRequester.java
+++ b/config/src/main/java/com/yahoo/config/subscription/impl/JRTConfigRequester.java
@@ -233,7 +233,7 @@ public class JRTConfigRequester implements RequestWaiter {
suspendWarningLogged = Instant.MIN;
noApplicationWarningLogged = Instant.MIN;
connection.setSuccess();
- sub.setLastCallBackOKTS(System.currentTimeMillis());
+ sub.setLastCallBackOKTS(Instant.now());
if (jrtReq.hasUpdatedGeneration()) {
// We only want this latest generation to be in the queue, we do not preserve history in this system
sub.getReqQueue().clear();
diff --git a/config/src/main/java/com/yahoo/config/subscription/impl/JRTConfigSubscription.java b/config/src/main/java/com/yahoo/config/subscription/impl/JRTConfigSubscription.java
index d9366c28b9b..8327f1c1351 100644
--- a/config/src/main/java/com/yahoo/config/subscription/impl/JRTConfigSubscription.java
+++ b/config/src/main/java/com/yahoo/config/subscription/impl/JRTConfigSubscription.java
@@ -1,6 +1,8 @@
// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.config.subscription.impl;
+import java.time.Duration;
+import java.time.Instant;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.TimeUnit;
@@ -28,8 +30,8 @@ public class JRTConfigSubscription<T extends ConfigInstance> extends ConfigSubsc
private JRTConfigRequester requester;
private TimingValues timingValues;
- // Last time we got an OK JRT callback for this
- private long lastOK = 0;
+ // Last time we got an OK JRT callback
+ private Instant lastOK = Instant.MIN;
/**
* The queue containing either nothing or the one (newest) request that has got callback from JRT,
@@ -40,9 +42,9 @@ public class JRTConfigSubscription<T extends ConfigInstance> extends ConfigSubsc
public JRTConfigSubscription(ConfigKey<T> key, ConfigSubscriber subscriber, ConfigSource source, TimingValues timingValues) {
super(key, subscriber);
- this.timingValues=timingValues;
+ this.timingValues = timingValues;
if (source instanceof ConfigSourceSet) {
- this.sources=(ConfigSourceSet) source;
+ this.sources = (ConfigSourceSet) source;
}
}
@@ -53,7 +55,7 @@ public class JRTConfigSubscription<T extends ConfigInstance> extends ConfigSubsc
ConfigState<T> configState = getConfigState();
boolean gotNew = configState.isGenerationChanged() || configState.isConfigChanged() || hasException();
// Return that now, if there's nothing in queue, so that ConfigSubscriber can move on to other subscriptions to check
- if (getReqQueue().peek()==null && gotNew) {
+ if (getReqQueue().peek() == null && gotNew) {
return true;
}
// Otherwise poll the queue for another generation or timeout
@@ -135,11 +137,11 @@ public class JRTConfigSubscription<T extends ConfigInstance> extends ConfigSubsc
@Override
public boolean subscribe(long timeout) {
- lastOK=System.currentTimeMillis();
+ lastOK = Instant.now();
requester = getRequester();
requester.request(this);
JRTClientConfigRequest req = reqQueue.peek();
- while (req == null && (System.currentTimeMillis() - lastOK <= timeout)) {
+ while (req == null && (Instant.now().isBefore(lastOK.plus(Duration.ofMillis(timeout))))) {
try {
Thread.sleep(10);
} catch (InterruptedException e) {
@@ -152,7 +154,7 @@ public class JRTConfigSubscription<T extends ConfigInstance> extends ConfigSubsc
private JRTConfigRequester getRequester() {
JRTConfigRequester requester = subscriber.requesters().get(sources);
- if (requester==null) {
+ if (requester == null) {
requester = new JRTConfigRequester(new JRTConnectionPool(sources), timingValues);
subscriber.requesters().put(sources, requester);
}
@@ -164,8 +166,9 @@ public class JRTConfigSubscription<T extends ConfigInstance> extends ConfigSubsc
public void close() {
super.close();
reqQueue = new LinkedBlockingQueue<>() {
+ @SuppressWarnings("NullableProblems")
@Override
- public void put(JRTClientConfigRequest e) throws InterruptedException {
+ public void put(JRTClientConfigRequest e) {
// When closed, throw away all requests that callbacks try to put
}
};
@@ -191,7 +194,7 @@ public class JRTConfigSubscription<T extends ConfigInstance> extends ConfigSubsc
log.log(LogLevel.DEBUG, "reload() is without effect on a JRTConfigSubscription.");
}
- void setLastCallBackOKTS(long lastCallBackOKTS) {
+ void setLastCallBackOKTS(Instant lastCallBackOKTS) {
this.lastOK = lastCallBackOKTS;
}
diff --git a/config/src/main/java/com/yahoo/config/subscription/impl/JarConfigSubscription.java b/config/src/main/java/com/yahoo/config/subscription/impl/JarConfigSubscription.java
index a96499482c5..05da9a72837 100644
--- a/config/src/main/java/com/yahoo/config/subscription/impl/JarConfigSubscription.java
+++ b/config/src/main/java/com/yahoo/config/subscription/impl/JarConfigSubscription.java
@@ -21,7 +21,7 @@ import com.yahoo.vespa.config.ConfigPayload;
/**
* Subscription to use when config id is jar:.../foo.jar[!/pathInJar/]
*
- * @author vegardh
+ * @author Vegard Havdal
* @author gjoranv
*/
public class JarConfigSubscription<T extends ConfigInstance> extends ConfigSubscription<T> {
@@ -33,8 +33,8 @@ public class JarConfigSubscription<T extends ConfigInstance> extends ConfigSubsc
// jar:configs/app.jar!/configs/
JarConfigSubscription(ConfigKey<T> key, ConfigSubscriber subscriber, String jarName, String path) {
super(key, subscriber);
- this.jarName=jarName;
- this.path=path;
+ this.jarName = jarName;
+ this.path = path;
}
@Override
@@ -43,17 +43,18 @@ public class JarConfigSubscription<T extends ConfigInstance> extends ConfigSubsc
// Not supporting changing the payload for jar
return true;
}
- if (zipEntry==null) {
+ if (zipEntry == null) {
// First time polled
- JarFile jarFile = null;
+ JarFile jarFile;
try {
jarFile = new JarFile(jarName);
} catch (IOException e) {
throw new IllegalArgumentException(e);
}
zipEntry = getEntry(jarFile, path);
- if (zipEntry==null) throw new IllegalArgumentException("Config '" + key.getName() + "' not found in '" + jarName + "!/" + path + "'.");
- T config = null;
+ if (zipEntry == null)
+ throw new IllegalArgumentException("Config '" + key.getName() + "' not found in '" + jarName + "!/" + path + "'.");
+ T config;
try {
ConfigPayload payload = new CfgConfigPayloadBuilder().deserialize(Arrays.asList(IOUtils.readAll(new InputStreamReader(jarFile.getInputStream(zipEntry), StandardCharsets.UTF_8)).split("\n")));
config = payload.toInstance(configClass, key.getConfigId());
@@ -78,6 +79,7 @@ public class JarConfigSubscription<T extends ConfigInstance> extends ConfigSubsc
}
return false;
}
+
/**
* Returns the entry corresponding to the ConfigInstance's defName/Version in the given directory in
* the given JarFile.
diff --git a/config/src/main/java/com/yahoo/config/subscription/impl/MockConnection.java b/config/src/main/java/com/yahoo/config/subscription/impl/MockConnection.java
index 9ad8f5c6ba2..58eed7f9e78 100644
--- a/config/src/main/java/com/yahoo/config/subscription/impl/MockConnection.java
+++ b/config/src/main/java/com/yahoo/config/subscription/impl/MockConnection.java
@@ -15,7 +15,6 @@ import com.yahoo.vespa.config.util.ConfigUtils;
* For unit testing
*
* @author hmusum
- * @since 5.1.11
*/
public class MockConnection implements ConnectionPool, com.yahoo.vespa.config.Connection {
diff --git a/config/src/main/java/com/yahoo/config/subscription/impl/RawConfigSubscription.java b/config/src/main/java/com/yahoo/config/subscription/impl/RawConfigSubscription.java
index bc546cfdd1f..68ff6bb0135 100644
--- a/config/src/main/java/com/yahoo/config/subscription/impl/RawConfigSubscription.java
+++ b/config/src/main/java/com/yahoo/config/subscription/impl/RawConfigSubscription.java
@@ -12,10 +12,10 @@ import com.yahoo.vespa.config.ConfigPayload;
/**
* Subscription used when config id is raw:...
- *
+ * <p>
* Config is the actual text given after the config id, with newlines
*
- * @author vegardh
+ * @author Vegard Havdal
*/
public class RawConfigSubscription<T extends ConfigInstance> extends ConfigSubscription<T> {
@@ -24,7 +24,7 @@ public class RawConfigSubscription<T extends ConfigInstance> extends ConfigSubsc
RawConfigSubscription(ConfigKey<T> key, ConfigSubscriber subscriber, String pl) {
super(key, subscriber);
- this.inputPayload=pl;
+ this.inputPayload = pl;
}
@Override
@@ -32,7 +32,7 @@ public class RawConfigSubscription<T extends ConfigInstance> extends ConfigSubsc
if (checkReloaded()) {
return true;
}
- if (payload==null) {
+ if (payload == null) {
payload = inputPayload;
ConfigPayload configPayload = new CfgConfigPayloadBuilder().deserialize(Arrays.asList(payload.split("\n")));
setConfig(0L, false, configPayload.toInstance(configClass, key.getConfigId()));
diff --git a/config/src/main/java/com/yahoo/vespa/config/ConfigKey.java b/config/src/main/java/com/yahoo/vespa/config/ConfigKey.java
index 55cb05ec230..6fcdc19cf97 100755
--- a/config/src/main/java/com/yahoo/vespa/config/ConfigKey.java
+++ b/config/src/main/java/com/yahoo/vespa/config/ConfigKey.java
@@ -78,7 +78,7 @@ public class ConfigKey<CONFIGCLASS extends ConfigInstance> implements Comparable
if (!(o instanceof ConfigKey)) {
return false;
}
- ConfigKey<?> key = (ConfigKey) o;
+ ConfigKey<?> key = (ConfigKey<?>) o;
return (name.equals(key.name) &&
configId.equals(key.configId) &&
namespace.equals(key.namespace));
diff --git a/config/src/main/java/com/yahoo/vespa/config/ConfigPayload.java b/config/src/main/java/com/yahoo/vespa/config/ConfigPayload.java
index 4f02df04b9b..f4858843574 100644
--- a/config/src/main/java/com/yahoo/vespa/config/ConfigPayload.java
+++ b/config/src/main/java/com/yahoo/vespa/config/ConfigPayload.java
@@ -20,7 +20,6 @@ import java.io.OutputStream;
* A class that holds a representation of a config payload.
*
* @author Ulf Lilleengen
- * @since 5.1.6
*/
public class ConfigPayload {
private final Slime slime;
diff --git a/config/src/main/java/com/yahoo/vespa/config/ConfigPayloadApplier.java b/config/src/main/java/com/yahoo/vespa/config/ConfigPayloadApplier.java
index 65106f158fc..97e85f1a39b 100644
--- a/config/src/main/java/com/yahoo/vespa/config/ConfigPayloadApplier.java
+++ b/config/src/main/java/com/yahoo/vespa/config/ConfigPayloadApplier.java
@@ -33,7 +33,6 @@ import java.util.logging.Logger;
* TODO: This can be refactored a lot, since many of the reflection methods are duplicated
*
* @author Ulf Lilleengen, hmusum, Tony Vaagenes
- * @since 5.1.6
*/
public class ConfigPayloadApplier<T extends ConfigInstance.Builder> {
private final static Logger log = Logger.getLogger(ConfigPayloadApplier.class.getPackage().getName());
diff --git a/config/src/main/java/com/yahoo/vespa/config/ConfigPayloadBuilder.java b/config/src/main/java/com/yahoo/vespa/config/ConfigPayloadBuilder.java
index bb974ddae42..2e470ec55e4 100644
--- a/config/src/main/java/com/yahoo/vespa/config/ConfigPayloadBuilder.java
+++ b/config/src/main/java/com/yahoo/vespa/config/ConfigPayloadBuilder.java
@@ -12,7 +12,6 @@ import java.util.*;
*
* TODO: Add toString
* @author Ulf Lilleengen
- * @since 5.1
*/
public class ConfigPayloadBuilder {
diff --git a/config/src/main/java/com/yahoo/vespa/config/ConfigTransformer.java b/config/src/main/java/com/yahoo/vespa/config/ConfigTransformer.java
index 163e010cdd6..6d2ae3ef13e 100644
--- a/config/src/main/java/com/yahoo/vespa/config/ConfigTransformer.java
+++ b/config/src/main/java/com/yahoo/vespa/config/ConfigTransformer.java
@@ -12,7 +12,6 @@ import static com.yahoo.vespa.config.ConfigPayloadApplier.IdentityPathAcquirer;
* A utility class that can be used to transform config from one format to another.
*
* @author Ulf Lilleengen, hmusum, Tony Vaagenes
- * @since 5.1.6
*/
public class ConfigTransformer<T extends ConfigInstance> {
/**
diff --git a/config/src/main/java/com/yahoo/vespa/config/DefaultValueApplier.java b/config/src/main/java/com/yahoo/vespa/config/DefaultValueApplier.java
index f34b66e17a8..5bbc985cce0 100644
--- a/config/src/main/java/com/yahoo/vespa/config/DefaultValueApplier.java
+++ b/config/src/main/java/com/yahoo/vespa/config/DefaultValueApplier.java
@@ -11,7 +11,6 @@ import com.yahoo.slime.*;
* TODO: Support giving correct type of default values
*
* @author Ulf Lilleengen
- * @since 5.1
*/
public class DefaultValueApplier {
@@ -31,21 +30,11 @@ public class DefaultValueApplier {
}
private void applyDefaultsToMap(final Cursor cursor, final InnerCNode def) {
- cursor.traverse(new ObjectTraverser() {
- @Override
- public void field(String name, Inspector inspector) {
- applyDefaultsToObject(cursor.field(name), def);
- }
- });
+ cursor.traverse((ObjectTraverser) (name, inspector) -> applyDefaultsToObject(cursor.field(name), def));
}
private void applyDefaultsToArray(final Cursor cursor, final InnerCNode def) {
- cursor.traverse(new ArrayTraverser() {
- @Override
- public void entry(int idx, Inspector inspector) {
- applyDefaultsToObject(cursor.entry(idx), def);
- }
- });
+ cursor.traverse((ArrayTraverser) (idx, inspector) -> applyDefaultsToObject(cursor.entry(idx), def));
}
private void applyDefaultsToObject(Cursor cursor, InnerCNode def) {
diff --git a/config/src/main/java/com/yahoo/vespa/config/GenerationCounter.java b/config/src/main/java/com/yahoo/vespa/config/GenerationCounter.java
index c151121f0e9..3da09d07016 100644
--- a/config/src/main/java/com/yahoo/vespa/config/GenerationCounter.java
+++ b/config/src/main/java/com/yahoo/vespa/config/GenerationCounter.java
@@ -5,7 +5,6 @@ package com.yahoo.vespa.config;
* Interface for counters.
*
* @author Ulf Lilleengen
- * @since 5.9
*/
public interface GenerationCounter {
/**
diff --git a/config/src/main/java/com/yahoo/vespa/config/GenericConfig.java b/config/src/main/java/com/yahoo/vespa/config/GenericConfig.java
index 2fbbda2aa38..2f351cc2bd4 100644
--- a/config/src/main/java/com/yahoo/vespa/config/GenericConfig.java
+++ b/config/src/main/java/com/yahoo/vespa/config/GenericConfig.java
@@ -6,7 +6,6 @@ import com.yahoo.config.ConfigInstance;
/**
*
- /**
* A generic config with an internal generic builder that mimics a real config builder in order to support builders
* when we don't have the schema.
*
diff --git a/config/src/main/java/com/yahoo/vespa/config/SlimeUtils.java b/config/src/main/java/com/yahoo/vespa/config/SlimeUtils.java
index 80e93977980..89b482c03f1 100644
--- a/config/src/main/java/com/yahoo/vespa/config/SlimeUtils.java
+++ b/config/src/main/java/com/yahoo/vespa/config/SlimeUtils.java
@@ -20,12 +20,7 @@ public class SlimeUtils {
if (from.type() != Type.OBJECT) {
throw new IllegalArgumentException("Cannot copy object: " + from);
}
- from.traverse(new ObjectTraverser() {
- @Override
- public void field(String name, Inspector inspector) {
- setObjectEntry(inspector, name, to);
- }
- });
+ from.traverse((ObjectTraverser) (name, inspector) -> setObjectEntry(inspector, name, to));
}
@@ -61,13 +56,7 @@ public class SlimeUtils {
}
private static void copyArray(Inspector from, final Cursor to) {
- from.traverse(new ArrayTraverser() {
- @Override
- public void entry(int i, Inspector inspector) {
- addValue(inspector, to);
- }
- });
-
+ from.traverse((ArrayTraverser) (i, inspector) -> addValue(inspector, to));
}
private static void addValue(Inspector from, Cursor to) {
diff --git a/config/src/main/java/com/yahoo/vespa/config/buildergen/CompilationTask.java b/config/src/main/java/com/yahoo/vespa/config/buildergen/CompilationTask.java
index 7ac03ea6151..05125f8b08b 100644
--- a/config/src/main/java/com/yahoo/vespa/config/buildergen/CompilationTask.java
+++ b/config/src/main/java/com/yahoo/vespa/config/buildergen/CompilationTask.java
@@ -11,7 +11,6 @@ import javax.tools.JavaFileObject;
* TODO: Assumes that diagnostics is the same as given to the task, not ideal.
*
* @author Ulf Lilleengen
- * @since 5.2
*/
class CompilationTask {
private final JavaCompiler.CompilationTask task;
diff --git a/config/src/main/java/com/yahoo/vespa/config/buildergen/CompiledBuilder.java b/config/src/main/java/com/yahoo/vespa/config/buildergen/CompiledBuilder.java
index 80909ab653d..fa5cf427aad 100644
--- a/config/src/main/java/com/yahoo/vespa/config/buildergen/CompiledBuilder.java
+++ b/config/src/main/java/com/yahoo/vespa/config/buildergen/CompiledBuilder.java
@@ -7,7 +7,6 @@ import com.yahoo.config.ConfigInstance;
* Represents a builder that can be instantiated.
*
* @author Ulf Lilleengen
- * @since 5.2
*/
public interface CompiledBuilder {
<BUILDER extends ConfigInstance.Builder> BUILDER newInstance();
diff --git a/config/src/main/java/com/yahoo/vespa/config/buildergen/StringSourceObject.java b/config/src/main/java/com/yahoo/vespa/config/buildergen/StringSourceObject.java
index 1939a007059..c67d2121844 100644
--- a/config/src/main/java/com/yahoo/vespa/config/buildergen/StringSourceObject.java
+++ b/config/src/main/java/com/yahoo/vespa/config/buildergen/StringSourceObject.java
@@ -8,7 +8,6 @@ import java.net.URI;
* Represents an in memory source object that can be compiled.
*
* @author Ulf Lilleengen
- * @since 5.2
*/
class StringSourceObject extends SimpleJavaFileObject {
private final String code;
diff --git a/config/src/main/java/com/yahoo/vespa/config/protocol/CompressionInfo.java b/config/src/main/java/com/yahoo/vespa/config/protocol/CompressionInfo.java
index ba9fa601c81..bc6f97a3e00 100644
--- a/config/src/main/java/com/yahoo/vespa/config/protocol/CompressionInfo.java
+++ b/config/src/main/java/com/yahoo/vespa/config/protocol/CompressionInfo.java
@@ -10,7 +10,6 @@ import java.io.IOException;
* Contains info relevant for compression and decompression.
*
* @author Ulf Lilleengen
- * @since 5.19
*/
public class CompressionInfo {
private static final String COMPRESSION_TYPE = "compressionType";
diff --git a/config/src/main/java/com/yahoo/vespa/config/protocol/CompressionType.java b/config/src/main/java/com/yahoo/vespa/config/protocol/CompressionType.java
index c6548f63cd7..5fbd61cde3d 100644
--- a/config/src/main/java/com/yahoo/vespa/config/protocol/CompressionType.java
+++ b/config/src/main/java/com/yahoo/vespa/config/protocol/CompressionType.java
@@ -3,7 +3,6 @@ package com.yahoo.vespa.config.protocol;
/**
* @author Ulf Lilleengen
- * @since 5.18
*/
public enum CompressionType {
UNCOMPRESSED, LZ4;
diff --git a/config/src/main/java/com/yahoo/vespa/config/protocol/DefContent.java b/config/src/main/java/com/yahoo/vespa/config/protocol/DefContent.java
index ea2cf1a1bd8..3aa1898fcd1 100644
--- a/config/src/main/java/com/yahoo/vespa/config/protocol/DefContent.java
+++ b/config/src/main/java/com/yahoo/vespa/config/protocol/DefContent.java
@@ -12,7 +12,6 @@ import java.util.List;
/**
* @author Ulf Lilleengen
-* @since 5.3
*/
public class DefContent {
private final List<String> data;
@@ -35,12 +34,7 @@ public class DefContent {
static DefContent fromSlime(Inspector data) {
final List<String> lst = new ArrayList<>();
- data.traverse(new ArrayTraverser() {
- @Override
- public void entry(int idx, Inspector inspector) {
- lst.add(inspector.asString());
- }
- });
+ data.traverse((ArrayTraverser) (idx, inspector) -> lst.add(inspector.asString()));
return new DefContent(lst);
}
diff --git a/config/src/main/java/com/yahoo/vespa/config/protocol/JRTConfigRequestFactory.java b/config/src/main/java/com/yahoo/vespa/config/protocol/JRTConfigRequestFactory.java
index 9dec28b13e0..0861c5008c0 100644
--- a/config/src/main/java/com/yahoo/vespa/config/protocol/JRTConfigRequestFactory.java
+++ b/config/src/main/java/com/yahoo/vespa/config/protocol/JRTConfigRequestFactory.java
@@ -12,7 +12,6 @@ import java.util.*;
* To hide JRT implementations.
*
* @author Ulf Lilleengen
- * @since 5.3
*/
public class JRTConfigRequestFactory {
diff --git a/config/src/main/java/com/yahoo/vespa/config/protocol/NoCopyByteArrayOutputStream.java b/config/src/main/java/com/yahoo/vespa/config/protocol/NoCopyByteArrayOutputStream.java
index 79746127bcd..61a742ade3b 100644
--- a/config/src/main/java/com/yahoo/vespa/config/protocol/NoCopyByteArrayOutputStream.java
+++ b/config/src/main/java/com/yahoo/vespa/config/protocol/NoCopyByteArrayOutputStream.java
@@ -7,7 +7,6 @@ import java.io.ByteArrayOutputStream;
* Subclass of {@link java.io.ByteArrayOutputStream} that gives effective {@link #toByteArray()} method.
*
* @author Ulf Lilleengen
- * @since 5.19
*/
class NoCopyByteArrayOutputStream extends ByteArrayOutputStream {
public NoCopyByteArrayOutputStream() {
diff --git a/config/src/main/java/com/yahoo/vespa/config/protocol/RequestValidation.java b/config/src/main/java/com/yahoo/vespa/config/protocol/RequestValidation.java
index fed9a96504d..8c7c7f2703e 100644
--- a/config/src/main/java/com/yahoo/vespa/config/protocol/RequestValidation.java
+++ b/config/src/main/java/com/yahoo/vespa/config/protocol/RequestValidation.java
@@ -14,7 +14,6 @@ import java.util.regex.Pattern;
* Static utility methods for verifying common request properties.
*
* @author Ulf Lilleengen
- * @since 5.3
*/
public class RequestValidation {
private static final Logger log = Logger.getLogger(RequestValidation.class.getName());
diff --git a/config/src/main/java/com/yahoo/vespa/config/protocol/SlimeClientConfigRequest.java b/config/src/main/java/com/yahoo/vespa/config/protocol/SlimeClientConfigRequest.java
index 3ccf71ec519..49fdde7c9a2 100644
--- a/config/src/main/java/com/yahoo/vespa/config/protocol/SlimeClientConfigRequest.java
+++ b/config/src/main/java/com/yahoo/vespa/config/protocol/SlimeClientConfigRequest.java
@@ -39,15 +39,15 @@ public abstract class SlimeClientConfigRequest implements JRTClientConfigRequest
CompressionType compressionType,
Optional<VespaVersion> vespaVersion) {
Slime data = SlimeRequestData.encodeRequest(key,
- hostname,
- defSchema,
- configMd5,
- generation,
- timeout,
- trace,
- getProtocolVersion(),
- compressionType,
- vespaVersion);
+ hostname,
+ defSchema,
+ configMd5,
+ generation,
+ timeout,
+ trace,
+ getProtocolVersion(),
+ compressionType,
+ vespaVersion);
Request jrtReq = new Request(getJRTMethodName());
jrtReq.parameters().add(new StringValue(encodeAsUtf8String(data, true)));
@@ -134,7 +134,7 @@ public abstract class SlimeClientConfigRequest implements JRTClientConfigRequest
protected long newGen() {
long newGen = getNewGeneration();
- if (newGen==0) return getRequestGeneration();
+ if (newGen == 0) return getRequestGeneration();
return newGen;
}
diff --git a/config/src/main/java/com/yahoo/vespa/config/protocol/SlimeServerConfigRequest.java b/config/src/main/java/com/yahoo/vespa/config/protocol/SlimeServerConfigRequest.java
index 34d6f90cbcb..19dc4a4d51a 100644
--- a/config/src/main/java/com/yahoo/vespa/config/protocol/SlimeServerConfigRequest.java
+++ b/config/src/main/java/com/yahoo/vespa/config/protocol/SlimeServerConfigRequest.java
@@ -3,10 +3,12 @@ package com.yahoo.vespa.config.protocol;
import com.fasterxml.jackson.core.JsonFactory;
import com.fasterxml.jackson.core.JsonGenerator;
-import com.yahoo.jrt.*;
-import com.yahoo.slime.*;
+import com.yahoo.jrt.Request;
+import com.yahoo.jrt.StringValue;
+import com.yahoo.jrt.Value;
+import com.yahoo.slime.Slime;
import com.yahoo.text.Utf8Array;
-import com.yahoo.vespa.config.*;
+import com.yahoo.vespa.config.ConfigKey;
import com.yahoo.vespa.config.ErrorCode;
import java.io.ByteArrayOutputStream;
diff --git a/config/src/main/java/com/yahoo/vespa/config/protocol/SlimeTraceDeserializer.java b/config/src/main/java/com/yahoo/vespa/config/protocol/SlimeTraceDeserializer.java
index f4754f4b0e1..998d2c8bcf5 100644
--- a/config/src/main/java/com/yahoo/vespa/config/protocol/SlimeTraceDeserializer.java
+++ b/config/src/main/java/com/yahoo/vespa/config/protocol/SlimeTraceDeserializer.java
@@ -9,7 +9,6 @@ import com.yahoo.yolean.trace.TraceNode;
* Deserializing from a {@link Inspector} (slime) representation to a {@link TraceNode}
*
* @author Ulf Lilleengen
- * @since 5.5
*/
public class SlimeTraceDeserializer {
private final Inspector entry;
diff --git a/config/src/main/java/com/yahoo/vespa/config/protocol/SlimeTraceSerializer.java b/config/src/main/java/com/yahoo/vespa/config/protocol/SlimeTraceSerializer.java
index 62fa6f8a882..5f5f057a26b 100644
--- a/config/src/main/java/com/yahoo/vespa/config/protocol/SlimeTraceSerializer.java
+++ b/config/src/main/java/com/yahoo/vespa/config/protocol/SlimeTraceSerializer.java
@@ -12,7 +12,6 @@ import java.util.Stack;
* Serialize a {@link TraceNode} to {@link com.yahoo.slime.Slime}.
*
* @author Ulf Lilleengen
- * @since 5.5
*/
public class SlimeTraceSerializer extends TraceVisitor {
static final String TIMESTAMP = "timestamp";
@@ -30,7 +29,6 @@ public class SlimeTraceSerializer extends TraceVisitor {
current.setLong(TIMESTAMP, node.timestamp());
encodePayload(current, node.payload());
addChildrenCursors(current, node);
-
}
private void encodePayload(Cursor current, Object payload) {
diff --git a/config/src/main/java/com/yahoo/vespa/config/protocol/Trace.java b/config/src/main/java/com/yahoo/vespa/config/protocol/Trace.java
index 17740c9f40d..fbeafdc3f6f 100644
--- a/config/src/main/java/com/yahoo/vespa/config/protocol/Trace.java
+++ b/config/src/main/java/com/yahoo/vespa/config/protocol/Trace.java
@@ -13,7 +13,6 @@ import java.time.Clock;
* A trace utility that can serialize/deserialize to/from {@link Slime}
*
* @author Ulf Lilleengen
- * @since 5.3
*/
public class Trace {
private static final String TRACE_TRACELOG = "traceLog";
diff --git a/config/src/main/java/com/yahoo/vespa/config/util/ConfigUtils.java b/config/src/main/java/com/yahoo/vespa/config/util/ConfigUtils.java
index d7653572773..8f856ff4771 100644
--- a/config/src/main/java/com/yahoo/vespa/config/util/ConfigUtils.java
+++ b/config/src/main/java/com/yahoo/vespa/config/util/ConfigUtils.java
@@ -33,6 +33,7 @@ import java.util.regex.Pattern;
* Utilities for mangling config text, finding md5sums, finding name and namespace in .def files etc.
*/
public class ConfigUtils {
+
/* Patterns used for finding ranges in config definitions */
private static final Pattern intPattern = Pattern.compile(".*int.*range.*");
private static final Pattern doublePattern = Pattern.compile(".*double.*range.*");
@@ -88,6 +89,7 @@ public class ConfigUtils {
/**
* Replaces sequences of spaces with 1 space, unless inside quotes. Public for testing;
+ *
* @param str String to strip spaces from
* @return String with spaces stripped
*/
@@ -103,15 +105,15 @@ public class ConfigUtils {
}
if (!inSpaceSequence) {
// start of space sequence
- inSpaceSequence=true;
+ inSpaceSequence = true;
ret.append(" ");
}
} else {
if (inSpaceSequence) {
- inSpaceSequence=false;
+ inSpaceSequence = false;
}
- if (c=='\"') {
- inQuotes=!inQuotes;
+ if (c == '\"') {
+ inQuotes = !inQuotes;
}
ret.append(c);
}
@@ -121,7 +123,7 @@ public class ConfigUtils {
/**
* Computes Md5 hash of a list of strings with the contents of a def-file.
- *
+ * <p>
* Each string is normalized according to the
* rules of Vespa config definition files before they are used:
* <ol>
@@ -132,12 +134,12 @@ public class ConfigUtils {
* <li>Remove 'version=&lt;version-number&gt;'</li>
* </ol>
*
- * @param lines A list of lines constituting a def-file
+ * @param lines A list of lines constituting a def-file
* @return the Md5 hash of the list, with lowercase letters
*/
public static String getDefMd5(List<String> lines) {
List<String> linesCopy = new ArrayList<>(lines);
- for (Iterator<String> it=linesCopy.iterator(); it.hasNext(); ) {
+ for (Iterator<String> it = linesCopy.iterator(); it.hasNext(); ) {
String line = it.next().trim();
if (! line.startsWith("#") && ! line.equals("")) {
if (line.startsWith("version")) {
@@ -169,7 +171,7 @@ public class ConfigUtils {
}
if (line.length() > 0) {
line = stripSpaces(line);
- m = spaceBeforeCommaPatter.matcher(line);
+ m = spaceBeforeCommaPatter.matcher(line);
line = m.replaceAll(","); // Remove space before comma (for enums)
sb.append(line).append("\n");
}
@@ -188,7 +190,7 @@ public class ConfigUtils {
public static String getDefNamespace(Reader in) {
List<String> defLines = getDefLines(in);
String defPackage = getDefKeyword(defLines, "package");
- if (! defPackage.isEmpty()) return defPackage;
+ if (!defPackage.isEmpty()) return defPackage;
return getDefKeyword(defLines, "namespace");
}
@@ -285,7 +287,7 @@ public class ConfigUtils {
*/
public static ConfigDefinitionKey createConfigDefinitionKeyFromDefFile(File file) throws IOException {
String[] fileName = file.getName().split("\\.");
- assert(fileName.length >= 2);
+ assert (fileName.length >= 2);
String name = fileName[fileName.length - 2];
byte[] content = IOUtils.readFileBytes(file);
@@ -295,7 +297,7 @@ public class ConfigUtils {
/**
* Creates a ConfigDefinitionKey from a name and the content of a config definition
*
- * @param name the name of the config definition
+ * @param name the name of the config definition
* @param content content of a config definition
* @return a ConfigDefinitionKey
*/
@@ -306,6 +308,7 @@ public class ConfigUtils {
/**
* Escapes a config value according to the cfg format.
+ *
* @param input the string to escape
* @return the escaped string
*/
@@ -350,10 +353,10 @@ public class ConfigUtils {
* Loop through values and return the first one that is set and non-empty.
*
* @param defaultValue The default value to use if no environment variables are set.
- * @param envVars one or more environment variable strings
+ * @param envVars one or more environment variable strings
* @return a String with the value of the environment variable
*/
- public static String getEnvValue(String defaultValue, String ... envVars) {
+ public static String getEnvValue(String defaultValue, String... envVars) {
String value = null;
for (String envVar : envVars) {
if (value == null || value.isEmpty()) {
diff --git a/config/src/test/java/com/yahoo/config/subscription/CfgConfigPayloadBuilderTest.java b/config/src/test/java/com/yahoo/config/subscription/CfgConfigPayloadBuilderTest.java
index d4280c6300f..c8693227920 100644
--- a/config/src/test/java/com/yahoo/config/subscription/CfgConfigPayloadBuilderTest.java
+++ b/config/src/test/java/com/yahoo/config/subscription/CfgConfigPayloadBuilderTest.java
@@ -18,7 +18,6 @@ import static org.junit.Assert.assertEquals;
/**
* @author hmusum
* @author Vegard Sjonfjell
- * @since 5.1
*/
public class CfgConfigPayloadBuilderTest {
@@ -135,7 +134,7 @@ public class CfgConfigPayloadBuilderTest {
" 'boolVal': 'true'",
" },",
" {",
- " 'stringVal': 'blue a=\\\'escaped\\\'',",
+ " 'stringVal': 'blue a=\\'escaped\\'',",
" 'boolVal': 'false'",
" }",
" ],",
@@ -182,7 +181,7 @@ public class CfgConfigPayloadBuilderTest {
assertDeserializedConfigEqualsJson("a b=\"escaped\"",
inputJson(
"{",
- " 'a': 'b=\\\'escaped\\\''",
+ " 'a': 'b=\\'escaped\\''",
"}"
)
);
@@ -305,7 +304,7 @@ public class CfgConfigPayloadBuilderTest {
}
private static void assertDeserializedConfigEqualsJson(String serializedConfig, String expectedJson) {
- assertDeserializedConfigEqualsJson(Arrays.asList(serializedConfig), expectedJson);
+ assertDeserializedConfigEqualsJson(List.of(serializedConfig), expectedJson);
}
private static void assertDeserializedConfigEqualsJson(List<String> inputConfig, String expectedJson) {
diff --git a/config/src/test/java/com/yahoo/config/subscription/ConfigInstancePayloadTest.java b/config/src/test/java/com/yahoo/config/subscription/ConfigInstancePayloadTest.java
index 2d0a40bcdc4..49fd25b6476 100644
--- a/config/src/test/java/com/yahoo/config/subscription/ConfigInstancePayloadTest.java
+++ b/config/src/test/java/com/yahoo/config/subscription/ConfigInstancePayloadTest.java
@@ -22,7 +22,6 @@ import static org.junit.Assert.fail;
/**
* @author gjoranv
- * @since 5.1.6
*/
public class ConfigInstancePayloadTest {
diff --git a/config/src/test/java/com/yahoo/config/subscription/ConfigInstanceSerializationTest.java b/config/src/test/java/com/yahoo/config/subscription/ConfigInstanceSerializationTest.java
index 8d9b2d96b04..95ffb31fe7d 100644
--- a/config/src/test/java/com/yahoo/config/subscription/ConfigInstanceSerializationTest.java
+++ b/config/src/test/java/com/yahoo/config/subscription/ConfigInstanceSerializationTest.java
@@ -3,7 +3,6 @@ package com.yahoo.config.subscription;
import com.yahoo.config.ConfigInstance;
import com.yahoo.foo.FunctionTestConfig;
-import com.yahoo.config.codegen.DefLine;
import com.yahoo.vespa.config.ConfigPayload;
import org.junit.Test;
@@ -18,14 +17,6 @@ import static org.junit.Assert.assertThat;
*/
public class ConfigInstanceSerializationTest {
- private DefLine.Type stringType = new DefLine.Type("string");
- private DefLine.Type intType = new DefLine.Type("int");
- private DefLine.Type longType = new DefLine.Type("long");
- private DefLine.Type boolType = new DefLine.Type("bool");
- private DefLine.Type doubleType = new DefLine.Type("double");
- private DefLine.Type fileType = new DefLine.Type("file");
- private DefLine.Type refType = new DefLine.Type("reference");
-
@Test
public void require_symmetrical_serialization_and_deserialization_with_builder() {
FunctionTestConfig config = ConfigInstancePayloadTest.createVariableAccessConfigWithBuilder();
@@ -40,218 +31,4 @@ public class ConfigInstanceSerializationTest {
assertThat(ConfigInstance.serialize(config), is(ConfigInstance.serialize(config2)));
}
-/** Looks like everything in the commented block tests unused api's.. remove?
-
- @Test
- public void testSerializeAgainstConfigDefinitionAllowNothing() {
- FunctionTestConfig config = ConfigInstancePayloadTest.createVariableAccessConfigWithBuilder();
- InnerCNode def = new InnerCNode("function-test");
- List<String> payload = Validator.serialize(config, def);
- Assert.assertEquals(payload.size(), 0);
- }
-
- @Test
- public void testSerializeAgainstConfigDefinitionAllLeaves() {
- FunctionTestConfig config = ConfigInstancePayloadTest.createVariableAccessConfigWithBuilder();
- InnerCNode def = new InnerCNode("function-test");
- def.children().put("bool_val", LeafCNode.newInstance(boolType, def, "bool_val"));
- def.children().put("bool_with_def", LeafCNode.newInstance(boolType, def, "bool_with_def"));
- def.children().put("int_val", LeafCNode.newInstance(intType, def, "int_val"));
- def.children().put("int_with_def", LeafCNode.newInstance(intType, def, "int_with_def"));
- def.children().put("long_val", LeafCNode.newInstance(longType, def, "long_val"));
- def.children().put("long_with_def", LeafCNode.newInstance(longType, def, "long_with_def"));
- def.children().put("double_val", LeafCNode.newInstance(doubleType, def, "double_val"));
- def.children().put("double_with_def", LeafCNode.newInstance(doubleType, def, "double_with_def"));
- def.children().put("string_val", LeafCNode.newInstance(stringType, def, "string_val"));
- def.children().put("stringwithdef", LeafCNode.newInstance(stringType, def, "stringwithdef"));
- def.children().put("enum_val", LeafCNode.newInstance(enumType(new String[]{"FOO", "BAR", "FOOBAR"}), def, "enum_val"));
- def.children().put("enumwithdef", LeafCNode.newInstance(enumType(new String[]{"FOO2", "BAR2", "FOOBAR2"}), def, "enumwithdef", "BAR2"));
- def.children().put("refval", LeafCNode.newInstance(refType, def, "refval"));
- def.children().put("refwithdef", LeafCNode.newInstance(refType, def, "refwithdef"));
- def.children().put("fileVal", LeafCNode.newInstance(fileType, def, "fileVal"));
-
- List<String> payload = Validator.serialize(config, def);
- String plString = payload.toString();
- Assert.assertTrue(plString.matches(".*bool_val false.*"));
- Assert.assertTrue(plString.matches(".*bool_with_def true.*"));
- Assert.assertTrue(plString.matches(".*int_val 5.*"));
- Assert.assertTrue(plString.matches(".*int_with_def -14.*"));
- Assert.assertTrue(plString.matches(".*long_val 12345678901.*"));
- Assert.assertTrue(plString.matches(".*long_with_def -9876543210.*"));
- Assert.assertTrue(plString.matches(".*double_val 41\\.23.*"));
- Assert.assertTrue(plString.matches(".*double_with_def -12.*"));
- Assert.assertTrue(plString.matches(".*string_val \"foo\".*"));
- Assert.assertTrue(plString.matches(".*stringwithdef \"bar and foo\".*"));
- Assert.assertTrue(plString.matches(".*enum_val FOOBAR.*"));
- Assert.assertTrue(plString.matches(".*enumwithdef BAR2.*"));
- Assert.assertTrue(plString.matches(".*refval \\:parent\\:.*"));
- Assert.assertTrue(plString.matches(".*refwithdef \\:parent\\:.*"));
- Assert.assertTrue(plString.matches(".*fileVal \"etc\".*"));
- }
-
- @Test
- public void testSerializeAgainstConfigDefinitionSomeLeaves() {
- FunctionTestConfig config = ConfigInstancePayloadTest.createVariableAccessConfigWithBuilder();
- InnerCNode def = new InnerCNode("function-test");
- def.children().put("stringwithdef", LeafCNode.newInstance(stringType, def, "stringwithdef"));
- def.children().put("long_with_def", LeafCNode.newInstance(longType, def, "long_with_def"));
- // But not double_with_def etc, and no structs/arrays
- List<String> payload = Validator.serialize(config, def);
- String plString = payload.toString();
- Assert.assertTrue(plString.matches(".*long_with_def \\-9876543210.*"));
- Assert.assertTrue(plString.matches(".*stringwithdef \"bar and foo\".*"));
- Assert.assertFalse(plString.matches(".*double_with_def.*"));
- Assert.assertFalse(plString.matches(".*fileVal \"etc\".*"));
- Assert.assertFalse(plString.matches(".*basicStruct.*"));
- }
-
- @Test
- public void testSerializationAgainstConfigDefinitionAddedValsInDef() {
- FunctionTestConfig config = ConfigInstancePayloadTest.createVariableAccessConfigWithBuilder();
- InnerCNode def = new InnerCNode("function-test");
- def.children().put("stringwithdef", LeafCNode.newInstance(stringType, def, "stringwithdef"));
- def.children().put("someotherstring", LeafCNode.newInstance(stringType, def, "someotherstring", "some other"));
- def.children().put("long_with_def", LeafCNode.newInstance(longType, def, "long_with_def"));
- def.children().put("some_other_long", LeafCNode.newInstance(longType, def, "some_other_long", "88"));
- def.children().put("some_other_enum", LeafCNode.newInstance(enumType(new String[]{"hey", "ho", "lets", "go"}), def, "some_other_enum", "lets"));
- def.children().put("int_val_nofdef", LeafCNode.newInstance(intType, def, "int_val_nodef", null));
-
- // But not double_with_def etc, and no structs/arrays
- List<String> payload = Validator.serialize(config, def);
- String plString = payload.toString();
- Assert.assertTrue(plString.matches(".*long_with_def \\-9876543210.*"));
- Assert.assertTrue(plString.matches(".*stringwithdef \"bar and foo\".*"));
- Assert.assertTrue(plString.matches(".*.someotherstring \"some other\".*"));
- Assert.assertTrue(plString.matches(".*some_other_long 88.*"));
- Assert.assertTrue(plString.matches(".*some_other_enum lets.*"));
- Assert.assertFalse(plString.matches(".*double_with_def.*"));
- Assert.assertFalse(plString.matches(".*fileVal \"etc\".*"));
- Assert.assertFalse(plString.matches(".*basicStruct.*"));
- Assert.assertFalse(plString.matches(".*int_val_nodef.*"));
- }
-
- @Test
- public void testSerializeAgainstConfigDefinitionMismatchAllWays() {
- FunctionTestConfig config = ConfigInstancePayloadTest.createVariableAccessConfigWithBuilder();
-
- // Create all sorts of mismatches in the def schema used to serialize
- InnerCNode def = new InnerCNode("function-test");
- def.children().put("long_with_def", LeafCNode.newInstance(longType, def, "long_with_def"));
- def.children().put("stringwithdef", LeafCNode.newInstance(intType, def, "stringwithdef"));
- def.children().put("basicStruct", LeafCNode.newInstance(intType, def, "basicStruct"));
- InnerCNode doubleValWrong = new InnerCNode("double_val");
- doubleValWrong.children().put("foo", LeafCNode.newInstance(intType, def, "foo"));
- def.children().put("double_val", doubleValWrong);
- InnerCNode myArray = new InnerCNode("myarray");
- myArray.children().put("intval", LeafCNode.newInstance(stringType, myArray, "foo"));
- InnerCNode myStruct = new InnerCNode("myStruct");
- myStruct.children().put("a", LeafCNode.newInstance(stringType, myStruct, "foo"));
- myArray.children().put("myStruct", myStruct);
- def.children().put("myarray", myArray);
-
- List<String> payload = Validator.serialize(config, def);
- String plString = payload.toString();
- Assert.assertTrue(plString.matches(".*long_with_def.*"));
- Assert.assertFalse(plString.matches(".*stringwithdef.*"));
- Assert.assertFalse(plString.matches(".*basicStruct.*"));
- Assert.assertFalse(plString.matches(".*double_val.*"));
- Assert.assertFalse(plString.matches(".*intval.*"));
- Assert.assertFalse(plString.matches(".*\\.a.*"));
- }
-
- @Test
- public void testSerializeAgainstConfigDefinitionComplex() {
- FunctionTestConfig config = ConfigInstancePayloadTest.createVariableAccessConfigWithBuilder();
-
- // Build a pretty complex def programatically
- InnerCNode def = new InnerCNode("function-test");
- def.children().put("stringwithdef", LeafCNode.newInstance(stringType, def, "stringwithdef"));
- def.children().put("someUnknownStringNoDefault", LeafCNode.newInstance(stringType, def, "someUnknownStringNoDefault"));
- InnerCNode basicStruct = new InnerCNode("basicStruct");
- basicStruct.children().put("foo", LeafCNode.newInstance(stringType, def, "foo")); // but not bar
- InnerCNode rootStruct = new InnerCNode("rootStruct");
- InnerCNode inner1 = new InnerCNode("inner1");
- InnerCNode someUnknwonStruct = new InnerCNode("someUnknownStruct");
- InnerCNode someUnknownInner = new InnerCNode("someUnknownInner");
- InnerCNode innerArr = new InnerCNode("innerArr");
- rootStruct.children().put("inner1", inner1);
- rootStruct.children().put("someUnknownStruct", someUnknwonStruct);
- rootStruct.children().put("someUnknownInner", someUnknownInner);
- rootStruct.children().put("innerArr", innerArr);
- InnerCNode myarray = new InnerCNode("myarray");
- InnerCNode unknownInner = new InnerCNode("unknownInner");
- def.children().put("basicStruct", basicStruct);
- def.children().put("rootStruct", rootStruct);
- def.children().put("myarray", myarray);
- def.children().put("unknownInner", unknownInner);
- inner1.children().put("index", LeafCNode.newInstance(intType, inner1, "index"));
- inner1.children().put("someUnknownInt", LeafCNode.newInstance(intType, inner1, "someUnknownInt", "-98"));
- inner1.children().put("someUnknownIntNoDefault", LeafCNode.newInstance(intType, inner1, "someUnknownIntNoDefault"));
- inner1.children().put("someUnknownEnum", LeafCNode.newInstance(enumType(new String[]{"goo", "go", "gorilla"}), inner1, "someUnknownEnum", "go"));
- inner1.children().put("someUnknownEnumNoDefault", LeafCNode.newInstance(enumType(new String[]{"foo", "bar", "baz"}), inner1, "someUnknownEnumNoDefault"));
- someUnknwonStruct.children().put("anint", LeafCNode.newInstance(intType, someUnknwonStruct, "anint", "3"));// But no instances of this in config
- someUnknownInner.children().put("along", LeafCNode.newInstance(longType, someUnknownInner, "along", "234"));// No instance in config
- innerArr.children().put("boolVal", LeafCNode.newInstance(boolType, innerArr, "boolVal"));
- innerArr.children().put("someUnknownDouble", LeafCNode.newInstance(doubleType, innerArr, "someUnknownDouble", "-675.789"));
- innerArr.children().put("someUnknownDoubleNoDefault", LeafCNode.newInstance(doubleType, innerArr, "someUnknownDoubleNoDefault"));
- myarray.children().put("fileVal", LeafCNode.newInstance(fileType, myarray, "fileVal"));
- myarray.children().put("stringval", new InnerCNode("stringval[]"));
- // TODO make sure default for file is not allowed
- //myarray.children().put("someUnknownFile", LeafCNode.newInstance(fileType, myarray, "someUnknownFile", "opt/"));
- unknownInner.children().put("aDouble", LeafCNode.newInstance(doubleType, unknownInner, "aDouble", "1234"));
- def.children().put("longarr", new InnerCNode("longarr[]"));
- def.children().put("boolarr", new InnerCNode("boolarr[]"));
- def.children().put("doublearr", new InnerCNode("doublearr[]"));
- def.children().put("stringarr", new InnerCNode("stringarr[]"));
- def.children().put("fileArr", new InnerCNode("fileArr[]"));
- def.children().put("refarr", new InnerCNode("refarr[]"));
- def.children().put("enumarr", new InnerCNode("enumarr[]"));
- List<String> payload = Validator.serialize(config, def);
- String plString = payload.toString();
- Assert.assertFalse(plString.matches(".*long_with_def \\-9876543210.*"));
- Assert.assertFalse(plString.matches(".*someUnknownStringNoDefault.*"));
- Assert.assertTrue(plString.matches(".*stringwithdef \"bar and foo\".*"));
- Assert.assertFalse(plString.matches(".*double_with_def.*"));
- Assert.assertFalse(plString.matches(".*fileVal etc.*"));
- Assert.assertTrue(plString.matches(".*basicStruct\\.foo \"basicFoo\".*"));
- Assert.assertFalse(plString.matches(".*basicStruct\\.bar.*"));
- Assert.assertFalse(plString.matches(".*rootStruct\\.inner0.*"));
- Assert.assertFalse(plString.matches(".*unknownInner.*"));
- Assert.assertFalse(plString.matches(".*rootStruct\\.someUnknownStruct.*"));
- Assert.assertFalse(plString.matches(".*rootStruct\\.someUnknownInner.*"));
- Assert.assertFalse(plString.matches(".*rootStruct\\.inner1\\.name.*"));
- Assert.assertTrue(plString.matches(".*rootStruct\\.inner1\\.index 12.*"));
- Assert.assertTrue(plString.matches(".*rootStruct\\.inner1\\.someUnknownInt -98.*"));
- Assert.assertTrue(plString.matches(".*rootStruct\\.inner1\\.someUnknownEnum go.*"));
- Assert.assertTrue(plString.matches(".*rootStruct\\.innerArr\\[0\\]\\.boolVal true.*"));
- Assert.assertFalse(plString.matches(".*someUnknownEnumNoDefault.*"));
- Assert.assertFalse(plString.matches(".*someUnknownDoubleNoDefault.*"));
- Assert.assertFalse(plString.matches(".*someUnknownIntNoDefault.*"));
- Assert.assertTrue(plString.matches(".*rootStruct\\.innerArr\\[0\\]\\.someUnknownDouble -675.789.*"));
- Assert.assertFalse(plString.matches(".*rootStruct\\.innerArr\\[0\\]\\.stringVal*"));
- Assert.assertFalse(plString.matches(".*myarray\\[0\\].intval.*"));
- Assert.assertTrue(plString.matches(".*myarray\\[0\\].fileVal \"file0\".*"));
- //assertTrue(plString.matches(".*myarray\\[0\\].someUnknownFile \"opt/\".*"));
- Assert.assertTrue(plString.matches(".*myarray\\[0\\].stringval\\[0\\] \"baah\".*"));
- Assert.assertTrue(plString.matches(".*myarray\\[0\\].stringval\\[1\\] \"yikes\".*"));
- Assert.assertTrue(plString.matches(".*myarray\\[1\\].fileVal \"file1\".*"));
- Assert.assertFalse(plString.matches(".*myarray\\[1\\].enumVal.*"));
- Assert.assertFalse(plString.matches(".*myarray\\[1\\].refVal.*"));
- Assert.assertTrue(plString.matches(".*boolarr\\[0\\] false.*"));
- Assert.assertTrue(plString.matches(".*longarr\\[0\\] 9223372036854775807.*"));
- Assert.assertTrue(plString.matches(".*longarr\\[1\\] -9223372036854775808.*"));
- Assert.assertTrue(plString.matches(".*doublearr\\[0\\] 2344\\.0.*"));
- Assert.assertTrue(plString.matches(".*doublearr\\[1\\] 123\\.0.*"));
- Assert.assertTrue(plString.matches(".*stringarr\\[0\\] \"bar\".*"));
- Assert.assertTrue(plString.matches(".*enumarr\\[0\\] VALUES.*"));
- Assert.assertTrue(plString.matches(".*refarr\\[0\\] \\:parent\\:.*"));
- Assert.assertTrue(plString.matches(".*refarr\\[1\\] \\:parent.*"));
- Assert.assertTrue(plString.matches(".*refarr\\[2\\] parent\\:.*"));
- Assert.assertTrue(plString.matches(".*fileArr\\[0\\] \"bin\".*"));
- }
-
- private DefLine.Type enumType(String[] strings) {
- return new DefLine.Type("enum").setEnumArray(strings);
- }
-**/
}
diff --git a/config/src/test/java/com/yahoo/config/subscription/ConfigInstanceSerializerTest.java b/config/src/test/java/com/yahoo/config/subscription/ConfigInstanceSerializerTest.java
index ee8682efe3c..7c08d0175a1 100644
--- a/config/src/test/java/com/yahoo/config/subscription/ConfigInstanceSerializerTest.java
+++ b/config/src/test/java/com/yahoo/config/subscription/ConfigInstanceSerializerTest.java
@@ -21,9 +21,9 @@ import static org.junit.Assert.fail;
/**
* @author Ulf Lilleengen
* @author Vegard Sjonfjell
- * @since 5.1
*/
public class ConfigInstanceSerializerTest {
+
@Test
public void test_that_leaf_types_are_serialized_to_json_types() {
SimpletypesConfig.Builder builder = new SimpletypesConfig.Builder();
@@ -225,4 +225,5 @@ public class ConfigInstanceSerializerTest {
assertJsonEquals(baos.toString(), expectedJson);
}
+
}
diff --git a/config/src/test/java/com/yahoo/config/subscription/ConfigInstanceTest.java b/config/src/test/java/com/yahoo/config/subscription/ConfigInstanceTest.java
index 1da53e4c3b9..4da0c3f51e0 100644
--- a/config/src/test/java/com/yahoo/config/subscription/ConfigInstanceTest.java
+++ b/config/src/test/java/com/yahoo/config/subscription/ConfigInstanceTest.java
@@ -12,7 +12,7 @@ import static org.junit.Assert.fail;
/**
* Tests different aspects of the ConfigInstance class and its underlying Nodes.
*
- * @author <a href="gv@yahoo-inc.com">G. Voldengen</a>
+ * @author gjoranv
*/
public class ConfigInstanceTest {
private ConfigSourceSet sourceSet = new ConfigSourceSet("config-instance-test");
diff --git a/config/src/test/java/com/yahoo/config/subscription/ConfigInstanceUtilTest.java b/config/src/test/java/com/yahoo/config/subscription/ConfigInstanceUtilTest.java
index 078be819d33..3bdaee09eaf 100644
--- a/config/src/test/java/com/yahoo/config/subscription/ConfigInstanceUtilTest.java
+++ b/config/src/test/java/com/yahoo/config/subscription/ConfigInstanceUtilTest.java
@@ -22,7 +22,6 @@ import static com.yahoo.foo.FunctionTestConfig.*;
/**
* @author Ulf Lilleengen
- * @since 5.1
*/
public class ConfigInstanceUtilTest {
diff --git a/config/src/test/java/com/yahoo/config/subscription/ConfigInterruptedExceptionTest.java b/config/src/test/java/com/yahoo/config/subscription/ConfigInterruptedExceptionTest.java
index 35f4f14b4f4..a80da63afc9 100644
--- a/config/src/test/java/com/yahoo/config/subscription/ConfigInterruptedExceptionTest.java
+++ b/config/src/test/java/com/yahoo/config/subscription/ConfigInterruptedExceptionTest.java
@@ -8,9 +8,9 @@ import static org.junit.Assert.assertThat;
/**
* @author Ulf Lilleengen
- * @since 5.1
*/
public class ConfigInterruptedExceptionTest {
+
@Test
public void require_that_throwable_is_preserved() {
ConfigInterruptedException e = new ConfigInterruptedException(new RuntimeException("foo"));
diff --git a/config/src/test/java/com/yahoo/config/subscription/ConfigSetTest.java b/config/src/test/java/com/yahoo/config/subscription/ConfigSetTest.java
index a120f19c17b..48a51bbc02c 100644
--- a/config/src/test/java/com/yahoo/config/subscription/ConfigSetTest.java
+++ b/config/src/test/java/com/yahoo/config/subscription/ConfigSetTest.java
@@ -10,9 +10,9 @@ import static org.junit.Assert.assertTrue;
/**
* @author Ulf Lilleengen
- * @since 5.1
*/
public class ConfigSetTest {
+
@Test
public void testToString() {
ConfigSet set = new ConfigSet();
diff --git a/config/src/test/java/com/yahoo/config/subscription/ConfigSourceSetTest.java b/config/src/test/java/com/yahoo/config/subscription/ConfigSourceSetTest.java
index 38d4a6a4571..0879c330f45 100755
--- a/config/src/test/java/com/yahoo/config/subscription/ConfigSourceSetTest.java
+++ b/config/src/test/java/com/yahoo/config/subscription/ConfigSourceSetTest.java
@@ -13,6 +13,7 @@ import static org.junit.Assert.*;
* @author <a href="gv@yahoo-inc.com">G. Voldengen</a>
*/
public class ConfigSourceSetTest {
+
@Test
public void testEquals() {
assertEquals(new ConfigSourceSet(), new ConfigSourceSet());
diff --git a/config/src/test/java/com/yahoo/config/subscription/ConfigURITest.java b/config/src/test/java/com/yahoo/config/subscription/ConfigURITest.java
index 43e3cf658f3..555434837c8 100644
--- a/config/src/test/java/com/yahoo/config/subscription/ConfigURITest.java
+++ b/config/src/test/java/com/yahoo/config/subscription/ConfigURITest.java
@@ -12,9 +12,9 @@ import static org.junit.Assert.assertTrue;
/**
* @author Ulf Lilleengen
- * @since 5.1
*/
public class ConfigURITest {
+
@Test
public void testDefaultUri() {
ConfigURI uri = ConfigURI.createFromId("foo");
diff --git a/config/src/test/java/com/yahoo/config/subscription/DefaultConfigTest.java b/config/src/test/java/com/yahoo/config/subscription/DefaultConfigTest.java
index 85346264837..1bcf09a4028 100644
--- a/config/src/test/java/com/yahoo/config/subscription/DefaultConfigTest.java
+++ b/config/src/test/java/com/yahoo/config/subscription/DefaultConfigTest.java
@@ -24,6 +24,7 @@ import static org.junit.Assert.assertTrue;
* @author hmusum
*/
public class DefaultConfigTest {
+
static final String CONFIG_ID = "raw:" +
"nondefaultstring ####-------missing--------\n" +
"defaultstring \"thedefault\"\n" +
diff --git a/config/src/test/java/com/yahoo/config/subscription/GenericConfigSubscriberTest.java b/config/src/test/java/com/yahoo/config/subscription/GenericConfigSubscriberTest.java
index cb25de89bfb..08d215670db 100644
--- a/config/src/test/java/com/yahoo/config/subscription/GenericConfigSubscriberTest.java
+++ b/config/src/test/java/com/yahoo/config/subscription/GenericConfigSubscriberTest.java
@@ -21,7 +21,6 @@ import static org.junit.Assert.*;
* Test cases for the "generic" (class-less) subscription mechanism.
*
* @author Ulf Lilleengen
- * @since 5.1
*/
public class GenericConfigSubscriberTest {
@@ -52,7 +51,6 @@ public class GenericConfigSubscriberTest {
GenericConfigSubscriber sub = new GenericConfigSubscriber(requesters);
assertEquals(sub.requesters().get(source1).getConnectionPool().getCurrent().getAddress(), "tcp/foo:78");
assertEquals(sub.requesters().get(source2).getConnectionPool().getCurrent().getAddress(), "tcp/bar:79");
-
}
@Test(expected=UnsupportedOperationException.class)
diff --git a/config/src/test/java/com/yahoo/config/subscription/NamespaceTest.java b/config/src/test/java/com/yahoo/config/subscription/NamespaceTest.java
index 5e60c273ee4..430de894629 100644
--- a/config/src/test/java/com/yahoo/config/subscription/NamespaceTest.java
+++ b/config/src/test/java/com/yahoo/config/subscription/NamespaceTest.java
@@ -11,6 +11,7 @@ import static org.junit.Assert.assertThat;
* @author gjoranv
*/
public class NamespaceTest {
+
@Test
public void verifyConfigClassWithExplicitNamespace() {
NamespaceConfig config = new ConfigGetter<>(NamespaceConfig.class).getConfig("raw: a 0\n");
diff --git a/config/src/test/java/com/yahoo/config/subscription/UnicodeTest.java b/config/src/test/java/com/yahoo/config/subscription/UnicodeTest.java
index 6512b5ea29e..f1bd164c874 100644
--- a/config/src/test/java/com/yahoo/config/subscription/UnicodeTest.java
+++ b/config/src/test/java/com/yahoo/config/subscription/UnicodeTest.java
@@ -13,6 +13,7 @@ import static org.junit.Assert.assertEquals;
* @author Harald Musum
*/
public class UnicodeTest {
+
/**
* Reads a config from a file which is exactly like one returned from
* the config server given only default values for this config.
diff --git a/config/src/test/java/com/yahoo/config/subscription/impl/FileConfigSubscriptionTest.java b/config/src/test/java/com/yahoo/config/subscription/impl/FileConfigSubscriptionTest.java
index 2f550fc8e1e..71e7a9f08d7 100644
--- a/config/src/test/java/com/yahoo/config/subscription/impl/FileConfigSubscriptionTest.java
+++ b/config/src/test/java/com/yahoo/config/subscription/impl/FileConfigSubscriptionTest.java
@@ -25,7 +25,6 @@ import static org.junit.Assert.assertTrue;
/**
* @author Ulf Lilleengen
- * @since 5.1.7
*/
public class FileConfigSubscriptionTest {
private File TEST_TYPES_FILE;
diff --git a/config/src/test/java/com/yahoo/vespa/config/ConfigBuilderMergeTest.java b/config/src/test/java/com/yahoo/vespa/config/ConfigBuilderMergeTest.java
index c77985c91d8..137d2894164 100644
--- a/config/src/test/java/com/yahoo/vespa/config/ConfigBuilderMergeTest.java
+++ b/config/src/test/java/com/yahoo/vespa/config/ConfigBuilderMergeTest.java
@@ -18,7 +18,6 @@ import static org.junit.Assert.assertThat;
* SEO keywords: test override() on builders. overrideTest, testOverride
*
* @author Ulf Lilleengen
- * @since 5.1
*/
public class ConfigBuilderMergeTest {
diff --git a/config/src/test/java/com/yahoo/vespa/config/ConfigCacheKeyTest.java b/config/src/test/java/com/yahoo/vespa/config/ConfigCacheKeyTest.java
index bb65fdaa153..fce61cc802c 100755
--- a/config/src/test/java/com/yahoo/vespa/config/ConfigCacheKeyTest.java
+++ b/config/src/test/java/com/yahoo/vespa/config/ConfigCacheKeyTest.java
@@ -12,6 +12,7 @@ import static org.junit.Assert.*;
* @author hmusum
*/
public class ConfigCacheKeyTest {
+
@Test
public void testConfigCacheKey() {
final String defMd5 = "md5";
diff --git a/config/src/test/java/com/yahoo/vespa/config/ConfigDefinitionBuilderTest.java b/config/src/test/java/com/yahoo/vespa/config/ConfigDefinitionBuilderTest.java
index dba73223097..810b9f58829 100644
--- a/config/src/test/java/com/yahoo/vespa/config/ConfigDefinitionBuilderTest.java
+++ b/config/src/test/java/com/yahoo/vespa/config/ConfigDefinitionBuilderTest.java
@@ -12,7 +12,6 @@ import java.io.IOException;
import static org.hamcrest.CoreMatchers.is;
import static org.junit.Assert.*;
-
/**
* Unit tests for ConfigDefinitionBuilder.
*
diff --git a/config/src/test/java/com/yahoo/vespa/config/ConfigFileFormatterTest.java b/config/src/test/java/com/yahoo/vespa/config/ConfigFileFormatterTest.java
index 3cc030d944b..5198759d3e2 100644
--- a/config/src/test/java/com/yahoo/vespa/config/ConfigFileFormatterTest.java
+++ b/config/src/test/java/com/yahoo/vespa/config/ConfigFileFormatterTest.java
@@ -24,7 +24,6 @@ import static org.junit.Assert.assertThat;
/**
* @author Ulf Lilleengen
- * @since 5.1
*/
public class ConfigFileFormatterTest {
diff --git a/config/src/test/java/com/yahoo/vespa/config/ConfigPayloadBuilderTest.java b/config/src/test/java/com/yahoo/vespa/config/ConfigPayloadBuilderTest.java
index eb0c47e3b0a..c13d3ec9b9b 100644
--- a/config/src/test/java/com/yahoo/vespa/config/ConfigPayloadBuilderTest.java
+++ b/config/src/test/java/com/yahoo/vespa/config/ConfigPayloadBuilderTest.java
@@ -17,7 +17,6 @@ import static org.junit.Assert.assertThat;
/**
* @author Ulf Lilleengen
- * @since 5.1
*/
public class ConfigPayloadBuilderTest {
diff --git a/config/src/test/java/com/yahoo/vespa/config/ConfigPayloadTest.java b/config/src/test/java/com/yahoo/vespa/config/ConfigPayloadTest.java
index f1b0adc03e7..b3db6e2ab43 100644
--- a/config/src/test/java/com/yahoo/vespa/config/ConfigPayloadTest.java
+++ b/config/src/test/java/com/yahoo/vespa/config/ConfigPayloadTest.java
@@ -24,8 +24,7 @@ import static org.junit.Assert.assertThat;
import static org.junit.Assert.assertTrue;
/**
- * @author Ulf Lilleengen 3
- * @since 5.1
+ * @author Ulf Lilleengen
*/
public class ConfigPayloadTest {
diff --git a/config/src/test/java/com/yahoo/vespa/config/DefaultValueApplierTest.java b/config/src/test/java/com/yahoo/vespa/config/DefaultValueApplierTest.java
index e67489b4030..6ab3059f1c0 100644
--- a/config/src/test/java/com/yahoo/vespa/config/DefaultValueApplierTest.java
+++ b/config/src/test/java/com/yahoo/vespa/config/DefaultValueApplierTest.java
@@ -16,9 +16,9 @@ import static org.junit.Assert.assertTrue;
/**
* @author Ulf Lilleengen
- * @since 5.1
*/
public class DefaultValueApplierTest {
+
public Slime apply(Slime slime, String ... extraFields) {
StringBuilder defBuilder = new StringBuilder();
defBuilder.append("namespace=test").append("\n");
diff --git a/config/src/test/java/com/yahoo/vespa/config/ErrorCodeTest.java b/config/src/test/java/com/yahoo/vespa/config/ErrorCodeTest.java
index 8e6cc1fc6a0..64210eaa4b8 100644
--- a/config/src/test/java/com/yahoo/vespa/config/ErrorCodeTest.java
+++ b/config/src/test/java/com/yahoo/vespa/config/ErrorCodeTest.java
@@ -8,9 +8,9 @@ import static org.junit.Assert.assertThat;
/**
* @author hmusum
- * @since 5.1.9
*/
public class ErrorCodeTest {
+
@Test
public void basic() {
assertThat(ErrorCode.getName(ErrorCode.INTERNAL_ERROR), is("INTERNAL_ERROR"));
diff --git a/config/src/test/java/com/yahoo/vespa/config/ErrorTypeTest.java b/config/src/test/java/com/yahoo/vespa/config/ErrorTypeTest.java
index 450d7e8d6d8..a0b2c1185a7 100644
--- a/config/src/test/java/com/yahoo/vespa/config/ErrorTypeTest.java
+++ b/config/src/test/java/com/yahoo/vespa/config/ErrorTypeTest.java
@@ -8,7 +8,6 @@ import static org.junit.Assert.assertThat;
/**
* @author Ulf Lilleengen
- * @since 5.1
*/
public class ErrorTypeTest {
@@ -32,4 +31,5 @@ public class ErrorTypeTest {
assertThat(ErrorType.getErrorType(ErrorCode.ILLEGAL_SUB_FLAG), is(ErrorType.FATAL));
assertThat(ErrorType.getErrorType(0xdeadc0de), is(ErrorType.FATAL));
}
+
}
diff --git a/config/src/test/java/com/yahoo/vespa/config/GenericConfigBuilderTest.java b/config/src/test/java/com/yahoo/vespa/config/GenericConfigBuilderTest.java
index ee7845086c2..2c60ff95fee 100644
--- a/config/src/test/java/com/yahoo/vespa/config/GenericConfigBuilderTest.java
+++ b/config/src/test/java/com/yahoo/vespa/config/GenericConfigBuilderTest.java
@@ -13,9 +13,9 @@ import static org.junit.Assert.assertThat;
/**
* @author Ulf Lilleengen
- * @since 5.1
*/
public class GenericConfigBuilderTest {
+
@Test
public void require_that_builder_can_be_overridden() throws IOException {
ConfigPayloadBuilder ba = new ConfigPayloadBuilder();
diff --git a/config/src/test/java/com/yahoo/vespa/config/LZ4CompressionTest.java b/config/src/test/java/com/yahoo/vespa/config/LZ4CompressionTest.java
index a3125c73bea..805efc90ade 100644
--- a/config/src/test/java/com/yahoo/vespa/config/LZ4CompressionTest.java
+++ b/config/src/test/java/com/yahoo/vespa/config/LZ4CompressionTest.java
@@ -15,7 +15,6 @@ import java.nio.file.Files;
* To run this test, place a payload in src/test/ca.json. The file is not checked in because it is huge.
*
* @author Ulf Lilleengen
- * @since 5.12
*/
public class LZ4CompressionTest {
private static LZ4Factory factory = LZ4Factory.safeInstance();
diff --git a/config/src/test/java/com/yahoo/vespa/config/LZ4PayloadCompressorTest.java b/config/src/test/java/com/yahoo/vespa/config/LZ4PayloadCompressorTest.java
index 06bbaad4271..635d6a96a7c 100644
--- a/config/src/test/java/com/yahoo/vespa/config/LZ4PayloadCompressorTest.java
+++ b/config/src/test/java/com/yahoo/vespa/config/LZ4PayloadCompressorTest.java
@@ -9,9 +9,9 @@ import static org.junit.Assert.assertThat;
/**
* @author Ulf Lilleengen
- * @since 5.19
*/
public class LZ4PayloadCompressorTest {
+
@Test
public void testCompression() {
assertCompression("hei hallo der");
diff --git a/config/src/test/java/com/yahoo/vespa/config/RawConfigTest.java b/config/src/test/java/com/yahoo/vespa/config/RawConfigTest.java
index a564fea8b2e..3f6c54ea46e 100644
--- a/config/src/test/java/com/yahoo/vespa/config/RawConfigTest.java
+++ b/config/src/test/java/com/yahoo/vespa/config/RawConfigTest.java
@@ -113,11 +113,11 @@ public class RawConfigTest {
assertThat(config.getDefMd5(), is(defMd5));
config = new RawConfig(key, null, payload, configMd5, generation, false, null, Optional.empty());
assertNull(config.getDefMd5());
- config = new RawConfig(key, null, payload, configMd5, generation, false, Arrays.asList(""), Optional.empty());
+ config = new RawConfig(key, null, payload, configMd5, generation, false,List.of(""), Optional.empty());
assertThat(config.getDefMd5(), is(defMd5ForEmptyDefContent));
config = new RawConfig(key, "", payload, configMd5, generation, false, null, Optional.empty());
assertThat(config.getDefMd5(), is(""));
- config = new RawConfig(key, "", payload, configMd5, generation, false, Arrays.asList(""), Optional.empty());
+ config = new RawConfig(key, "", payload, configMd5, generation, false, List.of(""), Optional.empty());
assertThat(config.getDefMd5(), is(defMd5ForEmptyDefContent));
}
diff --git a/config/src/test/java/com/yahoo/vespa/config/SlimeUtilsTest.java b/config/src/test/java/com/yahoo/vespa/config/SlimeUtilsTest.java
index ff7b640630e..5def6d899a0 100644
--- a/config/src/test/java/com/yahoo/vespa/config/SlimeUtilsTest.java
+++ b/config/src/test/java/com/yahoo/vespa/config/SlimeUtilsTest.java
@@ -14,9 +14,9 @@ import static org.junit.Assert.assertTrue;
/**
* @author Ulf Lilleengen
- * @since 5.8
*/
public class SlimeUtilsTest {
+
@Test
public void test_copying_slime_types_into_cursor() {
Slime slime = new Slime();
@@ -79,4 +79,5 @@ public class SlimeUtilsTest {
assertThat(slime.get().field("foo").asString(), is("foobie"));
assertTrue(slime.get().field("bar").valid());
}
+
}
diff --git a/config/src/test/java/com/yahoo/vespa/config/TimingValuesTest.java b/config/src/test/java/com/yahoo/vespa/config/TimingValuesTest.java
index 3b97c360c4b..0be5e98ff0a 100644
--- a/config/src/test/java/com/yahoo/vespa/config/TimingValuesTest.java
+++ b/config/src/test/java/com/yahoo/vespa/config/TimingValuesTest.java
@@ -13,6 +13,7 @@ import static org.junit.Assert.assertThat;
* @author hmusum
*/
public class TimingValuesTest {
+
@Test
public void basic() {
TimingValues tv = new TimingValues();
diff --git a/config/src/test/java/com/yahoo/vespa/config/protocol/JRTConfigRequestBase.java b/config/src/test/java/com/yahoo/vespa/config/protocol/JRTConfigRequestBase.java
index 75ba1392fd1..db594cb8c5b 100644
--- a/config/src/test/java/com/yahoo/vespa/config/protocol/JRTConfigRequestBase.java
+++ b/config/src/test/java/com/yahoo/vespa/config/protocol/JRTConfigRequestBase.java
@@ -24,7 +24,6 @@ import com.yahoo.vespa.config.util.ConfigUtils;
import org.junit.Before;
import org.junit.Test;
-import java.io.IOException;
import java.util.Collections;
import java.util.Optional;
@@ -36,7 +35,6 @@ import static org.junit.Assert.assertTrue;
/**
* @author Ulf Lilleengen
- * @since 5.3
*/
public abstract class JRTConfigRequestBase {
@@ -54,19 +52,19 @@ public abstract class JRTConfigRequestBase {
protected JRTServerConfigRequest serverReq;
@Before
- public void setupRequest() throws IOException {
+ public void setupRequest() {
clientReq = createReq();
serverReq = createReq(clientReq.getRequest());
assertTrue(serverReq.validateParameters());
}
- private JRTClientConfigRequest createReq() throws IOException {
+ private JRTClientConfigRequest createReq() {
trace = Trace.createNew(3, new ManualClock());
trace.trace(1, "hei");
return createReq(defName, defNamespace, defMd5, hostname, configId, configMd5, currentGeneration, timeout, trace);
}
- private JRTClientConfigRequest createReq(Payload payload) throws IOException {
+ private JRTClientConfigRequest createReq(Payload payload) {
trace = Trace.createNew(3, new ManualClock());
trace.trace(1, "hei");
return createReq(defName, defNamespace, defMd5, hostname, configId, ConfigUtils.getMd5(payload.getData()), currentGeneration, timeout, trace);
@@ -195,7 +193,7 @@ public abstract class JRTConfigRequestBase {
}
@Test
- public void payload_is_empty() throws IOException {
+ public void payload_is_empty() {
Payload payload = Payload.from(ConfigPayload.empty());
clientReq = createReq(payload);
serverReq = createReq(clientReq.getRequest());
@@ -258,7 +256,7 @@ public abstract class JRTConfigRequestBase {
protected abstract String getProtocolVersion();
@Test
- public void created_from_raw() throws IOException {
+ public void created_from_raw() {
RawConfig rawConfig = new RawConfig(new ConfigKey<>(defName, configId, defNamespace), defMd5);
long serverTimeout = 100000L;
JRTClientConfigRequest request = createFromRaw(rawConfig, serverTimeout, Trace.createNew(9));
@@ -269,9 +267,8 @@ public abstract class JRTConfigRequestBase {
assertThat(serverRequest.getDefContent().asList(), is(rawConfig.getDefContent()));
}
-
@Test
- public void parameters_are_validated() throws IOException {
+ public void parameters_are_validated() {
assertTrue(serverReq.validateParameters());
assertValidationFail(createReq("35#$#!$@#", defNamespace, defMd5, hostname, configId, configMd5, currentGeneration, timeout, trace));
assertValidationFail(createReq(defName, "abcd.o#$*(!&$", defMd5, hostname, configId, configMd5, currentGeneration, timeout, trace));
diff --git a/config/src/test/java/com/yahoo/vespa/config/protocol/JRTConfigRequestV3Test.java b/config/src/test/java/com/yahoo/vespa/config/protocol/JRTConfigRequestV3Test.java
index d50a92efc1a..e176ce6e8cd 100644
--- a/config/src/test/java/com/yahoo/vespa/config/protocol/JRTConfigRequestV3Test.java
+++ b/config/src/test/java/com/yahoo/vespa/config/protocol/JRTConfigRequestV3Test.java
@@ -17,7 +17,6 @@ import static org.junit.Assert.assertTrue;
/**
* @author Ulf Lilleengen
- * @since 5.19
*/
public class JRTConfigRequestV3Test extends JRTConfigRequestBase {
diff --git a/config/src/test/java/com/yahoo/vespa/config/protocol/PayloadTest.java b/config/src/test/java/com/yahoo/vespa/config/protocol/PayloadTest.java
index e5fc5190ad1..c52245ecb35 100644
--- a/config/src/test/java/com/yahoo/vespa/config/protocol/PayloadTest.java
+++ b/config/src/test/java/com/yahoo/vespa/config/protocol/PayloadTest.java
@@ -15,7 +15,6 @@ import static org.junit.Assert.assertThat;
/**
* @author Ulf Lilleengen
- * @since 5.21
*/
public class PayloadTest {
diff --git a/config/src/test/java/com/yahoo/vespa/config/protocol/SlimeTraceSerializerTest.java b/config/src/test/java/com/yahoo/vespa/config/protocol/SlimeTraceSerializerTest.java
index f2d9edc7c35..22baab85c1c 100644
--- a/config/src/test/java/com/yahoo/vespa/config/protocol/SlimeTraceSerializerTest.java
+++ b/config/src/test/java/com/yahoo/vespa/config/protocol/SlimeTraceSerializerTest.java
@@ -21,9 +21,9 @@ import static org.junit.Assert.assertTrue;
/**
* @author Ulf Lilleengen
- * @since 5.5
*/
public class SlimeTraceSerializerTest {
+
@Test
public void test_serializer() throws IOException {
TraceNode root = new TraceNode(null, 1);
diff --git a/config/src/test/java/com/yahoo/vespa/config/protocol/TraceTest.java b/config/src/test/java/com/yahoo/vespa/config/protocol/TraceTest.java
index 596983eebb8..4b191d85121 100644
--- a/config/src/test/java/com/yahoo/vespa/config/protocol/TraceTest.java
+++ b/config/src/test/java/com/yahoo/vespa/config/protocol/TraceTest.java
@@ -9,7 +9,6 @@ import static org.junit.Assert.assertTrue;
/**
* @author Ulf Lilleengen
- * @since 5.3
*/
public class TraceTest {
@@ -58,4 +57,5 @@ public class TraceTest {
assertTrue(trace2Str.contains("barbaz"));
assertTrue(trace2Str.contains("quux"));
}
+
}
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java
index 8b2c3e2cb0a..829a59a9598 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java
@@ -134,6 +134,7 @@ public class ModelContextImpl implements ModelContext {
private final Optional<EndpointCertificateSecrets> endpointCertificateSecrets;
private final double defaultTermwiseLimit;
private final boolean useBucketSpaceMetric;
+ private final boolean useNewAthenzFilter;
public Properties(ApplicationId applicationId,
boolean multitenantFromConfig,
@@ -166,6 +167,8 @@ public class ModelContextImpl implements ModelContext {
.with(FetchVector.Dimension.APPLICATION_ID, applicationId.serializedForm()).value();
this.useBucketSpaceMetric = Flags.USE_BUCKET_SPACE_METRIC.bindTo(flagSource)
.with(FetchVector.Dimension.APPLICATION_ID, applicationId.serializedForm()).value();
+ this.useNewAthenzFilter = Flags.USE_NEW_ATHENZ_FILTER.bindTo(flagSource)
+ .with(FetchVector.Dimension.APPLICATION_ID, applicationId.serializedForm()).value();
}
@Override
@@ -219,6 +222,8 @@ public class ModelContextImpl implements ModelContext {
@Override
public boolean useBucketSpaceMetric() { return useBucketSpaceMetric; }
+
+ @Override public boolean useNewAthenzFilter() { return useNewAthenzFilter; }
}
}
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/application/OrchestratorMock.java b/configserver/src/test/java/com/yahoo/vespa/config/server/application/OrchestratorMock.java
index 33193bafc66..837cbd4605a 100644
--- a/configserver/src/test/java/com/yahoo/vespa/config/server/application/OrchestratorMock.java
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/application/OrchestratorMock.java
@@ -11,8 +11,10 @@ import com.yahoo.vespa.orchestrator.status.HostStatus;
import java.time.Instant;
import java.util.Collections;
+import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
+import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.function.Function;
@@ -24,7 +26,7 @@ import java.util.function.Function;
*/
public class OrchestratorMock implements Orchestrator {
- private final Set<HostName> suspendedHosts = new HashSet<>();
+ private final Map<HostName, HostInfo> hostInfos = new HashMap<>();
private final Set<ApplicationId> suspendedApplications = new HashSet<>();
@Override
@@ -34,14 +36,13 @@ public class OrchestratorMock implements Orchestrator {
@Override
public HostStatus getNodeStatus(HostName hostName) {
- return suspendedHosts.contains(hostName) ? HostStatus.ALLOWED_TO_BE_DOWN : HostStatus.NO_REMARKS;
+ HostInfo hostInfo = hostInfos.get(hostName);
+ return hostInfo == null ? HostStatus.NO_REMARKS : hostInfo.status();
}
@Override
- public Function<HostName, Optional<HostInfo>> getNodeStatuses() {
- return hostName -> Optional.of(getNodeStatus(hostName))
- .map(status -> status.isSuspended() ? HostInfo.createSuspended(status, Instant.EPOCH)
- : HostInfo.createNoRemarks());
+ public Function<HostName, Optional<HostInfo>> getHostResolver() {
+ return hostName -> Optional.of(hostInfos.getOrDefault(hostName, HostInfo.createNoRemarks()));
}
@Override
@@ -49,12 +50,12 @@ public class OrchestratorMock implements Orchestrator {
@Override
public void resume(HostName hostName) {
- suspendedHosts.remove(hostName);
+ hostInfos.remove(hostName);
}
@Override
public void suspend(HostName hostName) {
- suspendedHosts.add(hostName);
+ hostInfos.put(hostName, HostInfo.createSuspended(HostStatus.ALLOWED_TO_BE_DOWN, Instant.EPOCH));
}
@Override
diff --git a/container-core/abi-spec.json b/container-core/abi-spec.json
index 91c0e4cc6bf..fe8fb91d71f 100644
--- a/container-core/abi-spec.json
+++ b/container-core/abi-spec.json
@@ -11,6 +11,23 @@
],
"fields": []
},
+ "com.yahoo.container.handler.ClustersStatus$Require": {
+ "superClass": "java.lang.Enum",
+ "interfaces": [],
+ "attributes": [
+ "public",
+ "final",
+ "enum"
+ ],
+ "methods": [
+ "public static com.yahoo.container.handler.ClustersStatus$Require[] values()",
+ "public static com.yahoo.container.handler.ClustersStatus$Require valueOf(java.lang.String)"
+ ],
+ "fields": [
+ "public static final enum com.yahoo.container.handler.ClustersStatus$Require ONE",
+ "public static final enum com.yahoo.container.handler.ClustersStatus$Require ALL"
+ ]
+ },
"com.yahoo.container.handler.ClustersStatus": {
"superClass": "com.yahoo.component.AbstractComponent",
"interfaces": [],
@@ -23,7 +40,8 @@
"public void setReceiveTrafficByDefault(boolean)",
"public void setUp(java.lang.Object)",
"public void setDown(java.lang.Object)",
- "public boolean containerShouldReceiveTraffic()"
+ "public boolean containerShouldReceiveTraffic()",
+ "public boolean containerShouldReceiveTraffic(com.yahoo.container.handler.ClustersStatus$Require)"
],
"fields": []
},
diff --git a/container-core/src/main/java/com/yahoo/container/handler/ClustersStatus.java b/container-core/src/main/java/com/yahoo/container/handler/ClustersStatus.java
index aab13a1cc7b..0ed0daa2141 100644
--- a/container-core/src/main/java/com/yahoo/container/handler/ClustersStatus.java
+++ b/container-core/src/main/java/com/yahoo/container/handler/ClustersStatus.java
@@ -27,6 +27,8 @@ public class ClustersStatus extends AbstractComponent {
@Inject
public ClustersStatus() { }
+ public enum Require {ONE, ALL}
+
/** Are there any (in-service influencing) clusters in this container? */
private boolean containerHasClusters;
@@ -72,12 +74,25 @@ public class ClustersStatus extends AbstractComponent {
setDown((String) clusterIdentifier);
}
- /** Returns whether this container should receive traffic based on the state of this */
+ @Deprecated // TODO: Remove on Vespa 8
public boolean containerShouldReceiveTraffic() {
+ return containerShouldReceiveTraffic(Require.ONE);
+ }
+ /**
+ * Returns whether this container should receive traffic based on the state of this
+ * @param require Requirement for being up, ALL or ONE.
+ */
+ public boolean containerShouldReceiveTraffic(Require require) {
synchronized (mutex) {
if (containerHasClusters) {
- // Should receive traffic when at least one cluster is up
- return clusterStatus.values().stream().anyMatch(status -> status==true);
+ switch (require) {
+ case ONE:
+ // Should receive traffic when at least one cluster is up
+ return clusterStatus.values().stream().anyMatch(status -> status == true);
+ case ALL:
+ default:
+ return !clusterStatus.isEmpty() && clusterStatus.values().stream().allMatch(status -> status == true);
+ }
}
else {
return true;
diff --git a/container-core/src/main/java/com/yahoo/container/handler/VipStatus.java b/container-core/src/main/java/com/yahoo/container/handler/VipStatus.java
index b9ef1627ce7..f712690efc5 100644
--- a/container-core/src/main/java/com/yahoo/container/handler/VipStatus.java
+++ b/container-core/src/main/java/com/yahoo/container/handler/VipStatus.java
@@ -102,10 +102,15 @@ public class VipStatus {
private void updateCurrentlyInRotation() {
synchronized (mutex) {
- if (rotationOverride != null)
+ if (rotationOverride != null) {
currentlyInRotation = rotationOverride;
- else
- currentlyInRotation = clustersStatus.containerShouldReceiveTraffic();
+ } else {
+ if (healthState.status() == StateMonitor.Status.up) {
+ currentlyInRotation = clustersStatus.containerShouldReceiveTraffic(ClustersStatus.Require.ONE);
+ } else {
+ currentlyInRotation = clustersStatus.containerShouldReceiveTraffic(ClustersStatus.Require.ALL);
+ }
+ }
// Change to/from 'up' when appropriate but don't change 'initializing' to 'down'
if (currentlyInRotation)
diff --git a/container-core/src/main/java/com/yahoo/container/jdisc/state/StateMonitor.java b/container-core/src/main/java/com/yahoo/container/jdisc/state/StateMonitor.java
index f690c240537..faa08402cdc 100644
--- a/container-core/src/main/java/com/yahoo/container/jdisc/state/StateMonitor.java
+++ b/container-core/src/main/java/com/yahoo/container/jdisc/state/StateMonitor.java
@@ -14,6 +14,7 @@ import java.util.TreeSet;
import java.util.concurrent.CopyOnWriteArrayList;
import java.util.concurrent.ThreadFactory;
import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
import java.util.logging.Logger;
/**
@@ -27,7 +28,7 @@ public class StateMonitor extends AbstractComponent {
private final static Logger log = Logger.getLogger(StateMonitor.class.getName());
- public enum Status {up, down, initializing};
+ public enum Status {up, down, initializing}
private final CopyOnWriteArrayList<StateMetricConsumer> consumers = new CopyOnWriteArrayList<>();
private final Thread thread;
@@ -37,6 +38,7 @@ public class StateMonitor extends AbstractComponent {
private volatile MetricSnapshot snapshot;
private volatile Status status;
private final TreeSet<String> valueNames = new TreeSet<>();
+ private final AtomicBoolean stopped = new AtomicBoolean(false);
/** For testing */
public StateMonitor() {
@@ -53,10 +55,16 @@ public class StateMonitor extends AbstractComponent {
}
StateMonitor(HealthMonitorConfig config, Timer timer, ThreadFactory threadFactory) {
+ this((long)(config.snapshot_interval() * TimeUnit.SECONDS.toMillis(1)),
+ Status.valueOf(config.initialStatus()),
+ timer, threadFactory);
+ }
+ /* For Testing */
+ public StateMonitor(long snapshotIntervalMS, Status status, Timer timer, ThreadFactory threadFactory) {
this.timer = timer;
- this.snapshotIntervalMs = (long)(config.snapshot_interval() * TimeUnit.SECONDS.toMillis(1));
+ this.snapshotIntervalMs = snapshotIntervalMS;
this.lastSnapshotTimeMs = timer.currentTimeMillis();
- this.status = Status.valueOf(config.initialStatus());
+ this.status = status;
thread = threadFactory.newThread(this::run);
thread.start();
}
@@ -99,13 +107,13 @@ public class StateMonitor extends AbstractComponent {
private void run() {
log.finest("StateMonitor started.");
try {
- while (!Thread.interrupted()) {
- checkTime();
- Thread.sleep((lastSnapshotTimeMs + snapshotIntervalMs) - timer.currentTimeMillis());
+ synchronized (stopped) {
+ while (!stopped.get()) {
+ checkTime();
+ stopped.wait((lastSnapshotTimeMs + snapshotIntervalMs) - timer.currentTimeMillis());
+ }
}
- } catch (InterruptedException e) {
-
- }
+ } catch (InterruptedException e) { }
log.finest("StateMonitor stopped.");
}
@@ -137,12 +145,13 @@ public class StateMonitor extends AbstractComponent {
@Override
public void deconstruct() {
- thread.interrupt();
+ synchronized (stopped) {
+ stopped.set(true);
+ stopped.notifyAll();
+ }
try {
thread.join(5000);
- } catch (InterruptedException e) {
- Thread.currentThread().interrupt();
- }
+ } catch (InterruptedException e) { }
if (thread.isAlive()) {
log.warning("StateMonitor failed to terminate within 5 seconds of interrupt signal. Ignoring.");
}
diff --git a/container-core/src/main/resources/configdefinitions/health-monitor.def b/container-core/src/main/resources/configdefinitions/health-monitor.def
index 5e70c72ae3f..4e91d85b2b8 100644
--- a/container-core/src/main/resources/configdefinitions/health-monitor.def
+++ b/container-core/src/main/resources/configdefinitions/health-monitor.def
@@ -6,4 +6,4 @@ namespace=container.jdisc.config
snapshot_interval double default=300
# Initial status used in /state/v1/health API (value for 'code' in 'status'). See StateMonitor for valid values
-initialStatus string default="up"
+initialStatus string default="initializing"
diff --git a/container-core/src/test/java/com/yahoo/container/handler/VipStatusTestCase.java b/container-core/src/test/java/com/yahoo/container/handler/VipStatusTestCase.java
index 52679c15957..d3479936544 100644
--- a/container-core/src/test/java/com/yahoo/container/handler/VipStatusTestCase.java
+++ b/container-core/src/test/java/com/yahoo/container/handler/VipStatusTestCase.java
@@ -4,6 +4,8 @@ package com.yahoo.container.handler;
import static org.junit.Assert.*;
import com.yahoo.container.QrSearchersConfig;
+import com.yahoo.container.jdisc.state.StateMonitor;
+import com.yahoo.jdisc.core.SystemTimer;
import org.junit.Test;
/**
@@ -12,45 +14,79 @@ import org.junit.Test;
* @author steinar
*/
public class VipStatusTestCase {
+ private static final String [] clusters = {"cluster1", "cluster2", "cluster3"};
- @Test
- public void testVipStatusWorksWithClusters() {
+ private static QrSearchersConfig getSearchersCfg() {
var b = new QrSearchersConfig.Builder();
var searchClusterB = new QrSearchersConfig.Searchcluster.Builder();
- searchClusterB.name("cluster1");
- searchClusterB.name("cluster2");
- searchClusterB.name("cluster3");
+ for (String cluster : clusters) {
+ searchClusterB.name(cluster);
+ }
b.searchcluster(searchClusterB);
- VipStatus v = new VipStatus(b.build());
+ return b.build();
+ }
+ private static VipStatus getVipStatus(StateMonitor.Status startState) {
+ return new VipStatus(getSearchersCfg(), new ClustersStatus(), new StateMonitor(1000, startState, new SystemTimer(), runnable -> {
+ Thread thread = new Thread(runnable, "StateMonitor");
+ thread.setDaemon(true);
+ return thread;
+ }));
+ }
- String cluster1 = "cluster1";
- String cluster2 = "cluster2";
- String cluster3 = "cluster3";
+ private static void removeAll(VipStatus v) {
+ for (String s : clusters) {
+ v.removeFromRotation(s);
+ }
+ }
+ private static void addAll(VipStatus v) {
+ for (String s : clusters) {
+ v.addToRotation(s);
+ }
+ }
+ private static void verifyUpOrDown(StateMonitor.Status status) {
+ VipStatus v = getVipStatus(status);
+ removeAll(v);
// initial state
assertFalse(v.isInRotation());
-
- // one cluster becomes up
- v.addToRotation(cluster1);
+ v.addToRotation(clusters[0]);
+ assertFalse(v.isInRotation());
+ v.addToRotation(clusters[1]);
+ assertFalse(v.isInRotation());
+ v.addToRotation(clusters[2]);
assertTrue(v.isInRotation());
+ }
- // all clusters down
- v.removeFromRotation(cluster1);
- v.removeFromRotation(cluster2);
- v.removeFromRotation(cluster3);
+ @Test
+ public void testInitializingOrDownRequireAllUp() {
+ verifyUpOrDown(StateMonitor.Status.initializing);
+ verifyUpOrDown(StateMonitor.Status.down);
+ }
+
+ @Test
+ public void testUpRequireAllDown() {
+ VipStatus v = getVipStatus(StateMonitor.Status.initializing);
assertFalse(v.isInRotation());
- // some clusters down
- v.addToRotation(cluster2);
+ addAll(v);
assertTrue(v.isInRotation());
- // all clusters up
- v.addToRotation(cluster1);
- v.addToRotation(cluster3);
+
+ v.removeFromRotation(clusters[0]);
+ assertTrue(v.isInRotation());
+ v.removeFromRotation(clusters[1]);
assertTrue(v.isInRotation());
- // and down again
- v.removeFromRotation(cluster1);
- v.removeFromRotation(cluster2);
- v.removeFromRotation(cluster3);
+ v.removeFromRotation(clusters[2]);
+ assertFalse(v.isInRotation()); // All down
+ v.addToRotation(clusters[1]);
assertFalse(v.isInRotation());
+ v.addToRotation(clusters[0]);
+ v.addToRotation(clusters[2]);
+ assertTrue(v.isInRotation()); // All up
+ v.removeFromRotation(clusters[0]);
+ v.removeFromRotation(clusters[2]);
+ assertTrue(v.isInRotation());
+ v.addToRotation(clusters[0]);
+ v.addToRotation(clusters[2]);
+ assertTrue(v.isInRotation());
}
}
diff --git a/container-core/src/test/java/com/yahoo/container/jdisc/state/StateHandlerTestBase.java b/container-core/src/test/java/com/yahoo/container/jdisc/state/StateHandlerTestBase.java
index 78541137db5..8a1640e2c0e 100644
--- a/container-core/src/test/java/com/yahoo/container/jdisc/state/StateHandlerTestBase.java
+++ b/container-core/src/test/java/com/yahoo/container/jdisc/state/StateHandlerTestBase.java
@@ -57,7 +57,8 @@ public class StateHandlerTestBase {
HealthMonitorConfig healthMonitorConfig =
new HealthMonitorConfig(
new HealthMonitorConfig.Builder()
- .snapshot_interval(TimeUnit.MILLISECONDS.toSeconds(SNAPSHOT_INTERVAL)));
+ .snapshot_interval(TimeUnit.MILLISECONDS.toSeconds(SNAPSHOT_INTERVAL))
+ .initialStatus("up"));
ThreadFactory threadFactory = ignored -> mock(Thread.class);
this.monitor = new StateMonitor(healthMonitorConfig, timer, threadFactory);
builder.guiceModules().install(new AbstractModule() {
diff --git a/container-disc/abi-spec.json b/container-disc/abi-spec.json
index 35280e12146..29ca56fa9bc 100644
--- a/container-disc/abi-spec.json
+++ b/container-disc/abi-spec.json
@@ -14,6 +14,8 @@
"public abstract javax.net.ssl.SSLContext getRoleSslContext(java.lang.String, java.lang.String)",
"public abstract java.lang.String getRoleToken(java.lang.String)",
"public abstract java.lang.String getRoleToken(java.lang.String, java.lang.String)",
+ "public abstract java.lang.String getAccessToken(java.lang.String)",
+ "public abstract java.lang.String getAccessToken(java.lang.String, java.util.List)",
"public abstract java.util.List getIdentityCertificate()",
"public abstract java.security.PrivateKey getPrivateKey()"
],
diff --git a/container-disc/src/main/java/com/yahoo/container/jdisc/AthenzIdentityProviderProvider.java b/container-disc/src/main/java/com/yahoo/container/jdisc/AthenzIdentityProviderProvider.java
index 9045db3eda5..bb862aeca82 100644
--- a/container-disc/src/main/java/com/yahoo/container/jdisc/AthenzIdentityProviderProvider.java
+++ b/container-disc/src/main/java/com/yahoo/container/jdisc/AthenzIdentityProviderProvider.java
@@ -59,6 +59,16 @@ public class AthenzIdentityProviderProvider implements Provider<AthenzIdentityPr
}
@Override
+ public String getAccessToken(String domain) {
+ throw new UnsupportedOperationException(message);
+ }
+
+ @Override
+ public String getAccessToken(String domain, List<String> roles) {
+ throw new UnsupportedOperationException(message);
+ }
+
+ @Override
public List<X509Certificate> getIdentityCertificate() {
throw new UnsupportedOperationException(message);
}
diff --git a/container-disc/src/main/java/com/yahoo/container/jdisc/athenz/AthenzIdentityProvider.java b/container-disc/src/main/java/com/yahoo/container/jdisc/athenz/AthenzIdentityProvider.java
index 1db3f4a3b42..696aab85b0c 100644
--- a/container-disc/src/main/java/com/yahoo/container/jdisc/athenz/AthenzIdentityProvider.java
+++ b/container-disc/src/main/java/com/yahoo/container/jdisc/athenz/AthenzIdentityProvider.java
@@ -16,6 +16,8 @@ public interface AthenzIdentityProvider {
SSLContext getRoleSslContext(String domain, String role);
String getRoleToken(String domain);
String getRoleToken(String domain, String role);
+ String getAccessToken(String domain);
+ String getAccessToken(String domain, List<String> roles);
List<X509Certificate> getIdentityCertificate();
PrivateKey getPrivateKey();
}
diff --git a/container-search/abi-spec.json b/container-search/abi-spec.json
index b5fbe235c43..437db99c45b 100644
--- a/container-search/abi-spec.json
+++ b/container-search/abi-spec.json
@@ -1954,6 +1954,7 @@
"methods": [
"public void <init>(com.yahoo.search.cluster.NodeManager)",
"public void <init>(com.yahoo.search.cluster.NodeManager, boolean)",
+ "public void start()",
"public com.yahoo.search.cluster.MonitorConfiguration getConfiguration()",
"public void add(java.lang.Object, boolean)",
"public com.yahoo.search.cluster.BaseNodeMonitor getNodeMonitor(java.lang.Object)",
@@ -1978,8 +1979,8 @@
"methods": [
"public void <init>(com.yahoo.component.ComponentId, java.util.List, boolean)",
"public void <init>(com.yahoo.component.ComponentId, java.util.List, com.yahoo.search.cluster.Hasher, boolean)",
- "public void <init>(com.yahoo.component.ComponentId, java.util.List, com.yahoo.search.cluster.Hasher, boolean, boolean)",
- "public final void ping(java.lang.Object, java.util.concurrent.Executor)",
+ "protected void <init>(com.yahoo.component.ComponentId, java.util.List, com.yahoo.search.cluster.Hasher, boolean, boolean)",
+ "public final void ping(com.yahoo.search.cluster.ClusterMonitor, java.lang.Object, java.util.concurrent.Executor)",
"protected abstract com.yahoo.prelude.Pong ping(com.yahoo.prelude.Ping, java.lang.Object)",
"protected java.lang.Object getFirstConnection(com.yahoo.search.cluster.Hasher$NodeList, int, int, com.yahoo.search.Query)",
"public final com.yahoo.search.Result search(com.yahoo.search.Query, com.yahoo.search.searchchain.Execution)",
@@ -2074,7 +2075,8 @@
"methods": [
"public abstract void working(java.lang.Object)",
"public abstract void failed(java.lang.Object)",
- "public abstract void ping(java.lang.Object, java.util.concurrent.Executor)",
+ "public void ping(java.lang.Object, java.util.concurrent.Executor)",
+ "public void ping(com.yahoo.search.cluster.ClusterMonitor, java.lang.Object, java.util.concurrent.Executor)",
"public void pingIterationCompleted()"
],
"fields": []
diff --git a/container-search/src/main/java/com/yahoo/prelude/Index.java b/container-search/src/main/java/com/yahoo/prelude/Index.java
index 65d5879b004..365ee299ca4 100644
--- a/container-search/src/main/java/com/yahoo/prelude/Index.java
+++ b/container-search/src/main/java/com/yahoo/prelude/Index.java
@@ -26,6 +26,7 @@ import java.util.Set;
public class Index {
public static class Attribute {
+
private boolean tokenizedContent = false;
public final String name;
@@ -207,20 +208,12 @@ public class Index {
}
}
- /**
- * Whether terms in this field are lower cased when indexing.
- *
- * @param lowercase true if terms are lowercased
- */
+ /** Sets whether terms in this field are lowercased when indexing. */
public void setLowercase(boolean lowercase) {
this.lowercase = lowercase;
}
- /**
- * Whether terms in this field are lower cased when indexing.
- *
- * @return true if terms are lowercased
- */
+ /** Returns whether terms in this field are lowercased when indexing. */
public boolean isLowercase() {
return lowercase;
}
diff --git a/container-search/src/main/java/com/yahoo/prelude/query/BlockItem.java b/container-search/src/main/java/com/yahoo/prelude/query/BlockItem.java
index 13673144a0a..d0ffcd2d0e0 100644
--- a/container-search/src/main/java/com/yahoo/prelude/query/BlockItem.java
+++ b/container-search/src/main/java/com/yahoo/prelude/query/BlockItem.java
@@ -3,10 +3,9 @@ package com.yahoo.prelude.query;
/**
- * An interface used for anything which represents a single block
- * of query input.
+ * An interface used for anything which represents a single block of query input.
*
- * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a>
+ * @author Steinar Knutsen
*/
public interface BlockItem extends HasIndexItem {
@@ -39,4 +38,5 @@ public interface BlockItem extends HasIndexItem {
* is necessary to change operator?
*/
SegmentingRule getSegmentingRule();
+
}
diff --git a/container-search/src/main/java/com/yahoo/prelude/querytransform/NormalizingSearcher.java b/container-search/src/main/java/com/yahoo/prelude/querytransform/NormalizingSearcher.java
index fdd6ad47a98..ce13045b518 100644
--- a/container-search/src/main/java/com/yahoo/prelude/querytransform/NormalizingSearcher.java
+++ b/container-search/src/main/java/com/yahoo/prelude/querytransform/NormalizingSearcher.java
@@ -111,25 +111,17 @@ public class NormalizingSearcher extends Searcher {
}
private void normalizeAlternatives(Language language, Session indexFacts, WordAlternativesItem block) {
- if (!block.isNormalizable()) {
- return;
- }
- {
- Index index = indexFacts.getIndex(block.getIndexName());
- if (index.isAttribute()) {
- return;
- }
- if (!index.getNormalize()) {
- return;
- }
- }
+ if ( ! block.isNormalizable()) return;
+
+ Index index = indexFacts.getIndex(block.getIndexName());
+ if (index.isAttribute()) return;
+ if ( ! index.getNormalize()) return;
List<Alternative> terms = block.getAlternatives();
for (Alternative term : terms) {
String accentDropped = linguistics.getTransformer().accentDrop(term.word, language);
- if (!term.word.equals(accentDropped) && accentDropped.length() > 0) {
+ if ( ! term.word.equals(accentDropped) && accentDropped.length() > 0)
block.addTerm(accentDropped, term.exactness * .7d);
- }
}
}
diff --git a/container-search/src/main/java/com/yahoo/prelude/querytransform/QueryRewrite.java b/container-search/src/main/java/com/yahoo/prelude/querytransform/QueryRewrite.java
index 84c793a6df1..5a936d42ccc 100644
--- a/container-search/src/main/java/com/yahoo/prelude/querytransform/QueryRewrite.java
+++ b/container-search/src/main/java/com/yahoo/prelude/querytransform/QueryRewrite.java
@@ -4,6 +4,8 @@ package com.yahoo.prelude.querytransform;
import com.yahoo.prelude.query.AndItem;
import com.yahoo.prelude.query.CompositeItem;
import com.yahoo.prelude.query.EquivItem;
+import com.yahoo.prelude.query.HasIndexItem;
+import com.yahoo.prelude.query.IndexedItem;
import com.yahoo.prelude.query.Item;
import com.yahoo.prelude.query.NearItem;
import com.yahoo.prelude.query.NotItem;
@@ -169,7 +171,9 @@ public class QueryRewrite {
removeOtherNonrankedChildren(item, i);
recall = Recall.RECALLS_EVERYTHING;
} else if ((item instanceof AndItem) || (item instanceof NearItem)) {
- item.removeItem(i);
+ if ( ! isRanked(item.getItem(i))) {
+ item.removeItem(i);
+ }
} else if (item instanceof RankItem) {
// empty
} else {
@@ -200,6 +204,20 @@ public class QueryRewrite {
parent.removeItem(i);
}
}
+
+ private static boolean isRanked(Item item) {
+ if (item instanceof CompositeItem) {
+ for (Item child : ((CompositeItem)item).items())
+ if (isRanked(child)) return true;
+ return false;
+ }
+ else if (item instanceof HasIndexItem && Hit.SDDOCNAME_FIELD.equals(((HasIndexItem)item).getIndexName())) {
+ return false; // No point in ranking by sddocname
+ }
+ else {
+ return item.isRanked();
+ }
+ }
private static Item collapseSingleComposites(Item item) {
if (!(item instanceof CompositeItem)) {
diff --git a/container-search/src/main/java/com/yahoo/prelude/querytransform/StemmingSearcher.java b/container-search/src/main/java/com/yahoo/prelude/querytransform/StemmingSearcher.java
index 655fbf6acc3..9a9044def2d 100644
--- a/container-search/src/main/java/com/yahoo/prelude/querytransform/StemmingSearcher.java
+++ b/container-search/src/main/java/com/yahoo/prelude/querytransform/StemmingSearcher.java
@@ -188,13 +188,10 @@ public class StemmingSearcher extends Searcher {
return (Item) w;
}
- if (context.isCJK) {
- composite = chooseCompositeForCJK(current,
- ((Item) current).getParent(),
- indexName);
- } else {
- composite = phraseSegment(current, indexName);
- }
+ if (context.isCJK)
+ composite = chooseCompositeForCJK(current, ((Item) current).getParent(), indexName);
+ else
+ composite = chooseComposite(current, ((Item) current).getParent(), indexName);
for (StemList segment : segments) {
TaggableItem w = singleWordSegment(current, segment, index, substring, context.insidePhrase);
@@ -331,39 +328,34 @@ public class StemmingSearcher extends Searcher {
}
}
+ private CompositeItem chooseComposite(BlockItem current, CompositeItem parent, String indexName) {
+ if (parent instanceof PhraseItem || current instanceof PhraseSegmentItem)
+ return createPhraseSegment(current, indexName);
+ else
+ return createAndSegment(current);
+
+ }
+
private CompositeItem chooseCompositeForCJK(BlockItem current, CompositeItem parent, String indexName) {
- CompositeItem composite;
- if (current.getSegmentingRule() == SegmentingRule.LANGUAGE_DEFAULT) {
- if (parent instanceof PhraseItem || current instanceof PhraseSegmentItem) {
- composite = phraseSegment(current, indexName);
- } else
- composite = createAndSegment(current);
- } else {
- switch (current.getSegmentingRule()) {
- case PHRASE:
- composite = phraseSegment(current, indexName);
- break;
- case BOOLEAN_AND:
- composite = createAndSegment(current);
- break;
+ if (current.getSegmentingRule() == SegmentingRule.LANGUAGE_DEFAULT)
+ return chooseComposite(current, parent, indexName);
+
+ switch (current.getSegmentingRule()) { // TODO: Why for CJK only? The segmentingRule says nothing about being for CJK only
+ case PHRASE: return createPhraseSegment(current, indexName);
+ case BOOLEAN_AND: return createAndSegment(current);
default:
- throw new IllegalArgumentException(
- "Unknown segmenting rule: "
- + current.getSegmentingRule()
- + ". This is a bug in Vespa, as the implementation has gotten out of sync."
- + " Please create a ticket as soon as possible.");
- }
+ throw new IllegalArgumentException("Unknown segmenting rule: " + current.getSegmentingRule() +
+ ". This is a bug in Vespa, as the implementation has gotten out of sync." +
+ " Please create a ticket as soon as possible.");
}
- return composite;
}
private AndSegmentItem createAndSegment(BlockItem current) {
return new AndSegmentItem(current.stringValue(), true, true);
}
- private CompositeItem phraseSegment(BlockItem current, String indexName) {
- CompositeItem composite;
- composite = new PhraseSegmentItem(current.getRawWord(), current.stringValue(), true, true);
+ private CompositeItem createPhraseSegment(BlockItem current, String indexName) {
+ CompositeItem composite = new PhraseSegmentItem(current.getRawWord(), current.stringValue(), true, true);
composite.setIndexName(indexName);
return composite;
}
diff --git a/container-search/src/main/java/com/yahoo/search/Query.java b/container-search/src/main/java/com/yahoo/search/Query.java
index 395d8853603..1e3f11f4f78 100644
--- a/container-search/src/main/java/com/yahoo/search/Query.java
+++ b/container-search/src/main/java/com/yahoo/search/Query.java
@@ -288,7 +288,6 @@ public class Query extends com.yahoo.processing.Request implements Cloneable {
this("");
}
-
/**
* Construct a query from a string formatted in the http style, e.g <code>?query=test&amp;offset=10&amp;hits=13</code>
* The query must be uri encoded.
@@ -297,7 +296,6 @@ public class Query extends com.yahoo.processing.Request implements Cloneable {
this(query, null);
}
-
/**
* Creates a query from a request
*
diff --git a/container-search/src/main/java/com/yahoo/search/cluster/ClusterMonitor.java b/container-search/src/main/java/com/yahoo/search/cluster/ClusterMonitor.java
index d4b6279be89..55f0816514d 100644
--- a/container-search/src/main/java/com/yahoo/search/cluster/ClusterMonitor.java
+++ b/container-search/src/main/java/com/yahoo/search/cluster/ClusterMonitor.java
@@ -38,6 +38,9 @@ public class ClusterMonitor<T> {
/** A map from Node to corresponding MonitoredNode */
private final Map<T, TrafficNodeMonitor<T>> nodeMonitors = Collections.synchronizedMap(new java.util.LinkedHashMap<>());
+ /** @deprecated It is not advised to start the monitoring thread in the constructor.
+ * Use ClusterMonitor(NodeManager manager, false) and explicit start(). */
+ @Deprecated
public ClusterMonitor(NodeManager<T> manager) {
this(manager, true);
}
@@ -50,6 +53,12 @@ public class ClusterMonitor<T> {
}
}
+ public void start() {
+ if ( ! monitorThread.isAlive()) {
+ monitorThread.start();
+ }
+ }
+
/** Returns the configuration of this cluster monitor */
public MonitorConfiguration getConfiguration() { return configuration; }
@@ -101,7 +110,7 @@ public class ClusterMonitor<T> {
public void ping(Executor executor) {
for (Iterator<BaseNodeMonitor<T>> i = nodeMonitorIterator(); i.hasNext() && !closed.get(); ) {
BaseNodeMonitor<T> monitor= i.next();
- nodeManager.ping(monitor.getNode(), executor); // Cause call to failed or responded
+ nodeManager.ping(this, monitor.getNode(), executor); // Cause call to failed or responded
}
if (closed.get()) return; // Do nothing to change state if close has started.
nodeManager.pingIterationCompleted();
diff --git a/container-search/src/main/java/com/yahoo/search/cluster/ClusterSearcher.java b/container-search/src/main/java/com/yahoo/search/cluster/ClusterSearcher.java
index 20f56c86f7b..2d05168731a 100644
--- a/container-search/src/main/java/com/yahoo/search/cluster/ClusterSearcher.java
+++ b/container-search/src/main/java/com/yahoo/search/cluster/ClusterSearcher.java
@@ -58,7 +58,7 @@ public abstract class ClusterSearcher<T> extends PingableSearcher implements Nod
this(id, connections, hasher, internal, true);
}
- public ClusterSearcher(ComponentId id, List<T> connections, Hasher<T> hasher, boolean internal, boolean startPingThread) {
+ protected ClusterSearcher(ComponentId id, List<T> connections, Hasher<T> hasher, boolean internal, boolean startPingThread) {
super(id);
this.hasher = hasher;
this.monitor = new ClusterMonitor<>(this, startPingThread);
@@ -70,7 +70,7 @@ public abstract class ClusterSearcher<T> extends PingableSearcher implements Nod
/** Pinging a node, called from ClusterMonitor */
@Override
- public final void ping(T p, Executor executor) {
+ public final void ping(ClusterMonitor<T> clusterMonitor, T p, Executor executor) {
log(LogLevel.FINE, "Sending ping to: ", p);
Pinger pinger = new Pinger(p);
FutureTask<Pong> future = new FutureTask<>(pinger);
@@ -80,7 +80,7 @@ public abstract class ClusterSearcher<T> extends PingableSearcher implements Nod
Throwable logThrowable = null;
try {
- pong = future.get(monitor.getConfiguration().getFailLimit(), TimeUnit.MILLISECONDS);
+ pong = future.get(clusterMonitor.getConfiguration().getFailLimit(), TimeUnit.MILLISECONDS);
} catch (InterruptedException e) {
pong = new Pong(ErrorMessage.createUnspecifiedError("Ping was interrupted: " + p));
logThrowable = e;
@@ -96,10 +96,10 @@ public abstract class ClusterSearcher<T> extends PingableSearcher implements Nod
future.cancel(true);
if (pong.badResponse()) {
- monitor.failed(p, pong.error().get());
+ clusterMonitor.failed(p, pong.error().get());
log(LogLevel.FINE, "Failed ping - ", pong);
} else {
- monitor.responded(p);
+ clusterMonitor.responded(p);
log(LogLevel.FINE, "Answered ping - ", p);
}
diff --git a/container-search/src/main/java/com/yahoo/search/cluster/MonitorConfiguration.java b/container-search/src/main/java/com/yahoo/search/cluster/MonitorConfiguration.java
index 226e0180d2e..7b10992dff8 100644
--- a/container-search/src/main/java/com/yahoo/search/cluster/MonitorConfiguration.java
+++ b/container-search/src/main/java/com/yahoo/search/cluster/MonitorConfiguration.java
@@ -30,7 +30,7 @@ public class MonitorConfiguration {
* The number of milliseconds to attempt to complete a request
* before giving up
*/
- private long requestTimeout = 5000;
+ private final long requestTimeout = 5000;
/**
* The number of milliseconds a node is allowed to fail before we
diff --git a/container-search/src/main/java/com/yahoo/search/cluster/NodeManager.java b/container-search/src/main/java/com/yahoo/search/cluster/NodeManager.java
index 9b20139e3c5..481f1e1b5a5 100644
--- a/container-search/src/main/java/com/yahoo/search/cluster/NodeManager.java
+++ b/container-search/src/main/java/com/yahoo/search/cluster/NodeManager.java
@@ -19,9 +19,21 @@ public interface NodeManager<T> {
/**
* Called when a node should be pinged.
- * This *must* lead to either a call to NodeMonitor.failed or NodeMonitor.responded
+ * This *must* lead to either a call to NodeMonitor.failed or NodeMonitor.responded
+ * @deprecated Use ping(ClusterMonitor clusterMonitor, T node, Executor executor) instead.
*/
- void ping(T node, Executor executor);
+ @Deprecated
+ default void ping(T node, Executor executor) {
+ throw new IllegalStateException("If you have not overrriden ping(ClusterMonitor<T> clusterMonitor, T node, Executor executor), you should at least have overriden this method.");
+ }
+
+ /**
+ * Called when a node should be pinged.
+ * This *must* lead to either a call to ClusterMonitor.failed or ClusterMonitor.responded
+ */
+ default void ping(ClusterMonitor<T> clusterMonitor, T node, Executor executor) {
+ ping(node, executor);
+ }
/** Called right after a ping has been issued to each node. This default implementation does nothing. */
default void pingIterationCompleted() {}
diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java b/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java
index 03b51fbaf70..91bd5c6da11 100644
--- a/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java
+++ b/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java
@@ -11,8 +11,10 @@ import com.yahoo.prelude.fastsearch.VespaBackEndSearcher;
import com.yahoo.processing.request.CompoundName;
import com.yahoo.search.Query;
import com.yahoo.search.Result;
+import com.yahoo.search.cluster.ClusterMonitor;
import com.yahoo.search.dispatch.SearchPath.InvalidSearchPathException;
import com.yahoo.search.dispatch.rpc.RpcInvokerFactory;
+import com.yahoo.search.dispatch.rpc.RpcPingFactory;
import com.yahoo.search.dispatch.rpc.RpcResourcePool;
import com.yahoo.search.dispatch.searchcluster.Group;
import com.yahoo.search.dispatch.searchcluster.Node;
@@ -58,6 +60,7 @@ public class Dispatcher extends AbstractComponent {
/** A model of the search cluster this dispatches to */
private final SearchCluster searchCluster;
+ private final ClusterMonitor clusterMonitor;
private final LoadBalancer loadBalancer;
@@ -87,44 +90,48 @@ public class Dispatcher extends AbstractComponent {
ClusterInfoConfig clusterInfoConfig,
VipStatus vipStatus,
Metric metric) {
- this(new SearchCluster(clusterId.stringValue(), dispatchConfig, clusterInfoConfig.nodeCount(), vipStatus),
- dispatchConfig,
- metric);
+ this(new RpcResourcePool(dispatchConfig), clusterId, dispatchConfig, clusterInfoConfig, vipStatus, metric);
}
- private Dispatcher(SearchCluster searchCluster, DispatchConfig dispatchConfig, Metric metric) {
- this(searchCluster,
- dispatchConfig,
- new RpcInvokerFactory(new RpcResourcePool(dispatchConfig), searchCluster),
- metric);
+ private Dispatcher(RpcResourcePool resourcePool,
+ ComponentId clusterId,
+ DispatchConfig dispatchConfig,
+ ClusterInfoConfig clusterInfoConfig,
+ VipStatus vipStatus,
+ Metric metric) {
+ this(resourcePool, new SearchCluster(clusterId.stringValue(), dispatchConfig, clusterInfoConfig.nodeCount(),
+ vipStatus, new RpcPingFactory(resourcePool)),
+ dispatchConfig, metric);
+
}
- /* Protected for simple mocking in tests. Beware that searchCluster is shutdown on in deconstruct() */
- protected Dispatcher(SearchCluster searchCluster,
- DispatchConfig dispatchConfig,
- RpcInvokerFactory rcpInvokerFactory,
- Metric metric) {
- this(searchCluster, dispatchConfig, rcpInvokerFactory, rcpInvokerFactory, metric);
+ private Dispatcher(RpcResourcePool resourcePool, SearchCluster searchCluster, DispatchConfig dispatchConfig, Metric metric) {
+ this(new ClusterMonitor<>(searchCluster, true), searchCluster, dispatchConfig, new RpcInvokerFactory(resourcePool, searchCluster), metric);
}
/* Protected for simple mocking in tests. Beware that searchCluster is shutdown on in deconstruct() */
- protected Dispatcher(SearchCluster searchCluster,
+ protected Dispatcher(ClusterMonitor clusterMonitor,
+ SearchCluster searchCluster,
DispatchConfig dispatchConfig,
InvokerFactory invokerFactory,
- PingFactory pingFactory,
Metric metric) {
if (dispatchConfig.useMultilevelDispatch())
throw new IllegalArgumentException(searchCluster + " is configured with multilevel dispatch, but this is not supported");
this.searchCluster = searchCluster;
+ this.clusterMonitor = clusterMonitor;
this.loadBalancer = new LoadBalancer(searchCluster,
dispatchConfig.distributionPolicy() == DispatchConfig.DistributionPolicy.ROUNDROBIN);
this.invokerFactory = invokerFactory;
this.metric = metric;
this.metricContext = metric.createContext(null);
this.maxHitsPerNode = dispatchConfig.maxHitsPerNode();
-
- searchCluster.startClusterMonitoring(pingFactory);
+ searchCluster.addMonitoring(clusterMonitor);
+ try {
+ while ( ! searchCluster.hasInformationAboutAllNodes()) {
+ Thread.sleep(1);
+ }
+ } catch (InterruptedException e) {}
}
/** Returns the search cluster this dispatches to */
@@ -134,8 +141,8 @@ public class Dispatcher extends AbstractComponent {
@Override
public void deconstruct() {
- /* The seach cluster must be shutdown first as it uses the invokerfactory. */
- searchCluster.shutDown();
+ /* The clustermonitor must be shutdown first as it uses the invokerfactory through the searchCluster. */
+ clusterMonitor.shutdown();
invokerFactory.release();
}
diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcInvokerFactory.java b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcInvokerFactory.java
index 5c9928de924..a45ec59c3ee 100644
--- a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcInvokerFactory.java
+++ b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcInvokerFactory.java
@@ -14,6 +14,8 @@ import com.yahoo.search.dispatch.InvokerFactory;
import com.yahoo.search.dispatch.SearchInvoker;
import com.yahoo.search.dispatch.searchcluster.Node;
import com.yahoo.search.dispatch.searchcluster.PingFactory;
+import com.yahoo.search.dispatch.searchcluster.Pinger;
+import com.yahoo.search.dispatch.searchcluster.PongHandler;
import com.yahoo.search.dispatch.searchcluster.SearchCluster;
import java.util.Optional;
@@ -22,7 +24,7 @@ import java.util.concurrent.Callable;
/**
* @author ollivir
*/
-public class RpcInvokerFactory extends InvokerFactory implements PingFactory {
+public class RpcInvokerFactory extends InvokerFactory {
/** Unless turned off this will fill summaries by dispatching directly to search nodes over RPC when possible */
private final static CompoundName dispatchSummaries = new CompoundName("dispatch.summaries");
@@ -63,9 +65,4 @@ public class RpcInvokerFactory extends InvokerFactory implements PingFactory {
public void release() {
rpcResourcePool.release();
}
-
- @Override
- public Callable<Pong> createPinger(Node node, ClusterMonitor<Node> monitor) {
- return new RpcPing(node, monitor, rpcResourcePool);
- }
}
diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcPing.java b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcPing.java
index e0f1dc5e675..ba3b050149c 100644
--- a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcPing.java
+++ b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcPing.java
@@ -10,55 +10,64 @@ import com.yahoo.search.cluster.ClusterMonitor;
import com.yahoo.search.dispatch.rpc.Client.ProtobufResponse;
import com.yahoo.search.dispatch.rpc.Client.ResponseOrError;
import com.yahoo.search.dispatch.searchcluster.Node;
+import com.yahoo.search.dispatch.searchcluster.Pinger;
+import com.yahoo.search.dispatch.searchcluster.PongHandler;
import com.yahoo.search.result.ErrorMessage;
import com.yahoo.yolean.Exceptions;
-import java.util.concurrent.Callable;
-import java.util.concurrent.LinkedBlockingQueue;
-import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.logging.Logger;
-public class RpcPing implements Callable<Pong> {
+public class RpcPing implements Pinger, Client.ResponseReceiver {
+ private static final Logger log = Logger.getLogger(RpcPing.class.getName());
private static final String RPC_METHOD = "vespa.searchprotocol.ping";
private static final CompressionType PING_COMPRESSION = CompressionType.NONE;
private final Node node;
private final RpcResourcePool resourcePool;
private final ClusterMonitor<Node> clusterMonitor;
+ private final long pingSequenceId;
+ private final PongHandler pongHandler;
- public RpcPing(Node node, ClusterMonitor<Node> clusterMonitor, RpcResourcePool rpcResourcePool) {
+ public RpcPing(Node node, ClusterMonitor<Node> clusterMonitor, RpcResourcePool rpcResourcePool, PongHandler pongHandler) {
this.node = node;
this.resourcePool = rpcResourcePool;
this.clusterMonitor = clusterMonitor;
+ pingSequenceId = node.createPingSequenceId();
+ this.pongHandler = pongHandler;
}
@Override
- public Pong call() throws Exception {
+ public void ping() {
try {
- var queue = new LinkedBlockingQueue<ResponseOrError<ProtobufResponse>>(1);
-
- sendPing(queue);
+ sendPing();
+ } catch (RuntimeException e) {
+ pongHandler.handle(new Pong(ErrorMessage.createBackendCommunicationError("Exception when pinging " + node +
+ ": " + Exceptions.toMessageString(e))));
+ }
+ }
- var responseOrError = queue.poll(clusterMonitor.getConfiguration().getRequestTimeout(), TimeUnit.MILLISECONDS);
- if (responseOrError == null) {
- return new Pong(ErrorMessage.createNoAnswerWhenPingingNode("Timed out waiting for pong from " + node));
- } else if (responseOrError.error().isPresent()) {
- return new Pong(ErrorMessage.createBackendCommunicationError(responseOrError.error().get()));
- }
+ private Pong toPong(ResponseOrError<ProtobufResponse> responseOrError) {
+ if (responseOrError == null) {
+ return new Pong(ErrorMessage.createNoAnswerWhenPingingNode("Timed out waiting for pong from " + node));
+ } else if (responseOrError.error().isPresent()) {
+ return new Pong(ErrorMessage.createBackendCommunicationError(responseOrError.error().get()));
+ }
+ try {
return decodeReply(responseOrError.response().get());
- } catch (RuntimeException e) {
- return new Pong(
- ErrorMessage.createBackendCommunicationError("Exception when pinging " + node + ": " + Exceptions.toMessageString(e)));
+ } catch (InvalidProtocolBufferException e) {
+ return new Pong(ErrorMessage.createBackendCommunicationError(e.getMessage()));
}
}
- private void sendPing(LinkedBlockingQueue<ResponseOrError<ProtobufResponse>> queue) {
+ private void sendPing() {
var connection = resourcePool.getConnection(node.key());
var ping = SearchProtocol.MonitorRequest.newBuilder().build().toByteArray();
double timeoutSeconds = ((double) clusterMonitor.getConfiguration().getRequestTimeout()) / 1000.0;
Compressor.Compression compressionResult = resourcePool.compressor().compress(PING_COMPRESSION, ping);
- connection.request(RPC_METHOD, compressionResult.type(), ping.length, compressionResult.data(), rsp -> queue.add(rsp), timeoutSeconds);
+ connection.request(RPC_METHOD, compressionResult.type(), ping.length, compressionResult.data(),this, timeoutSeconds);
}
private Pong decodeReply(ProtobufResponse response) throws InvalidProtocolBufferException {
@@ -76,4 +85,13 @@ public class RpcPing implements Callable<Pong> {
}
}
+ @Override
+ public void receive(ResponseOrError<ProtobufResponse> response) {
+ if (node.isLastReceivedPong(pingSequenceId)) {
+ pongHandler.handle(toPong(response));
+ } else {
+ //TODO Reduce to debug or remove once we have enumerated what happens here.
+ log.info("Pong " + pingSequenceId + " received too late, latest is " + node.getLastReceivedPongId());
+ }
+ }
}
diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcPingFactory.java b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcPingFactory.java
new file mode 100644
index 00000000000..ac8f0a59c20
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcPingFactory.java
@@ -0,0 +1,18 @@
+package com.yahoo.search.dispatch.rpc;
+
+import com.yahoo.search.cluster.ClusterMonitor;
+import com.yahoo.search.dispatch.searchcluster.Node;
+import com.yahoo.search.dispatch.searchcluster.PingFactory;
+import com.yahoo.search.dispatch.searchcluster.Pinger;
+import com.yahoo.search.dispatch.searchcluster.PongHandler;
+
+public class RpcPingFactory implements PingFactory {
+ private final RpcResourcePool rpcResourcePool;
+ public RpcPingFactory(RpcResourcePool rpcResourcePool) {
+ this.rpcResourcePool = rpcResourcePool;
+ }
+ @Override
+ public Pinger createPinger(Node node, ClusterMonitor<Node> monitor, PongHandler pongHandler) {
+ return new RpcPing(node, monitor, rpcResourcePool, pongHandler);
+ }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/Node.java b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/Node.java
index 2f70c37cd48..e93b633f09d 100644
--- a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/Node.java
+++ b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/Node.java
@@ -21,6 +21,8 @@ public class Node {
private final AtomicBoolean statusIsKnown = new AtomicBoolean(false);
private final AtomicBoolean working = new AtomicBoolean(true);
private final AtomicLong activeDocuments = new AtomicLong(0);
+ private final AtomicLong pingSequence = new AtomicLong(0);
+ private final AtomicLong lastPong = new AtomicLong(0);
public Node(int key, String hostname, int group) {
this.key = key;
@@ -28,6 +30,18 @@ public class Node {
this.group = group;
}
+ /** Give a monotonically increasing sequence number.*/
+ public long createPingSequenceId() { return pingSequence.incrementAndGet(); }
+ /** Checks if this pong is received in line and accepted, or out of band and should be ignored..*/
+ public boolean isLastReceivedPong(long pingId ) {
+ long last = lastPong.get();
+ while ((pingId > last) && ! lastPong.compareAndSet(last, pingId)) {
+ last = lastPong.get();
+ }
+ return last < pingId;
+ }
+ public long getLastReceivedPongId() { return lastPong.get(); }
+
/** Returns the unique and stable distribution key of this node */
public int key() { return key; }
diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/PingFactory.java b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/PingFactory.java
index b16fa941f68..2e07d8d61e6 100644
--- a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/PingFactory.java
+++ b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/PingFactory.java
@@ -1,13 +1,11 @@
// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.search.dispatch.searchcluster;
-import com.yahoo.prelude.Pong;
import com.yahoo.search.cluster.ClusterMonitor;
-import java.util.concurrent.Callable;
public interface PingFactory {
- Callable<Pong> createPinger(Node node, ClusterMonitor<Node> monitor);
+ Pinger createPinger(Node node, ClusterMonitor<Node> monitor, PongHandler pongHandler);
}
diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/Pinger.java b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/Pinger.java
new file mode 100644
index 00000000000..b4a7ccbf98c
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/Pinger.java
@@ -0,0 +1,12 @@
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.dispatch.searchcluster;
+
+/**
+ * Send a ping and ensure that the pong is propagated to the ponghandler.
+ * Should not wait as this should be done in parallel on all nodes.
+ *
+ * @author baldersheim
+ */
+public interface Pinger {
+ void ping();
+}
diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/PongHandler.java b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/PongHandler.java
new file mode 100644
index 00000000000..c0579b5d36e
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/PongHandler.java
@@ -0,0 +1,12 @@
+package com.yahoo.search.dispatch.searchcluster;
+
+import com.yahoo.prelude.Pong;
+
+/**
+ * Handle the Pong result of a Ping.
+ *
+ * @author baldersheim
+ */
+public interface PongHandler {
+ void handle(Pong pong);
+}
diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java
index 5f211c37917..d462479226a 100644
--- a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java
+++ b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java
@@ -10,7 +10,6 @@ import com.yahoo.net.HostName;
import com.yahoo.prelude.Pong;
import com.yahoo.search.cluster.ClusterMonitor;
import com.yahoo.search.cluster.NodeManager;
-import com.yahoo.search.result.ErrorMessage;
import com.yahoo.vespa.config.search.DispatchConfig;
import java.util.LinkedHashMap;
@@ -18,13 +17,7 @@ import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.OptionalInt;
-import java.util.concurrent.ExecutionException;
import java.util.concurrent.Executor;
-import java.util.concurrent.FutureTask;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.TimeoutException;
-import java.util.function.Predicate;
-import java.util.logging.Level;
import java.util.logging.Logger;
import java.util.stream.Collectors;
@@ -43,9 +36,8 @@ public class SearchCluster implements NodeManager<Node> {
private final ImmutableMap<Integer, Group> groups;
private final ImmutableMultimap<String, Node> nodesByHost;
private final ImmutableList<Group> orderedGroups;
- private final ClusterMonitor<Node> clusterMonitor;
private final VipStatus vipStatus;
- private PingFactory pingFactory;
+ private final PingFactory pingFactory;
private long nextLogTime = 0;
/**
@@ -58,10 +50,12 @@ public class SearchCluster implements NodeManager<Node> {
*/
private final Optional<Node> localCorpusDispatchTarget;
- public SearchCluster(String clusterId, DispatchConfig dispatchConfig, int containerClusterSize, VipStatus vipStatus) {
+ public SearchCluster(String clusterId, DispatchConfig dispatchConfig, int containerClusterSize,
+ VipStatus vipStatus, PingFactory pingFactory) {
this.clusterId = clusterId;
this.dispatchConfig = dispatchConfig;
this.vipStatus = vipStatus;
+ this.pingFactory = pingFactory;
List<Node> nodes = toNodes(dispatchConfig);
this.size = nodes.size();
@@ -84,29 +78,18 @@ public class SearchCluster implements NodeManager<Node> {
this.nodesByHost = nodesByHostBuilder.build();
this.localCorpusDispatchTarget = findLocalCorpusDispatchTarget(HostName.getLocalhost(),
- size,
- containerClusterSize,
- nodesByHost,
- groups);
-
- this.clusterMonitor = new ClusterMonitor<>(this);
- }
-
- public void shutDown() {
- clusterMonitor.shutdown();
+ size,
+ containerClusterSize,
+ nodesByHost,
+ groups);
}
-
- public void startClusterMonitoring(PingFactory pingFactory) {
- this.pingFactory = pingFactory;
-
+ public void addMonitoring(ClusterMonitor clusterMonitor) {
for (var group : orderedGroups) {
for (var node : group.nodes())
clusterMonitor.add(node, true);
}
}
- ClusterMonitor<Node> clusterMonitor() { return clusterMonitor; }
-
private static Optional<Node> findLocalCorpusDispatchTarget(String selfHostname,
int searchClusterSize,
int containerClusterSize,
@@ -247,7 +230,7 @@ public class SearchCluster implements NodeManager<Node> {
vipStatus.removeFromRotation(clusterId);
}
- private boolean hasInformationAboutAllNodes() {
+ public boolean hasInformationAboutAllNodes() {
return nodesByHost.values().stream().allMatch(node -> node.isWorking() != null);
}
@@ -263,24 +246,33 @@ public class SearchCluster implements NodeManager<Node> {
return localCorpusDispatchTarget.isPresent() && localCorpusDispatchTarget.get().group() == group.id();
}
+ private static class PongCallback implements PongHandler {
+ private final ClusterMonitor<Node> clusterMonitor;
+ private final Node node;
+ PongCallback(Node node, ClusterMonitor<Node> clusterMonitor) {
+ this.node = node;
+ this.clusterMonitor = clusterMonitor;
+ }
+ @Override
+ public void handle(Pong pong) {
+ if (pong.badResponse()) {
+ clusterMonitor.failed(node, pong.error().get());
+ } else {
+ if (pong.activeDocuments().isPresent()) {
+ node.setActiveDocuments(pong.activeDocuments().get());
+ }
+ clusterMonitor.responded(node);
+ }
+ }
+ }
+
/** Used by the cluster monitor to manage node status */
@Override
- public void ping(Node node, Executor executor) {
+ public void ping(ClusterMonitor clusterMonitor, Node node, Executor executor) {
if (pingFactory == null) return; // not initialized yet
- FutureTask<Pong> futurePong = new FutureTask<>(pingFactory.createPinger(node, clusterMonitor));
- executor.execute(futurePong);
- Pong pong = getPong(futurePong, node);
- futurePong.cancel(true);
-
- if (pong.badResponse()) {
- clusterMonitor.failed(node, pong.error().get());
- } else {
- if (pong.activeDocuments().isPresent()) {
- node.setActiveDocuments(pong.activeDocuments().get());
- }
- clusterMonitor.responded(node);
- }
+ Pinger pinger = pingFactory.createPinger(node, clusterMonitor, new PongCallback(node, clusterMonitor));
+ pinger.ping();
}
private void pingIterationCompletedSingleGroup() {
@@ -353,20 +345,6 @@ public class SearchCluster implements NodeManager<Node> {
return workingNodes + nodesAllowedDown >= nodesInGroup;
}
- private Pong getPong(FutureTask<Pong> futurePong, Node node) {
- try {
- return futurePong.get(clusterMonitor.getConfiguration().getFailLimit(), TimeUnit.MILLISECONDS);
- } catch (InterruptedException e) {
- log.log(Level.WARNING, "Exception pinging " + node, e);
- return new Pong(ErrorMessage.createUnspecifiedError("Ping was interrupted: " + node));
- } catch (ExecutionException e) {
- log.log(Level.WARNING, "Exception pinging " + node, e);
- return new Pong(ErrorMessage.createUnspecifiedError("Execution was interrupted: " + node));
- } catch (TimeoutException e) {
- return new Pong(ErrorMessage.createNoAnswerWhenPingingNode("Ping thread timed out"));
- }
- }
-
/**
* Calculate whether a subset of nodes in a group has enough coverage
*/
diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileProperties.java b/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileProperties.java
index b250560e2f3..5d4f39cecbf 100644
--- a/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileProperties.java
+++ b/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileProperties.java
@@ -70,7 +70,7 @@ public class QueryProfileProperties extends Properties {
* @throws IllegalArgumentException if this property cannot be set in the wrapped query profile
*/
@Override
- public void set(CompoundName name, Object value, Map<String,String> context) {
+ public void set(CompoundName name, Object value, Map<String, String> context) {
// TODO: Refactor
try {
name = unalias(name, context);
diff --git a/container-search/src/main/java/com/yahoo/search/query/properties/PropertyMap.java b/container-search/src/main/java/com/yahoo/search/query/properties/PropertyMap.java
index 30fc98ac6b1..4f30331e738 100644
--- a/container-search/src/main/java/com/yahoo/search/query/properties/PropertyMap.java
+++ b/container-search/src/main/java/com/yahoo/search/query/properties/PropertyMap.java
@@ -37,7 +37,7 @@ public class PropertyMap extends Properties {
* Return true if this value should be set in this map, false if the set should be propagated instead
* This default implementation always returns true.
*/
- protected boolean shouldSet(CompoundName name,Object value) { return true; }
+ protected boolean shouldSet(CompoundName name, Object value) { return true; }
@Override
public Object get(CompoundName name, Map<String,String> context,
diff --git a/container-search/src/main/java/com/yahoo/search/query/properties/QueryProperties.java b/container-search/src/main/java/com/yahoo/search/query/properties/QueryProperties.java
index a4c150b606e..dfe6c2af44b 100644
--- a/container-search/src/main/java/com/yahoo/search/query/properties/QueryProperties.java
+++ b/container-search/src/main/java/com/yahoo/search/query/properties/QueryProperties.java
@@ -44,7 +44,7 @@ public class QueryProperties extends Properties {
@Override
public Object get(CompoundName key,
- Map<String,String> context,
+ Map<String, String> context,
com.yahoo.processing.request.Properties substitution) {
if (key.size() == 2 && key.first().equals(Model.MODEL)) {
Model model = query.getModel();
diff --git a/container-search/src/main/java/com/yahoo/search/query/properties/RequestContextProperties.java b/container-search/src/main/java/com/yahoo/search/query/properties/RequestContextProperties.java
index ee09521fa74..6cf27fc9a3e 100644
--- a/container-search/src/main/java/com/yahoo/search/query/properties/RequestContextProperties.java
+++ b/container-search/src/main/java/com/yahoo/search/query/properties/RequestContextProperties.java
@@ -7,7 +7,7 @@ import com.yahoo.search.query.Properties;
import java.util.Map;
/**
- * Turns get(name) into get(name,request) using the request given at construction time.
+ * Turns get(name) into get(name, request) using the request given at construction time.
* This is used to allow the query's request to be supplied to all property requests
* without forcing users of the query.properties() to supply this explicitly.
*
@@ -22,18 +22,18 @@ public class RequestContextProperties extends Properties {
}
@Override
- public Object get(CompoundName name,Map<String,String> context,
+ public Object get(CompoundName name, Map<String,String> context,
com.yahoo.processing.request.Properties substitution) {
return super.get(name, context == null ? requestMap : context, substitution);
}
@Override
- public void set(CompoundName name,Object value,Map<String,String> context) {
+ public void set(CompoundName name, Object value, Map<String,String> context) {
super.set(name, value, context == null ? requestMap : context);
}
@Override
- public Map<String, Object> listProperties(CompoundName path,Map<String,String> context,
+ public Map<String, Object> listProperties(CompoundName path, Map<String,String> context,
com.yahoo.processing.request.Properties substitution) {
return super.listProperties(path, context == null ? requestMap : context, substitution);
}
diff --git a/container-search/src/main/java/com/yahoo/search/querytransform/VespaLowercasingSearcher.java b/container-search/src/main/java/com/yahoo/search/querytransform/VespaLowercasingSearcher.java
index 1e8f436a05a..25488aa7bbc 100644
--- a/container-search/src/main/java/com/yahoo/search/querytransform/VespaLowercasingSearcher.java
+++ b/container-search/src/main/java/com/yahoo/search/querytransform/VespaLowercasingSearcher.java
@@ -44,4 +44,5 @@ public class VespaLowercasingSearcher extends LowercasingSearcher {
Index index = indexFacts.getIndex(sb.toString());
return index.isLowercase() || index.isAttribute();
}
+
}
diff --git a/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/MockDispatcher.java b/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/MockDispatcher.java
index 440e3b8d78f..d5e43fba92d 100644
--- a/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/MockDispatcher.java
+++ b/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/MockDispatcher.java
@@ -2,8 +2,10 @@
package com.yahoo.prelude.fastsearch.test;
import com.yahoo.container.handler.VipStatus;
+import com.yahoo.search.cluster.ClusterMonitor;
import com.yahoo.search.dispatch.Dispatcher;
import com.yahoo.search.dispatch.rpc.RpcInvokerFactory;
+import com.yahoo.search.dispatch.rpc.RpcPingFactory;
import com.yahoo.search.dispatch.rpc.RpcResourcePool;
import com.yahoo.search.dispatch.searchcluster.Node;
import com.yahoo.search.dispatch.searchcluster.SearchCluster;
@@ -22,7 +24,7 @@ class MockDispatcher extends Dispatcher {
public static MockDispatcher create(List<Node> nodes, RpcResourcePool rpcResourcePool,
int containerClusterSize, VipStatus vipStatus) {
var dispatchConfig = toDispatchConfig(nodes);
- var searchCluster = new SearchCluster("a", dispatchConfig, containerClusterSize, vipStatus);
+ var searchCluster = new SearchCluster("a", dispatchConfig, containerClusterSize, vipStatus, new RpcPingFactory(rpcResourcePool));
return new MockDispatcher(searchCluster, dispatchConfig, rpcResourcePool);
}
@@ -31,7 +33,7 @@ class MockDispatcher extends Dispatcher {
}
private MockDispatcher(SearchCluster searchCluster, DispatchConfig dispatchConfig, RpcInvokerFactory invokerFactory) {
- super(searchCluster, dispatchConfig, invokerFactory, new MockMetric());
+ super(new ClusterMonitor<>(searchCluster, true), searchCluster, dispatchConfig, invokerFactory, new MockMetric());
}
static DispatchConfig toDispatchConfig(List<Node> nodes) {
diff --git a/container-search/src/test/java/com/yahoo/prelude/querytransform/test/QueryRewriteTestCase.java b/container-search/src/test/java/com/yahoo/prelude/querytransform/test/QueryRewriteTestCase.java
index 11922cf640a..36137abd9b8 100644
--- a/container-search/src/test/java/com/yahoo/prelude/querytransform/test/QueryRewriteTestCase.java
+++ b/container-search/src/test/java/com/yahoo/prelude/querytransform/test/QueryRewriteTestCase.java
@@ -1,15 +1,22 @@
// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.prelude.querytransform.test;
+import com.yahoo.prelude.IndexFacts;
import com.yahoo.prelude.query.AndItem;
import com.yahoo.prelude.query.NotItem;
import com.yahoo.prelude.query.OrItem;
+import com.yahoo.prelude.query.QueryCanonicalizer;
import com.yahoo.prelude.query.WordItem;
import com.yahoo.prelude.querytransform.QueryRewrite;
+import com.yahoo.prelude.querytransform.RecallSearcher;
import com.yahoo.search.Query;
+import com.yahoo.search.Result;
+import com.yahoo.search.searchchain.Execution;
+import com.yahoo.search.test.QueryTestCase;
import org.junit.Test;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
/**
@@ -38,9 +45,17 @@ public class QueryRewriteTestCase {
assertRewritten(query, "OR sddocname:per foo bar");
((OrItem)query.getModel().getQueryTree().getRoot()).getItem(2).setRanked(false); // set 'bar' unranked
assertRewritten(query, "OR sddocname:per foo");
-
assertRewritten("sddocname:per OR foo OR (bar AND fuz)", "per", "OR sddocname:per foo (AND bar fuz)");
+ }
+ @Test
+ public void testRankContributingTermsAreNotRemovedOnFullRecall() {
+ Query query = new Query(QueryTestCase.httpEncode("?query=default:term1 OR default:term2 OR default:term3 OR sddocname:per&type=adv&recall=+id:1&restrict=per"));
+ RecallSearcher searcher = new RecallSearcher();
+ Result result = new Execution(searcher, Execution.Context.createContextStub(new IndexFacts())).search(query);
+ assertNull(result.hits().getError());
+ assertNull(QueryCanonicalizer.canonicalize(query));
+ assertRewritten(query, "AND (OR default:term1 default:term2 default:term3 sddocname:per) |id:1");
}
@Test
@@ -88,6 +103,7 @@ public class QueryRewriteTestCase {
private static void assertRewritten(Query query, String expectedOptimizedQuery) {
QueryRewrite.optimizeByRestrict(query);
+ QueryRewrite.optimizeAndNot(query);
QueryRewrite.collapseSingleComposites(query);
assertEquals(expectedOptimizedQuery, query.getModel().getQueryTree().toString());
}
diff --git a/container-search/src/test/java/com/yahoo/search/dispatch/DispatcherTest.java b/container-search/src/test/java/com/yahoo/search/dispatch/DispatcherTest.java
index de6bafa267a..5433a28dd6e 100644
--- a/container-search/src/test/java/com/yahoo/search/dispatch/DispatcherTest.java
+++ b/container-search/src/test/java/com/yahoo/search/dispatch/DispatcherTest.java
@@ -1,22 +1,21 @@
// Copyright 2019 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.search.dispatch;
-import com.yahoo.prelude.Pong;
import com.yahoo.prelude.fastsearch.VespaBackEndSearcher;
import com.yahoo.prelude.fastsearch.test.MockMetric;
-import com.yahoo.processing.request.CompoundName;
import com.yahoo.search.Query;
import com.yahoo.search.Result;
import com.yahoo.search.cluster.ClusterMonitor;
import com.yahoo.search.dispatch.searchcluster.Node;
import com.yahoo.search.dispatch.searchcluster.PingFactory;
+import com.yahoo.search.dispatch.searchcluster.Pinger;
+import com.yahoo.search.dispatch.searchcluster.PongHandler;
import com.yahoo.search.dispatch.searchcluster.SearchCluster;
import org.junit.Test;
import java.util.List;
import java.util.Optional;
import java.util.OptionalInt;
-import java.util.concurrent.Callable;
import static com.yahoo.search.dispatch.MockSearchCluster.createDispatchConfig;
import static org.junit.Assert.assertEquals;
@@ -39,9 +38,10 @@ public class DispatcherTest {
assertEquals(2, nodes.get(0).key());
return true;
});
- Dispatcher disp = new Dispatcher(cl, createDispatchConfig(), invokerFactory, invokerFactory, new MockMetric());
+ Dispatcher disp = new Dispatcher(new ClusterMonitor(cl, false), cl, createDispatchConfig(), invokerFactory, new MockMetric());
SearchInvoker invoker = disp.getSearchInvoker(q, null);
invokerFactory.verifyAllEventsProcessed();
+ disp.deconstruct();
}
@Test
@@ -53,9 +53,10 @@ public class DispatcherTest {
}
};
MockInvokerFactory invokerFactory = new MockInvokerFactory(cl, (n, a) -> true);
- Dispatcher disp = new Dispatcher(cl, createDispatchConfig(), invokerFactory, invokerFactory, new MockMetric());
+ Dispatcher disp = new Dispatcher(new ClusterMonitor(cl, false), cl, createDispatchConfig(), invokerFactory, new MockMetric());
SearchInvoker invoker = disp.getSearchInvoker(new Query(), null);
invokerFactory.verifyAllEventsProcessed();
+ disp.deconstruct();
}
@Test
@@ -69,9 +70,10 @@ public class DispatcherTest {
assertTrue(acceptIncompleteCoverage);
return true;
});
- Dispatcher disp = new Dispatcher(cl, createDispatchConfig(), invokerFactory, invokerFactory, new MockMetric());
+ Dispatcher disp = new Dispatcher(new ClusterMonitor(cl, false), cl, createDispatchConfig(), invokerFactory, new MockMetric());
SearchInvoker invoker = disp.getSearchInvoker(new Query(), null);
invokerFactory.verifyAllEventsProcessed();
+ disp.deconstruct();
}
@Test
@@ -80,8 +82,9 @@ public class DispatcherTest {
SearchCluster cl = new MockSearchCluster("1", 2, 1);
MockInvokerFactory invokerFactory = new MockInvokerFactory(cl, (n, a) -> false, (n, a) -> false);
- Dispatcher disp = new Dispatcher(cl, createDispatchConfig(), invokerFactory, invokerFactory, new MockMetric());
+ Dispatcher disp = new Dispatcher(new ClusterMonitor(cl, false), cl, createDispatchConfig(), invokerFactory, new MockMetric());
disp.getSearchInvoker(new Query(), null);
+ disp.deconstruct();
fail("Expected exception");
}
catch (IllegalStateException e) {
@@ -142,7 +145,7 @@ public class DispatcherTest {
}
@Override
- public Callable<Pong> createPinger(Node node, ClusterMonitor<Node> monitor) {
+ public Pinger createPinger(Node node, ClusterMonitor<Node> monitor, PongHandler pongHandler) {
fail("Unexpected call to createPinger");
return null;
}
diff --git a/container-search/src/test/java/com/yahoo/search/dispatch/LoadBalancerTest.java b/container-search/src/test/java/com/yahoo/search/dispatch/LoadBalancerTest.java
index 0496194f8ed..6eedb8239a9 100644
--- a/container-search/src/test/java/com/yahoo/search/dispatch/LoadBalancerTest.java
+++ b/container-search/src/test/java/com/yahoo/search/dispatch/LoadBalancerTest.java
@@ -29,7 +29,7 @@ public class LoadBalancerTest {
@Test
public void requireThatLoadBalancerServesSingleNodeSetups() {
Node n1 = new Node(0, "test-node1", 0);
- SearchCluster cluster = new SearchCluster("a", createDispatchConfig(n1), 1, null);
+ SearchCluster cluster = new SearchCluster("a", createDispatchConfig(n1), 1, null, null);
LoadBalancer lb = new LoadBalancer(cluster, true);
Optional<Group> grp = lb.takeGroup(null);
@@ -43,7 +43,7 @@ public class LoadBalancerTest {
public void requireThatLoadBalancerServesMultiGroupSetups() {
Node n1 = new Node(0, "test-node1", 0);
Node n2 = new Node(1, "test-node2", 1);
- SearchCluster cluster = new SearchCluster("a", createDispatchConfig(n1, n2), 1, null);
+ SearchCluster cluster = new SearchCluster("a", createDispatchConfig(n1, n2), 1, null, null);
LoadBalancer lb = new LoadBalancer(cluster, true);
Optional<Group> grp = lb.takeGroup(null);
@@ -59,7 +59,7 @@ public class LoadBalancerTest {
Node n2 = new Node(1, "test-node2", 0);
Node n3 = new Node(0, "test-node3", 1);
Node n4 = new Node(1, "test-node4", 1);
- SearchCluster cluster = new SearchCluster("a", createDispatchConfig(n1, n2, n3, n4), 2, null);
+ SearchCluster cluster = new SearchCluster("a", createDispatchConfig(n1, n2, n3, n4), 2, null, null);
LoadBalancer lb = new LoadBalancer(cluster, true);
Optional<Group> grp = lb.takeGroup(null);
@@ -70,7 +70,7 @@ public class LoadBalancerTest {
public void requireThatLoadBalancerReturnsDifferentGroups() {
Node n1 = new Node(0, "test-node1", 0);
Node n2 = new Node(1, "test-node2", 1);
- SearchCluster cluster = new SearchCluster("a", createDispatchConfig(n1, n2), 1, null);
+ SearchCluster cluster = new SearchCluster("a", createDispatchConfig(n1, n2), 1, null,null);
LoadBalancer lb = new LoadBalancer(cluster, true);
// get first group
diff --git a/container-search/src/test/java/com/yahoo/search/dispatch/MockSearchCluster.java b/container-search/src/test/java/com/yahoo/search/dispatch/MockSearchCluster.java
index a976b287f63..3b4d58cdfc2 100644
--- a/container-search/src/test/java/com/yahoo/search/dispatch/MockSearchCluster.java
+++ b/container-search/src/test/java/com/yahoo/search/dispatch/MockSearchCluster.java
@@ -30,7 +30,7 @@ public class MockSearchCluster extends SearchCluster {
}
public MockSearchCluster(String clusterId, DispatchConfig dispatchConfig, int groups, int nodesPerGroup) {
- super(clusterId, dispatchConfig, 1, null);
+ super(clusterId, dispatchConfig, 1, null, null);
ImmutableList.Builder<Group> orderedGroupBuilder = ImmutableList.builder();
ImmutableMap.Builder<Integer, Group> groupBuilder = ImmutableMap.builder();
diff --git a/container-search/src/test/java/com/yahoo/search/dispatch/searchcluster/SearchClusterTest.java b/container-search/src/test/java/com/yahoo/search/dispatch/searchcluster/SearchClusterTest.java
index 10a579b0e4f..766f9ea6c2d 100644
--- a/container-search/src/test/java/com/yahoo/search/dispatch/searchcluster/SearchClusterTest.java
+++ b/container-search/src/test/java/com/yahoo/search/dispatch/searchcluster/SearchClusterTest.java
@@ -20,6 +20,7 @@ import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
+import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
@@ -34,6 +35,7 @@ public class SearchClusterTest {
final int nodesPerGroup;
final VipStatus vipStatus;
final SearchCluster searchCluster;
+ final ClusterMonitor clusterMonitor;
final List<AtomicInteger> numDocsPerNode;
List<AtomicInteger> pingCounts;
@@ -57,74 +59,76 @@ public class SearchClusterTest {
numDocsPerNode.add(new AtomicInteger(1));
pingCounts.add(new AtomicInteger(0));
}
- searchCluster = new SearchCluster(clusterId, MockSearchCluster.createDispatchConfig(nodes), nodes.size() / nodesPerGroup, vipStatus);
+ searchCluster = new SearchCluster(clusterId, MockSearchCluster.createDispatchConfig(nodes), nodes.size() / nodesPerGroup,
+ vipStatus, new Factory(nodesPerGroup, numDocsPerNode, pingCounts));
+ clusterMonitor = new ClusterMonitor(searchCluster, false);
+ searchCluster.addMonitoring(clusterMonitor);
}
- void startMonitoring() {
- searchCluster.startClusterMonitoring(new Factory(nodesPerGroup, numDocsPerNode, pingCounts));
- }
-
- static private int maxFrom(List<AtomicInteger> list) {
- int max = list.get(0).get();
- for (AtomicInteger v : list) {
- if (v.get() > max) {
- max = v.get();
+ private int maxPingCount() {
+ int max = pingCounts.get(0).get();
+ for (AtomicInteger count : pingCounts) {
+ if (count.get() > max) {
+ max = count.get();
}
}
return max;
}
- private static int minFrom(List<AtomicInteger> list) {
- int min = list.get(0).get();
- for (AtomicInteger v : list) {
- if (v.get() < min) {
- min = v.get();
+ private int minPingCount() {
+ int min = pingCounts.get(0).get();
+ for (AtomicInteger count : pingCounts) {
+ if (count.get() < min) {
+ min = count.get();
}
}
return min;
}
- private void waitAtLeast(int atLeast, List<AtomicInteger> list) {
- while (minFrom(list) < atLeast) {
+ void waitOneFullPingRound() {
+ int minPingCount = minPingCount();
+ int atLeast = maxPingCount() + 1;
+ while (minPingCount < atLeast) {
ExecutorService executor = Executors.newCachedThreadPool();
- searchCluster.clusterMonitor().ping(executor);
+ clusterMonitor.ping(executor);
executor.shutdown();
try {
boolean completed = executor.awaitTermination(120, TimeUnit.SECONDS);
if ( ! completed )
throw new IllegalStateException("Ping thread timed out");
+ // Since a separate thread will be modifying values in pingCounts, we need to wait for the thread to
+ // finish before re-reading the minimum value
+ minPingCount = minPingCount();
} catch (InterruptedException e) {
- System.out.println("Ping thread interrupted");
+ throw new RuntimeException(e);
}
}
}
- void waitOneFullPingRound() {
- waitAtLeast(maxFrom(pingCounts) + 1, pingCounts);
- }
-
@Override
public void close() {
- searchCluster.shutDown();
+ clusterMonitor.shutdown();
}
static class Factory implements PingFactory {
- static class Pinger implements Callable<Pong> {
+ static class PingJob implements Pinger {
private final AtomicInteger numDocs;
private final AtomicInteger pingCount;
- Pinger(AtomicInteger numDocs, AtomicInteger pingCount) {
+ private final PongHandler pongHandler;
+ PingJob(AtomicInteger numDocs, AtomicInteger pingCount, PongHandler pongHandler) {
this.numDocs = numDocs;
this.pingCount = pingCount;
+ this.pongHandler = pongHandler;
}
@Override
- public Pong call() {
+ public void ping() {
int docs = numDocs.get();
- pingCount.incrementAndGet();
- return (docs < 0)
+ pongHandler.handle ((docs < 0)
? new Pong(ErrorMessage.createBackendCommunicationError("Negative numDocs = " + docs))
- : new Pong(docs);
+ : new Pong(docs));
+ pingCount.incrementAndGet();
}
}
@@ -139,9 +143,9 @@ public class SearchClusterTest {
}
@Override
- public Callable<Pong> createPinger(Node node, ClusterMonitor<Node> monitor) {
+ public Pinger createPinger(Node node, ClusterMonitor<Node> monitor, PongHandler pongHandler) {
int index = node.group() * numPerGroup + node.key();
- return new Pinger(activeDocs.get(index), pingCounts.get(index));
+ return new PingJob(activeDocs.get(index), pingCounts.get(index), pongHandler);
}
}
@@ -153,7 +157,6 @@ public class SearchClusterTest {
assertTrue(test.searchCluster.localCorpusDispatchTarget().isEmpty());
assertFalse(test.vipStatus.isInRotation());
- test.startMonitoring();
test.waitOneFullPingRound();
assertTrue(test.vipStatus.isInRotation());
}
@@ -162,7 +165,6 @@ public class SearchClusterTest {
@Test
public void requireThatZeroDocsAreFine() {
try (State test = new State("cluster.1", 2, "a", "b")) {
- test.startMonitoring();
test.waitOneFullPingRound();
assertTrue(test.vipStatus.isInRotation());
@@ -184,7 +186,6 @@ public class SearchClusterTest {
assertTrue(test.searchCluster.localCorpusDispatchTarget().isPresent());
assertFalse(test.vipStatus.isInRotation());
- test.startMonitoring();
test.waitOneFullPingRound();
assertTrue(test.vipStatus.isInRotation());
}
@@ -196,7 +197,6 @@ public class SearchClusterTest {
assertTrue(test.searchCluster.localCorpusDispatchTarget().isPresent());
assertFalse(test.vipStatus.isInRotation());
- test.startMonitoring();
test.waitOneFullPingRound();
assertTrue(test.vipStatus.isInRotation());
test.numDocsPerNode.get(0).set(-1);
@@ -209,7 +209,6 @@ public class SearchClusterTest {
public void requireThatVipStatusDownWhenLocalIsDown() {
try (State test = new State("cluster.1",1,HostName.getLocalhost(), "b")) {
- test.startMonitoring();
test.waitOneFullPingRound();
assertTrue(test.vipStatus.isInRotation());
assertTrue(test.searchCluster.localCorpusDispatchTarget().isPresent());
@@ -245,7 +244,6 @@ public class SearchClusterTest {
List<String> nodeNames = generateNodeNames(numGroups, nodesPerGroup);
try (State test = new State("cluster.1", nodesPerGroup, nodeNames)) {
- test.startMonitoring();
test.waitOneFullPingRound();
assertTrue(test.vipStatus.isInRotation());
assertTrue(test.searchCluster.localCorpusDispatchTarget().isEmpty());
@@ -273,8 +271,8 @@ public class SearchClusterTest {
static private List<String> generateNodeNames(int numGroups, int nodesPerGroup) {
List<String> nodeNames = new ArrayList<>(numGroups*nodesPerGroup);
for (int g = 0; g < numGroups; g++) {
- for (int n=0; n < nodesPerGroup; n++) {
- nodeNames.add(new StringBuilder("node.").append(g).append('.').append(n).toString());
+ for (int n = 0; n < nodesPerGroup; n++) {
+ nodeNames.add("node." + g + '.' + n);
}
}
return nodeNames;
@@ -284,7 +282,6 @@ public class SearchClusterTest {
List<String> nodeNames = generateNodeNames(numGroups, nodesPerGroup);
try (State test = new State("cluster.1", nodesPerGroup, nodeNames)) {
- test.startMonitoring();
test.waitOneFullPingRound();
assertTrue(test.vipStatus.isInRotation());
assertTrue(test.searchCluster.localCorpusDispatchTarget().isEmpty());
@@ -310,4 +307,18 @@ public class SearchClusterTest {
verifyThatVipStatusUpRequireOnlyOneOnlineNode(3, 3);
}
+ @Test
+ public void requireThatPingSequenceIsUpHeld() {
+ Node node = new Node(1, "n", 1);
+ assertEquals(1, node.createPingSequenceId());
+ assertEquals(2, node.createPingSequenceId());
+ assertEquals(0, node.getLastReceivedPongId());
+ assertTrue(node.isLastReceivedPong(2));
+ assertEquals(2, node.getLastReceivedPongId());
+ assertFalse(node.isLastReceivedPong(1));
+ assertFalse(node.isLastReceivedPong(2));
+ assertTrue(node.isLastReceivedPong(3));
+ assertEquals(3, node.getLastReceivedPongId());
+ }
+
}
diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/test/QueryProfileTestCase.java b/container-search/src/test/java/com/yahoo/search/query/profile/test/QueryProfileTestCase.java
index 46efb736918..bda191ee910 100644
--- a/container-search/src/test/java/com/yahoo/search/query/profile/test/QueryProfileTestCase.java
+++ b/container-search/src/test/java/com/yahoo/search/query/profile/test/QueryProfileTestCase.java
@@ -326,14 +326,14 @@ public class QueryProfileTestCase {
assertEquals("mormor-model.b", annetBarnMap.get("venn.model.b"));
}
- /** Tests that dots are followed when setting overridability */
+ /** Dots are followed when setting overridability */
@Test
public void testInstanceOverridable() {
QueryProfile profile = new QueryProfile("root/unoverridableIndex");
profile.set("model.defaultIndex","default", null);
profile.setOverridable("model.defaultIndex", false,null);
- assertFalse(profile.isDeclaredOverridable("model.defaultIndex",null).booleanValue());
+ assertFalse(profile.isDeclaredOverridable("model.defaultIndex",null));
// Parameters should be ignored
Query query = new Query(HttpRequest.createTestRequest("?model.defaultIndex=title", Method.GET), profile.compile(null));
@@ -345,7 +345,7 @@ public class QueryProfileTestCase {
assertEquals("de", query.getModel().getLanguage().languageCode());
}
- /** Tests that dots are followed when setting overridability...also with variants */
+ /** Dots are followed when setting overridability, also with variants */
@Test
public void testInstanceOverridableWithVariants() {
QueryProfile profile = new QueryProfile("root/unoverridableIndex");
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/ServiceRegistry.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/ServiceRegistry.java
index 6ca5cae0455..9e5b01a91d7 100644
--- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/ServiceRegistry.java
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/ServiceRegistry.java
@@ -3,7 +3,7 @@ package com.yahoo.vespa.hosted.controller.api.integration;
import com.yahoo.vespa.hosted.controller.api.integration.aws.AwsEventFetcher;
import com.yahoo.vespa.hosted.controller.api.integration.aws.ResourceTagger;
-import com.yahoo.vespa.hosted.controller.api.integration.certificates.ApplicationCertificateProvider;
+import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificateProvider;
import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServer;
import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationStore;
import com.yahoo.vespa.hosted.controller.api.integration.deployment.ArtifactRepository;
@@ -46,7 +46,7 @@ public interface ServiceRegistry {
Mailer mailer();
- ApplicationCertificateProvider applicationCertificateProvider();
+ EndpointCertificateProvider endpointCertificateProvider();
MeteringClient meteringService();
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/ZtsClientMock.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/ZtsClientMock.java
index ec60f11060d..53e80fcb2ed 100644
--- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/ZtsClientMock.java
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/ZtsClientMock.java
@@ -2,6 +2,7 @@
package com.yahoo.vespa.hosted.controller.api.integration.athenz;
import com.yahoo.security.Pkcs10Csr;
+import com.yahoo.vespa.athenz.api.AthenzAccessToken;
import com.yahoo.vespa.athenz.api.AthenzDomain;
import com.yahoo.vespa.athenz.api.AthenzIdentity;
import com.yahoo.vespa.athenz.api.AthenzRole;
@@ -74,6 +75,16 @@ public class ZtsClientMock implements ZtsClient {
}
@Override
+ public AthenzAccessToken getAccessToken(AthenzDomain domain) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public AthenzAccessToken getAccessToken(List<AthenzRole> athenzRole) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
public X509Certificate getRoleCertificate(AthenzRole role, Pkcs10Csr csr, Duration expiry) {
throw new UnsupportedOperationException();
}
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/ApplicationCertificate.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/ApplicationCertificate.java
deleted file mode 100644
index 41f5b65d263..00000000000
--- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/ApplicationCertificate.java
+++ /dev/null
@@ -1,43 +0,0 @@
-// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.hosted.controller.api.integration.certificates;
-
-import java.util.Objects;
-
-/**
- * Represents a reference to a certificate and private key.
- *
- * @author mortent
- * @author andreer
- */
-public class ApplicationCertificate {
-
- private final String secretsKeyNamePrefix;
-
- public ApplicationCertificate(String secretsKeyNamePrefix) {
- this.secretsKeyNamePrefix = Objects.requireNonNull(secretsKeyNamePrefix, "secretsKeyNamePrefix must be non-null");
- }
-
- /** The prefix of keys identifying this certificate and its private key in a key store */
- public String secretsKeyNamePrefix() {
- return secretsKeyNamePrefix;
- }
-
- @Override
- public boolean equals(Object o) {
- if (this == o) return true;
- if (o == null || getClass() != o.getClass()) return false;
- ApplicationCertificate that = (ApplicationCertificate) o;
- return Objects.equals(secretsKeyNamePrefix, that.secretsKeyNamePrefix);
- }
-
- @Override
- public int hashCode() {
- return Objects.hash(secretsKeyNamePrefix);
- }
-
- @Override
- public String toString() {
- return "application certificate '" + secretsKeyNamePrefix + "'";
- }
-
-}
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/ApplicationCertificateMock.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/EndpointCertificateMock.java
index cc2d08c3fcd..aa0ac5f8296 100644
--- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/ApplicationCertificateMock.java
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/EndpointCertificateMock.java
@@ -12,7 +12,7 @@ import java.util.UUID;
/**
* @author tokle
*/
-public class ApplicationCertificateMock implements ApplicationCertificateProvider {
+public class EndpointCertificateMock implements EndpointCertificateProvider {
private final Map<ApplicationId, List<String>> dnsNames = new HashMap<>();
@@ -21,11 +21,12 @@ public class ApplicationCertificateMock implements ApplicationCertificateProvide
}
@Override
- public ApplicationCertificate requestCaSignedCertificate(ApplicationId applicationId, List<String> dnsNames) {
+ public EndpointCertificateMetadata requestCaSignedCertificate(ApplicationId applicationId, List<String> dnsNames) {
this.dnsNames.put(applicationId, dnsNames);
- return new ApplicationCertificate(String.format("vespa.tls.%s.%s@%s", applicationId.tenant(),
- applicationId.application(),
- UUID.randomUUID().toString()));
+ String endpointCertificatePrefix = String.format("vespa.tls.%s.%s@%s", applicationId.tenant(),
+ applicationId.application(),
+ UUID.randomUUID().toString());
+ return new EndpointCertificateMetadata(endpointCertificatePrefix + "-key", endpointCertificatePrefix + "-cert", 0);
}
}
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/ApplicationCertificateProvider.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/EndpointCertificateProvider.java
index b6ad1701449..147ada51816 100644
--- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/ApplicationCertificateProvider.java
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/EndpointCertificateProvider.java
@@ -6,12 +6,12 @@ import com.yahoo.config.provision.ApplicationId;
import java.util.List;
/**
- * Generates a certificate.
+ * Generates an endpoint certificate for an application instance.
*
* @author andreer
*/
-public interface ApplicationCertificateProvider {
+public interface EndpointCertificateProvider {
- ApplicationCertificate requestCaSignedCertificate(ApplicationId applicationId, List<String> dnsNames);
+ EndpointCertificateMetadata requestCaSignedCertificate(ApplicationId applicationId, List<String> dnsNames);
}
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServer.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServer.java
index 14d3e88e631..722f9c4e33b 100644
--- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServer.java
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServer.java
@@ -44,7 +44,9 @@ public interface ConfigServer {
ApplicationView getApplicationView(String tenantName, String applicationName, String instanceName, String environment, String region);
- Map<?,?> getServiceApiResponse(String tenantName, String applicationName, String instanceName, String environment, String region, String serviceName, String restPath);
+ Map<?,?> getServiceApiResponse(DeploymentId deployment, String serviceName, String restPath);
+
+ String getClusterControllerStatus(DeploymentId deployment, String restPath);
/**
* Gets the Vespa logs of the given deployment.
@@ -111,15 +113,12 @@ public interface ConfigServer {
TesterCloud.Status getTesterStatus(DeploymentId deployment);
/** Starts tests on tester node */
- // TODO: Remove default implementation when implemented in internal repo
- default String startTests(DeploymentId deployment, TesterCloud.Suite suite, byte[] config) { return "Tests started"; }
+ String startTests(DeploymentId deployment, TesterCloud.Suite suite, byte[] config);
/** Gets log from tester node */
- // TODO: Remove default implementation when implemented in internal repo
- default List<LogEntry> getTesterLog(DeploymentId deployment, long after) { return List.of(); }
+ List<LogEntry> getTesterLog(DeploymentId deployment, long after);
/** Is tester node ready */
- // TODO: Remove default implementation when implemented in internal repo
- default boolean isTesterReady(DeploymentId deployment) { return false; }
+ boolean isTesterReady(DeploymentId deployment);
}
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/TesterCloud.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/TesterCloud.java
index 36b13b3496a..a1c4d379b6c 100644
--- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/TesterCloud.java
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/TesterCloud.java
@@ -18,15 +18,13 @@ public interface TesterCloud {
void startTests(URI testerUrl, Suite suite, byte[] config);
/** Signals the tester to run its tests. */
- // TODO: Remove default implementation when implementations have been updated
- default void startTests(DeploymentId deploymentId, Suite suite, byte[] config) {}
+ void startTests(DeploymentId deploymentId, Suite suite, byte[] config);
/** Returns the log entries from the tester with ids after the given threshold. */
List<LogEntry> getLog(URI testerUrl, long after);
/** Returns the log entries from the tester with ids after the given threshold. */
- // TODO: Remove default implementation when implementations have been updated
- default List<LogEntry> getLog(DeploymentId deploymentId, long after) { return List.of(); }
+ List<LogEntry> getLog(DeploymentId deploymentId, long after);
/** Returns the current status of the tester. */
Status getStatus(URI testerUrl);
@@ -41,8 +39,7 @@ public interface TesterCloud {
boolean testerReady(URI endpointUrl);
/** Returns whether the test container is ready to serve */
- // TODO: Remove default implementation when implementations have been updated
- default boolean testerReady(DeploymentId deploymentId) { return false; }
+ boolean testerReady(DeploymentId deploymentId);
/** Returns whether the given URL is registered in DNS. */
boolean exists(URI endpointUrl);
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/MemoryNameService.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/MemoryNameService.java
index 75d49e542dc..680613e8065 100644
--- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/MemoryNameService.java
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/MemoryNameService.java
@@ -68,6 +68,11 @@ public class MemoryNameService implements NameService {
@Override
public List<Record> findRecords(Record.Type type, RecordData data) {
+ if (type == Record.Type.ALIAS && data.asString().contains("/")) {
+ // Validate the same expectation as of a real name service
+ throw new IllegalArgumentException("Finding " + Record.Type.ALIAS + " record by data should only include " +
+ "the FQDN name");
+ }
return records.stream()
.filter(record -> record.type() == type && record.data().equals(data))
.collect(Collectors.toUnmodifiableList());
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/routing/RoutingGeneratorMock.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/routing/RoutingGeneratorMock.java
index a7dc5f81bc0..cd0e6552596 100644
--- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/routing/RoutingGeneratorMock.java
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/routing/RoutingGeneratorMock.java
@@ -1,8 +1,10 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.hosted.controller.api.integration.routing;
import com.yahoo.config.provision.ClusterSpec;
+import com.yahoo.config.provision.zone.RoutingMethod;
import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId;
+import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneRegistry;
import java.net.URI;
import java.util.List;
@@ -27,17 +29,19 @@ public class RoutingGeneratorMock implements RoutingGenerator {
private final Map<DeploymentId, List<RoutingEndpoint>> routingTable = new ConcurrentHashMap<>();
private final List<RoutingEndpoint> defaultEndpoints;
+ private final ZoneRegistry zoneRegistry;
- public RoutingGeneratorMock() {
- this(List.of());
- }
-
- public RoutingGeneratorMock(List<RoutingEndpoint> endpoints) {
+ public RoutingGeneratorMock(List<RoutingEndpoint> endpoints, ZoneRegistry zoneRegistry) {
this.defaultEndpoints = List.copyOf(endpoints);
+ this.zoneRegistry = zoneRegistry;
}
@Override
public List<RoutingEndpoint> endpoints(DeploymentId deployment) {
+ if (!zoneRegistry.zones().routingMethod(RoutingMethod.shared).ids().contains(deployment.zoneId())) {
+ throw new IllegalArgumentException(deployment.zoneId() + " does not support routing method " +
+ RoutingMethod.shared);
+ }
if (routingTable.isEmpty()) return defaultEndpoints;
return routingTable.getOrDefault(deployment, List.of());
}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java
index fe1aaa001b1..17c9e852bd9 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java
@@ -17,6 +17,7 @@ import com.yahoo.vespa.athenz.api.AthenzPrincipal;
import com.yahoo.vespa.athenz.api.AthenzService;
import com.yahoo.vespa.athenz.api.AthenzUser;
import com.yahoo.vespa.curator.Lock;
+import com.yahoo.vespa.flags.FlagSource;
import com.yahoo.vespa.hosted.controller.api.ActivateResult;
import com.yahoo.vespa.hosted.controller.api.application.v4.model.DeployOptions;
import com.yahoo.vespa.hosted.controller.api.application.v4.model.configserverbindings.ConfigChangeActions;
@@ -106,8 +107,10 @@ public class ApplicationController {
private final ApplicationPackageValidator applicationPackageValidator;
private final EndpointCertificateManager endpointCertificateManager;
- ApplicationController(Controller controller, CuratorDb curator, AccessControl accessControl, Clock clock,
- SecretStore secretStore) {
+ ApplicationController(Controller controller, CuratorDb curator,
+ AccessControl accessControl, Clock clock,
+ SecretStore secretStore, FlagSource flagSource) {
+
this.controller = controller;
this.curator = curator;
this.accessControl = accessControl;
@@ -119,7 +122,7 @@ public class ApplicationController {
deploymentTrigger = new DeploymentTrigger(controller, clock);
applicationPackageValidator = new ApplicationPackageValidator(controller);
endpointCertificateManager = new EndpointCertificateManager(controller.zoneRegistry(), curator, secretStore,
- controller.serviceRegistry().applicationCertificateProvider(), clock);
+ controller.serviceRegistry().endpointCertificateProvider(), clock, flagSource);
// Update serialization format of all applications
Once.after(Duration.ofMinutes(1), () -> {
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java
index b4136e0de99..3d492bc00d4 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java
@@ -103,7 +103,7 @@ public class Controller extends AbstractComponent implements ApplicationIdSource
metrics = new ConfigServerMetrics(serviceRegistry.configServer());
nameServiceForwarder = new NameServiceForwarder(curator);
jobController = new JobController(this);
- applicationController = new ApplicationController(this, curator, accessControl, clock, secretStore);
+ applicationController = new ApplicationController(this, curator, accessControl, clock, secretStore, flagSource);
tenantController = new TenantController(this, curator, accessControl);
routingController = new RoutingController(this, Objects.requireNonNull(rotationsConfig, "RotationsConfig cannot be null"));
auditLogger = new AuditLogger(curator, clock);
@@ -151,13 +151,6 @@ public class Controller extends AbstractComponent implements ApplicationIdSource
return serviceRegistry.configServer().getApplicationView(tenantName, applicationName, instanceName, environment, region);
}
- // TODO: Model the response properly
- public Map<?,?> getServiceApiResponse(String tenantName, String applicationName, String instanceName,
- String environment, String region, String serviceName, String restPath) {
- return serviceRegistry.configServer().getServiceApiResponse(tenantName, applicationName, instanceName, environment, region,
- serviceName, restPath);
- }
-
/** Replace the current version status by a new one */
public void updateVersionStatus(VersionStatus newStatus) {
VersionStatus currentStatus = versionStatus();
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java
index 32b87b8f92b..29d6de858dc 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java
@@ -5,6 +5,7 @@ import com.yahoo.config.application.api.DeploymentInstanceSpec;
import com.yahoo.config.application.api.DeploymentSpec;
import com.yahoo.config.provision.ClusterSpec;
import com.yahoo.config.provision.InstanceName;
+import com.yahoo.config.provision.zone.RoutingMethod;
import com.yahoo.config.provision.zone.ZoneId;
import com.yahoo.vespa.hosted.controller.api.application.v4.model.EndpointStatus;
import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId;
@@ -72,18 +73,10 @@ public class RoutingController {
/** Returns all known endpoint URLs for given deployment, including global, in the shared routing layer */
public List<URI> legacyEndpointsOf(DeploymentId deployment) {
- if (controller.zoneRegistry().zones().directlyRouted().ids().contains(deployment.zoneId())) {
- return List.of(); // No shared routing layer in this zone.
- }
- try {
- return routingGenerator.endpoints(deployment).stream()
- .map(RoutingEndpoint::endpoint)
- .map(URI::create)
- .collect(Collectors.toUnmodifiableList());
- } catch (RuntimeException e) {
- log.log(Level.WARNING, "Failed to get endpoints for " + deployment, e);
- return List.of();
- }
+ return routingEndpointsOf(deployment).stream()
+ .map(RoutingEndpoint::endpoint)
+ .map(URI::create)
+ .collect(Collectors.toUnmodifiableList());
}
/** Returns all non-global endpoint URLs for given deployment, grouped by their cluster ID */
@@ -146,9 +139,9 @@ public class RoutingController {
/** Find the global endpoints of given deployment */
private List<RoutingEndpoint> legacyGlobalEndpointsOf(DeploymentId deployment) {
- return controller.serviceRegistry().routingGenerator().endpoints(deployment).stream()
- .filter(RoutingEndpoint::isGlobal)
- .collect(Collectors.toUnmodifiableList());
+ return routingEndpointsOf(deployment).stream()
+ .filter(RoutingEndpoint::isGlobal)
+ .collect(Collectors.toUnmodifiableList());
}
/**
@@ -222,4 +215,16 @@ public class RoutingController {
});
}
+ private List<RoutingEndpoint> routingEndpointsOf(DeploymentId deployment) {
+ if (!controller.zoneRegistry().zones().routingMethod(RoutingMethod.shared).ids().contains(deployment.zoneId())) {
+ return List.of(); // No rotations/shared routing layer in this zone.
+ }
+ try {
+ return routingGenerator.endpoints(deployment);
+ } catch (RuntimeException e) {
+ log.log(Level.WARNING, "Failed to get endpoints for " + deployment, e);
+ return List.of();
+ }
+ }
+
}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java
index 9a6daf026c3..c39255fd7a8 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java
@@ -111,7 +111,7 @@ public class Endpoint {
return URI.create(scheme + "://" +
sanitize(namePart(name, separator)) +
systemPart(system, separator) +
- sanitize(instancePart(application, zone, separator)) +
+ sanitize(instancePart(application, separator)) +
sanitize(application.application().value()) +
separator +
sanitize(application.tenant().value()) +
@@ -144,7 +144,7 @@ public class Endpoint {
return zone.region().value() + "." + zone.environment().value();
}
- private static String instancePart(ApplicationId application, ZoneId zone, String separator) {
+ private static String instancePart(ApplicationId application, String separator) {
if (application.instance().isDefault()) return ""; // Skip "default"
return application.instance().value() + separator;
}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java
index 6da93ce90a5..e5ab59b814a 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java
@@ -3,6 +3,7 @@ package com.yahoo.vespa.hosted.controller.deployment;
import com.google.common.collect.ImmutableSet;
import com.yahoo.component.Version;
+import com.yahoo.config.application.api.DeploymentInstanceSpec;
import com.yahoo.config.application.api.DeploymentSpec;
import com.yahoo.config.application.api.Notifications;
import com.yahoo.config.application.api.Notifications.When;
@@ -10,6 +11,7 @@ import com.yahoo.config.provision.ApplicationId;
import com.yahoo.config.provision.AthenzDomain;
import com.yahoo.config.provision.AthenzService;
import com.yahoo.config.provision.ClusterSpec;
+import com.yahoo.config.provision.Environment;
import com.yahoo.config.provision.NodeResources;
import com.yahoo.config.provision.zone.ZoneId;
import com.yahoo.log.LogLevel;
@@ -106,11 +108,12 @@ import static java.util.stream.Collectors.toList;
public class InternalStepRunner implements StepRunner {
private static final Logger logger = Logger.getLogger(InternalStepRunner.class.getName());
- private static final NodeResources DEFAULT_TESTER_RESOURCES =
+
+ static final NodeResources DEFAULT_TESTER_RESOURCES =
new NodeResources(1, 4, 50, 0.3, NodeResources.DiskSpeed.any);
// Must match exactly the advertised resources of an AWS instance type. Also consider that the container
// will have ~1.8 GB less memory than equivalent resources in AWS (VESPA-16259).
- private static final NodeResources DEFAULT_TESTER_RESOURCES_AWS =
+ static final NodeResources DEFAULT_TESTER_RESOURCES_AWS =
new NodeResources(2, 8, 50, 0.3, NodeResources.DiskSpeed.any);
static final Duration endpointTimeout = Duration.ofMinutes(15);
@@ -550,13 +553,14 @@ public class InternalStepRunner implements StepRunner {
}
logEndpoints(endpoints, logger);
- Optional<URI> testerEndpoint = controller.jobController().testerEndpoint(id);
+ Optional<URI> testerEndpoint = Optional.empty();
if (useConfigServerForTesterAPI(zoneId)) {
if ( ! controller.jobController().cloud().testerReady(getTesterDeploymentId(id))) {
logger.log(WARNING, "Tester container went bad!");
return Optional.of(error);
}
} else {
+ testerEndpoint = controller.jobController().testerEndpoint(id);
if (testerEndpoint.isEmpty()) {
logger.log(WARNING, "Endpoints for the tester container vanished again, while it was still active!");
return Optional.of(error);
@@ -764,10 +768,7 @@ public class InternalStepRunner implements StepRunner {
byte[] servicesXml = servicesXml(controller.zoneRegistry().accessControlDomain(),
! controller.system().isPublic(),
useTesterCertificate,
- testerFlavorFor(id, spec)
- .map(NodeResources::fromLegacyName)
- .orElse(zone.region().value().contains("aws-") ?
- DEFAULT_TESTER_RESOURCES_AWS : DEFAULT_TESTER_RESOURCES));
+ testerResourcesFor(zone, spec.requireInstance(id.application().instance())));
byte[] testPackage = controller.applications().applicationStore().getTester(id.application().tenant(), id.application().application(), version);
byte[] deploymentXml = deploymentXml(id.tester(),
spec.athenzDomain(),
@@ -813,12 +814,14 @@ public class InternalStepRunner implements StepRunner {
return useConfigServer;
}
- private static Optional<String> testerFlavorFor(RunId id, DeploymentSpec spec) {
- for (DeploymentSpec.Step step : spec.steps())
- if (step.concerns(id.type().environment()))
- return step.zones().get(0).testerFlavor();
-
- return Optional.empty();
+ static NodeResources testerResourcesFor(ZoneId zone, DeploymentInstanceSpec spec) {
+ return spec.steps().stream()
+ .filter(step -> step.concerns(zone.environment()))
+ .findFirst()
+ .flatMap(step -> step.zones().get(0).testerFlavor())
+ .map(NodeResources::fromLegacyName)
+ .orElse(zone.region().value().contains("aws-") ?
+ DEFAULT_TESTER_RESOURCES_AWS : DEFAULT_TESTER_RESOURCES);
}
/** Returns the generated services.xml content for the tester application. */
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java
index 021d710ebe4..10021441943 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java
@@ -186,7 +186,8 @@ public class JobController {
List<LogEntry> entries;
ZoneId zone = id.type().zone(controller.system());
if (useConfigServerForTesterAPI(zone)) {
- entries = cloud.getLog(new DeploymentId(id.application(), zone), run.lastTestLogEntry());
+ var testerId = new DeploymentId(id.tester().id(), zone);
+ entries = cloud.getLog(testerId, run.lastTestLogEntry());
} else {
Optional<URI> testerEndpoint = testerEndpoint(id);
if (testerEndpoint.isEmpty())
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/endpointcertificates/EndpointCertificateManager.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/endpointcertificates/EndpointCertificateManager.java
index ad8c0c73754..cf43e83d735 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/endpointcertificates/EndpointCertificateManager.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/endpointcertificates/EndpointCertificateManager.java
@@ -1,5 +1,6 @@
package com.yahoo.vespa.hosted.controller.endpointcertificates;
+import com.google.common.collect.Sets;
import com.yahoo.config.provision.ApplicationId;
import com.yahoo.config.provision.ClusterSpec;
import com.yahoo.config.provision.zone.ZoneApi;
@@ -8,9 +9,12 @@ import com.yahoo.container.jdisc.secretstore.SecretStore;
import com.yahoo.log.LogLevel;
import com.yahoo.security.SubjectAlternativeName;
import com.yahoo.security.X509CertificateUtils;
+import com.yahoo.vespa.flags.BooleanFlag;
+import com.yahoo.vespa.flags.FetchVector;
+import com.yahoo.vespa.flags.FlagSource;
+import com.yahoo.vespa.flags.Flags;
import com.yahoo.vespa.hosted.controller.Instance;
-import com.yahoo.vespa.hosted.controller.api.integration.certificates.ApplicationCertificate;
-import com.yahoo.vespa.hosted.controller.api.integration.certificates.ApplicationCertificateProvider;
+import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificateProvider;
import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificateMetadata;
import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneRegistry;
import com.yahoo.vespa.hosted.controller.application.Endpoint;
@@ -23,14 +27,19 @@ import java.time.Clock;
import java.time.Instant;
import java.util.ArrayList;
import java.util.Collections;
+import java.util.HashSet;
import java.util.List;
import java.util.Optional;
+import java.util.OptionalInt;
import java.util.Set;
import java.util.logging.Logger;
import java.util.stream.Collectors;
import java.util.stream.Stream;
/**
+ * Looks up stored endpoint certificate metadata, provisions new certificates if none is found,
+ * and refreshes certificates if a newer version is available.
+ *
* @author andreer
*/
public class EndpointCertificateManager {
@@ -40,64 +49,88 @@ public class EndpointCertificateManager {
private final ZoneRegistry zoneRegistry;
private final CuratorDb curator;
private final SecretStore secretStore;
- private final ApplicationCertificateProvider applicationCertificateProvider;
+ private final EndpointCertificateProvider endpointCertificateProvider;
private final Clock clock;
+ private final BooleanFlag useRefreshedEndpointCertificate;
public EndpointCertificateManager(ZoneRegistry zoneRegistry,
CuratorDb curator,
SecretStore secretStore,
- ApplicationCertificateProvider applicationCertificateProvider,
- Clock clock) {
+ EndpointCertificateProvider endpointCertificateProvider,
+ Clock clock, FlagSource flagSource) {
this.zoneRegistry = zoneRegistry;
this.curator = curator;
this.secretStore = secretStore;
- this.applicationCertificateProvider = applicationCertificateProvider;
+ this.endpointCertificateProvider = endpointCertificateProvider;
this.clock = clock;
+ this.useRefreshedEndpointCertificate = Flags.USE_REFRESHED_ENDPOINT_CERTIFICATE.bindTo(flagSource);
}
public Optional<EndpointCertificateMetadata> getEndpointCertificateMetadata(Instance instance, ZoneId zone) {
if (!zoneRegistry.zones().directlyRouted().ids().contains(zone)) return Optional.empty();
- // Re-use certificate if already provisioned
- Optional<EndpointCertificateMetadata> endpointCertificateMetadata =
+ // Re-use existing certificate if already provisioned
+ var endpointCertificateMetadata =
curator.readEndpointCertificateMetadata(instance.id())
- .or(() -> Optional.of(provisionEndpointCertificate(instance)));
+ .orElseGet(() -> provisionEndpointCertificate(instance));
- // Only logs warnings for now
- endpointCertificateMetadata.ifPresent(certificateMetadata -> verifyEndpointCertificate(certificateMetadata, instance, zone));
+ // If feature flag set for application, look for and use refreshed certificate
+ if (useRefreshedEndpointCertificate.with(FetchVector.Dimension.APPLICATION_ID, instance.id().serializedForm()).value()) {
+ var latestAvailableVersion = latestVersionInSecretStore(endpointCertificateMetadata);
- return endpointCertificateMetadata;
+ if (latestAvailableVersion.isPresent() && latestAvailableVersion.getAsInt() > endpointCertificateMetadata.version()) {
+ var refreshedCertificateMetadata = new EndpointCertificateMetadata(
+ endpointCertificateMetadata.keyName(),
+ endpointCertificateMetadata.certName(),
+ latestAvailableVersion.getAsInt()
+ );
+
+ if (verifyEndpointCertificate(refreshedCertificateMetadata, instance, zone, "Did not refresh, problems with refreshed certificate: "))
+ return Optional.of(refreshedCertificateMetadata);
+ }
+ }
+
+ // Only log warnings
+ verifyEndpointCertificate(endpointCertificateMetadata, instance, zone, "Problems while verifying certificate: ");
+
+ return Optional.of(endpointCertificateMetadata);
+ }
+
+ private OptionalInt latestVersionInSecretStore(EndpointCertificateMetadata originalCertificateMetadata) {
+ var certVersions = new HashSet<>(secretStore.listSecretVersions(originalCertificateMetadata.certName()));
+ var keyVersions = new HashSet<>(secretStore.listSecretVersions(originalCertificateMetadata.keyName()));
+
+ return Sets.intersection(certVersions, keyVersions).stream().mapToInt(Integer::intValue).max();
}
private EndpointCertificateMetadata provisionEndpointCertificate(Instance instance) {
List<ZoneId> directlyRoutedZones = zoneRegistry.zones().directlyRouted().zones().stream().map(ZoneApi::getId).collect(Collectors.toUnmodifiableList());
- ApplicationCertificate newCertificate = applicationCertificateProvider
+ EndpointCertificateMetadata provisionedCertificateMetadata = endpointCertificateProvider
.requestCaSignedCertificate(instance.id(), dnsNamesOf(instance.id(), directlyRoutedZones));
- EndpointCertificateMetadata provisionedCertificateMetadata = EndpointCertificateMetadataSerializer.fromTlsSecretsKeysString(newCertificate.secretsKeyNamePrefix());
curator.writeEndpointCertificateMetadata(instance.id(), provisionedCertificateMetadata);
return provisionedCertificateMetadata;
}
- private boolean verifyEndpointCertificate(EndpointCertificateMetadata endpointCertificateMetadata, Instance instance, ZoneId zone) {
+ private boolean verifyEndpointCertificate(EndpointCertificateMetadata endpointCertificateMetadata, Instance instance, ZoneId zone, String warningPrefix) {
try {
var pemEncodedEndpointCertificate = secretStore.getSecret(endpointCertificateMetadata.certName(), endpointCertificateMetadata.version());
- if (pemEncodedEndpointCertificate == null) return logWarning("Certificate not found in secret store");
+ if (pemEncodedEndpointCertificate == null) return logWarning(warningPrefix, "Certificate not found in secret store");
List<X509Certificate> x509CertificateList = X509CertificateUtils.certificateListFromPem(pemEncodedEndpointCertificate);
- if (x509CertificateList.isEmpty()) return logWarning("Empty certificate list");
+ if (x509CertificateList.isEmpty()) return logWarning(warningPrefix, "Empty certificate list");
if (x509CertificateList.size() < 2)
- return logWarning("Only a single certificate found in chain - intermediate certificates likely missing");
+ return logWarning(warningPrefix, "Only a single certificate found in chain - intermediate certificates likely missing");
Instant now = clock.instant();
Instant firstExpiry = Instant.MAX;
for (X509Certificate x509Certificate : x509CertificateList) {
Instant notBefore = x509Certificate.getNotBefore().toInstant();
Instant notAfter = x509Certificate.getNotAfter().toInstant();
- if (now.isBefore(notBefore)) return logWarning("Certificate is not yet valid");
- if (now.isAfter(notAfter)) return logWarning("Certificate has expired");
+ if (now.isBefore(notBefore)) return logWarning(warningPrefix, "Certificate is not yet valid");
+ if (now.isAfter(notAfter)) return logWarning(warningPrefix, "Certificate has expired");
if (notAfter.isBefore(firstExpiry)) firstExpiry = notAfter;
}
@@ -107,7 +140,7 @@ public class EndpointCertificateManager {
.map(SubjectAlternativeName::getValue).collect(Collectors.toSet());
if (!subjectAlternativeNames.equals(Set.copyOf(dnsNamesOf(instance.id(), List.of(zone)))))
- return logWarning("The list of SANs in the certificate does not match what we expect");
+ return logWarning(warningPrefix, "The list of SANs in the certificate does not match what we expect");
return true; // All good then, hopefully
} catch (Exception e) {
@@ -116,8 +149,8 @@ public class EndpointCertificateManager {
}
}
- private static boolean logWarning(String message) {
- log.log(LogLevel.WARNING, message);
+ private static boolean logWarning(String warningPrefix, String message) {
+ log.log(LogLevel.WARNING, warningPrefix + message);
return false;
}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java
index b730b63d426..abaf04610ea 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java
@@ -258,15 +258,13 @@ public class ApplicationSerializer {
}
private void toSlime(ApplicationVersion applicationVersion, Cursor object) {
- if (applicationVersion.buildNumber().isPresent() && applicationVersion.source().isPresent()) {
- object.setLong(applicationBuildNumberField, applicationVersion.buildNumber().getAsLong());
- toSlime(applicationVersion.source().get(), object.setObject(sourceRevisionField));
- applicationVersion.authorEmail().ifPresent(email -> object.setString(authorEmailField, email));
- applicationVersion.compileVersion().ifPresent(version -> object.setString(compileVersionField, version.toString()));
- applicationVersion.buildTime().ifPresent(time -> object.setLong(buildTimeField, time.toEpochMilli()));
- applicationVersion.sourceUrl().ifPresent(url -> object.setString(sourceUrlField, url));
- applicationVersion.commit().ifPresent(commit -> object.setString(commitField, commit));
- }
+ applicationVersion.buildNumber().ifPresent(number -> object.setLong(applicationBuildNumberField, number));
+ applicationVersion.source().ifPresent(source -> toSlime(source, object.setObject(sourceRevisionField)));
+ applicationVersion.authorEmail().ifPresent(email -> object.setString(authorEmailField, email));
+ applicationVersion.compileVersion().ifPresent(version -> object.setString(compileVersionField, version.toString()));
+ applicationVersion.buildTime().ifPresent(time -> object.setLong(buildTimeField, time.toEpochMilli()));
+ applicationVersion.sourceUrl().ifPresent(url -> object.setString(sourceUrlField, url));
+ applicationVersion.commit().ifPresent(commit -> object.setString(commitField, commit));
}
private void toSlime(SourceRevision sourceRevision, Cursor object) {
@@ -355,10 +353,8 @@ public class ApplicationSerializer {
}
private Optional<ApplicationVersion> latestVersionFromSlime(Inspector latestVersionObject) {
- if (latestVersionObject.valid())
- return Optional.of(applicationVersionFromSlime(latestVersionObject));
-
- return Optional.empty();
+ return Optional.of(applicationVersionFromSlime(latestVersionObject))
+ .filter(version -> ! version.isUnknown());
}
private List<Instance> instancesFromSlime(TenantAndApplicationId id, DeploymentSpec deploymentSpec, Inspector field) {
@@ -474,22 +470,16 @@ public class ApplicationSerializer {
private ApplicationVersion applicationVersionFromSlime(Inspector object) {
if ( ! object.valid()) return ApplicationVersion.unknown;
OptionalLong applicationBuildNumber = Serializers.optionalLong(object.field(applicationBuildNumberField));
- Optional<SourceRevision> sourceRevision = sourceRevisionFromSlime(object.field(sourceRevisionField));
- if (sourceRevision.isEmpty() || applicationBuildNumber.isEmpty()) {
+ if (applicationBuildNumber.isEmpty())
return ApplicationVersion.unknown;
- }
+
+ Optional<SourceRevision> sourceRevision = sourceRevisionFromSlime(object.field(sourceRevisionField));
Optional<String> authorEmail = Serializers.optionalString(object.field(authorEmailField));
Optional<Version> compileVersion = Serializers.optionalString(object.field(compileVersionField)).map(Version::fromString);
Optional<Instant> buildTime = Serializers.optionalInstant(object.field(buildTimeField));
Optional<String> sourceUrl = Serializers.optionalString(object.field(sourceUrlField));
Optional<String> commit = Serializers.optionalString(object.field(commitField));
- if (authorEmail.isEmpty())
- return ApplicationVersion.from(sourceRevision.get(), applicationBuildNumber.getAsLong());
-
- if (compileVersion.isEmpty() || buildTime.isEmpty())
- return ApplicationVersion.from(sourceRevision.get(), applicationBuildNumber.getAsLong(), authorEmail.get());
-
return new ApplicationVersion(sourceRevision, applicationBuildNumber, authorEmail, compileVersion, buildTime, sourceUrl, commit);
}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java
index 2c51a9bdc00..368621cac96 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java
@@ -15,7 +15,6 @@ import com.yahoo.vespa.config.SlimeUtils;
import com.yahoo.vespa.curator.Curator;
import com.yahoo.vespa.curator.Lock;
import com.yahoo.vespa.hosted.controller.Application;
-import com.yahoo.vespa.hosted.controller.api.integration.certificates.ApplicationCertificate;
import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificateMetadata;
import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType;
import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId;
@@ -84,7 +83,7 @@ public class CuratorDb {
private static final Path controllerRoot = root.append("controllers");
private static final Path routingPoliciesRoot = root.append("routingPolicies");
private static final Path zoneRoutingPoliciesRoot = root.append("zoneRoutingPolicies");
- private static final Path applicationCertificateRoot = root.append("applicationCertificates");
+ private static final Path endpointCertificateRoot = root.append("applicationCertificates");
private final StringSetSerializer stringSetSerializer = new StringSetSerializer();
private final NodeVersionSerializer nodeVersionSerializer = new NodeVersionSerializer();
@@ -516,11 +515,11 @@ public class CuratorDb {
// -------------- Application web certificates ----------------------------
public void writeEndpointCertificateMetadata(ApplicationId applicationId, EndpointCertificateMetadata endpointCertificateMetadata) {
- curator.set(applicationCertificatePath(applicationId), asJson(EndpointCertificateMetadataSerializer.toSlime(endpointCertificateMetadata)));
+ curator.set(endpointCertificatePath(applicationId), asJson(EndpointCertificateMetadataSerializer.toSlime(endpointCertificateMetadata)));
}
public Optional<EndpointCertificateMetadata> readEndpointCertificateMetadata(ApplicationId applicationId) {
- Optional<String> zkData = curator.getData(applicationCertificatePath(applicationId)).map(String::new);
+ Optional<String> zkData = curator.getData(endpointCertificatePath(applicationId)).map(String::new);
return zkData.map(EndpointCertificateMetadataSerializer::fromJsonOrTlsSecretsKeysString);
}
@@ -641,8 +640,8 @@ public class CuratorDb {
return controllerRoot.append(hostname);
}
- private static Path applicationCertificatePath(ApplicationId id) {
- return applicationCertificateRoot.append(id.serializedForm());
+ private static Path endpointCertificatePath(ApplicationId id) {
+ return endpointCertificateRoot.append(id.serializedForm());
}
}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java
index 03f9bcd84e6..9e674134347 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java
@@ -161,17 +161,19 @@ class RunSerializer {
if ( ! versionObject.field(buildField).valid())
return ApplicationVersion.unknown;
- SourceRevision revision = new SourceRevision(versionObject.field(repositoryField).asString(),
- versionObject.field(branchField).asString(),
- versionObject.field(commitField).asString());
long buildNumber = versionObject.field(buildField).asLong();
+ // TODO jonmv: Remove source revision
+ Optional<SourceRevision> source = Optional.of(new SourceRevision(versionObject.field(repositoryField).asString(),
+ versionObject.field(branchField).asString(),
+ versionObject.field(commitField).asString()))
+ .filter(revision -> ! revision.commit().isBlank() && ! revision.repository().isBlank() && ! revision.branch().isBlank());
Optional<String> authorEmail = Serializers.optionalString(versionObject.field(authorEmailField));
Optional<Version> compileVersion = Serializers.optionalString(versionObject.field(compileVersionField)).map(Version::fromString);
Optional<Instant> buildTime = Serializers.optionalInstant(versionObject.field(buildTimeField));
Optional<String> sourceUrl = Serializers.optionalString(versionObject.field(sourceUrlField));
Optional<String> commit = Serializers.optionalString(versionObject.field(commitField));
- return new ApplicationVersion(Optional.of(revision), OptionalLong.of(buildNumber), authorEmail,
+ return new ApplicationVersion(source, OptionalLong.of(buildNumber), authorEmail,
compileVersion, buildTime, sourceUrl, commit);
}
@@ -244,12 +246,11 @@ class RunSerializer {
private void toSlime(Version platformVersion, ApplicationVersion applicationVersion, Cursor versionsObject) {
versionsObject.setString(platformVersionField, platformVersion.toString());
- if ( ! applicationVersion.isUnknown()) {
- versionsObject.setString(repositoryField, applicationVersion.source().get().repository());
- versionsObject.setString(branchField, applicationVersion.source().get().branch());
- versionsObject.setString(commitField, applicationVersion.source().get().commit());
- versionsObject.setLong(buildField, applicationVersion.buildNumber().getAsLong());
- }
+ applicationVersion.buildNumber().ifPresent(number -> versionsObject.setLong(buildField, number));
+ // TODO jonmv: Remove source revision.
+ applicationVersion.source().map(SourceRevision::repository).ifPresent(repository -> versionsObject.setString(repositoryField, repository));
+ applicationVersion.source().map(SourceRevision::branch).ifPresent(branch -> versionsObject.setString(branchField, branch));
+ applicationVersion.source().map(SourceRevision::commit).ifPresent(commit -> versionsObject.setString(commitField, commit));
applicationVersion.authorEmail().ifPresent(email -> versionsObject.setString(authorEmailField, email));
applicationVersion.compileVersion().ifPresent(version -> versionsObject.setString(compileVersionField, version.toString()));
applicationVersion.buildTime().ifPresent(time -> versionsObject.setLong(buildTimeField, time.toEpochMilli()));
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java
index 7e4ad0d3384..6911dd974d2 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java
@@ -1331,10 +1331,17 @@ public class ApplicationApiHandler extends LoggingRequestHandler {
}
private HttpResponse service(String tenantName, String applicationName, String instanceName, String environment, String region, String serviceName, String restPath, HttpRequest request) {
- Map<?,?> result = controller.getServiceApiResponse(tenantName, applicationName, instanceName, environment, region, serviceName, restPath);
- ServiceApiResponse response = new ServiceApiResponse(ZoneId.from(environment, region),
- new ApplicationId.Builder().tenant(tenantName).applicationName(applicationName).instanceName(instanceName).build(),
- controller.zoneRegistry().getConfigServerApiUris(ZoneId.from(environment, region)),
+ DeploymentId deploymentId = new DeploymentId(ApplicationId.from(tenantName, applicationName, instanceName), ZoneId.from(environment, region));
+
+ if ("container-clustercontroller".equals((serviceName)) && restPath.contains("/status/")) {
+ String result = controller.serviceRegistry().configServer().getClusterControllerStatus(deploymentId, restPath);
+ return new HtmlResponse(result);
+ }
+
+ Map<?,?> result = controller.serviceRegistry().configServer().getServiceApiResponse(deploymentId, serviceName, restPath);
+ ServiceApiResponse response = new ServiceApiResponse(deploymentId.zoneId(),
+ deploymentId.applicationId(),
+ controller.zoneRegistry().getConfigServerApiUris(deploymentId.zoneId()),
request.getUri());
response.setResponse(result, serviceName, restPath);
return response;
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/HtmlResponse.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/HtmlResponse.java
new file mode 100644
index 00000000000..99884875a64
--- /dev/null
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/HtmlResponse.java
@@ -0,0 +1,30 @@
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.restapi.application;
+
+import com.yahoo.container.jdisc.HttpResponse;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.nio.charset.StandardCharsets;
+
+/**
+ * @author freva
+ */
+public class HtmlResponse extends HttpResponse {
+
+ private final String content;
+
+ public HtmlResponse(String content) {
+ super(200);
+ this.content = content;
+ }
+
+ @Override
+ public void render(OutputStream stream) throws IOException {
+ stream.write(content.getBytes(StandardCharsets.UTF_8));
+ }
+
+ @Override
+ public String getContentType() { return "text/html"; }
+
+}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java
index 9ea530c7886..73cdf28c366 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java
@@ -415,9 +415,11 @@ class JobControllerApiHandlerHelper {
versionObject.setLong("build", version.buildNumber().getAsLong());
Cursor sourceObject = versionObject.setObject("source");
- sourceObject.setString("gitRepository", version.source().get().repository());
- sourceObject.setString("gitBranch", version.source().get().branch());
- sourceObject.setString("gitCommit", version.source().get().commit());
+ version.source().ifPresent(source -> {
+ sourceObject.setString("gitRepository", source.repository());
+ sourceObject.setString("gitBranch", source.branch());
+ sourceObject.setString("gitCommit", source.commit());
+ });
version.sourceUrl().ifPresent(url -> versionObject.setString("sourceUrl", url));
version.commit().ifPresent(commit -> versionObject.setString("commit", commit));
}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/routing/RoutingApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/routing/RoutingApiHandler.java
index 8451f8d4f17..1815628a1ee 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/routing/RoutingApiHandler.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/routing/RoutingApiHandler.java
@@ -2,6 +2,7 @@
package com.yahoo.vespa.hosted.controller.restapi.routing;
import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.zone.RoutingMethod;
import com.yahoo.config.provision.zone.ZoneId;
import com.yahoo.container.jdisc.HttpRequest;
import com.yahoo.container.jdisc.HttpResponse;
@@ -93,13 +94,13 @@ public class RoutingApiHandler extends AuditLoggingRequestHandler {
var root = slime.setObject();
if (controller.zoneRegistry().zones().directlyRouted().ids().contains(zone)) {
var zonePolicy = controller.routingController().policies().get(zone);
- zoneStatusToSlime(root, zonePolicy.zone(), zonePolicy.globalRouting(), RoutingType.policy);
+ zoneStatusToSlime(root, zonePolicy.zone(), zonePolicy.globalRouting(), RoutingMethod.exclusive);
} else {
// Rotation status per zone only exposes in/out status, no agent or time of change.
var in = controller.serviceRegistry().configServer().getGlobalRotationStatus(zone);
var globalRouting = new GlobalRouting(in ? GlobalRouting.Status.in : GlobalRouting.Status.out,
GlobalRouting.Agent.operator, Instant.EPOCH);
- zoneStatusToSlime(root, zone, globalRouting, RoutingType.rotation);
+ zoneStatusToSlime(root, zone, globalRouting, RoutingMethod.shared);
}
return new SlimeJsonResponse(slime);
}
@@ -147,7 +148,7 @@ public class RoutingApiHandler extends AuditLoggingRequestHandler {
: GlobalRouting.Status.out;
deploymentStatusToSlime(deploymentsObject.addObject(), deployment,
new GlobalRouting(status, agent, changedAt),
- RoutingType.rotation);
+ RoutingMethod.shared);
}
}
@@ -165,8 +166,8 @@ public class RoutingApiHandler extends AuditLoggingRequestHandler {
return !instance.rotations().isEmpty() && instance.deployments().containsKey(zone);
}
- private static void zoneStatusToSlime(Cursor object, ZoneId zone, GlobalRouting globalRouting, RoutingType routingType) {
- object.setString("routingType", routingType.name());
+ private static void zoneStatusToSlime(Cursor object, ZoneId zone, GlobalRouting globalRouting, RoutingMethod method) {
+ object.setString("routingMethod", asString(method));
object.setString("environment", zone.environment().value());
object.setString("region", zone.region().value());
object.setString("status", asString(globalRouting.status()));
@@ -174,8 +175,8 @@ public class RoutingApiHandler extends AuditLoggingRequestHandler {
object.setLong("changedAt", globalRouting.changedAt().toEpochMilli());
}
- private static void deploymentStatusToSlime(Cursor object, DeploymentId deployment, GlobalRouting globalRouting, RoutingType routingType) {
- object.setString("routingType", routingType.name());
+ private static void deploymentStatusToSlime(Cursor object, DeploymentId deployment, GlobalRouting globalRouting, RoutingMethod method) {
+ object.setString("routingMethod", asString(method));
object.setString("instance", deployment.applicationId().serializedForm());
object.setString("environment", deployment.zoneId().environment().value());
object.setString("region", deployment.zoneId().region().value());
@@ -186,7 +187,7 @@ public class RoutingApiHandler extends AuditLoggingRequestHandler {
private static void deploymentStatusToSlime(Cursor object, RoutingPolicy policy) {
deploymentStatusToSlime(object, new DeploymentId(policy.id().owner(), policy.id().zone()),
- policy.status().globalRouting(), RoutingType.policy);
+ policy.status().globalRouting(), RoutingMethod.exclusive);
}
private DeploymentId deploymentFrom(Path path) {
@@ -219,12 +220,12 @@ public class RoutingApiHandler extends AuditLoggingRequestHandler {
}
}
- private enum RoutingType {
- /** Global routing is configured by use of an {@link com.yahoo.vespa.hosted.controller.application.AssignedRotation} */
- rotation,
-
- /** Global routing is configured by a {@link com.yahoo.vespa.hosted.controller.routing.RoutingPolicy} */
- policy,
+ private static String asString(RoutingMethod method) {
+ switch (method) {
+ case shared: return "shared";
+ case exclusive: return "exclusive";
+ default: return "unknonwn";
+ }
}
}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java
index 961a3674471..bda51bcc7b1 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java
@@ -146,7 +146,9 @@ public class RoutingPolicies {
routeEntry.getKey().endpointId(), controller.system());
controller.nameServiceForwarder().createAlias(RecordName.from(endpoint.dnsName()), targets, Priority.normal);
}
- staleTargets.forEach(t -> controller.nameServiceForwarder().removeRecords(Record.Type.ALIAS, t.asData(), Priority.normal));
+ staleTargets.forEach(t -> controller.nameServiceForwarder().removeRecords(Record.Type.ALIAS,
+ RecordData.fqdn(t.name().value()),
+ Priority.normal));
}
}
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java
index bd59fc45c96..50e567b2024 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java
@@ -1,4 +1,4 @@
-// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.hosted.controller;
import com.google.common.collect.Sets;
@@ -178,7 +178,7 @@ public class ControllerTest {
public void testGlobalRotations() {
// Setup
ControllerTester tester = this.tester.controllerTester();
- ZoneId zone = ZoneId.from(Environment.defaultEnvironment(), RegionName.defaultName());
+ ZoneId zone = ZoneId.from("prod", "us-west-1");
ApplicationId app = ApplicationId.from("tenant", "app1", "default");
DeploymentId deployment = new DeploymentId(app, zone);
tester.serviceRegistry().routingGeneratorMock().putEndpoints(deployment, List.of(
@@ -705,7 +705,7 @@ public class ControllerTest {
// Create app1
var context1 = tester.newDeploymentContext("tenant1", "app1", "default");
var prodZone = ZoneId.from("prod", "us-west-1");
- tester.controllerTester().zoneRegistry().setDirectlyRouted(ZoneApiMock.from(prodZone));
+ tester.controllerTester().zoneRegistry().exclusiveRoutingIn(ZoneApiMock.from(prodZone));
var applicationPackage = new ApplicationPackageBuilder().environment(prodZone.environment())
.region(prodZone.region())
.build();
@@ -722,7 +722,7 @@ public class ControllerTest {
(zone.environment() == Environment.prod ? "" : "." + zone.environment().value()) +
".vespa.oath.cloud")))
.collect(Collectors.toUnmodifiableList()),
- tester.controllerTester().serviceRegistry().applicationCertificateMock().dnsNamesOf(context1.instanceId()));
+ tester.controllerTester().serviceRegistry().endpointCertificateMock().dnsNamesOf(context1.instanceId()));
// Next deployment reuses certificate
context1.submit(applicationPackage).deploy();
@@ -732,8 +732,7 @@ public class ControllerTest {
var context2 = tester.newDeploymentContext("tenant1", "app2", "default");
var devZone = ZoneId.from("dev", "us-east-1");
- // Deploy app2, after "removing" direct routing everywhere
- tester.controllerTester().zoneRegistry().setDirectlyRouted();
+ // Deploy app2 in a zone with shared routing
tester.controller().applications().deploy(context2.instanceId(), devZone, Optional.of(applicationPackage), DeployOptions.none());
assertTrue("Application deployed and activated",
tester.configServer().application(context2.instanceId(), devZone).get().activated());
@@ -774,4 +773,19 @@ public class ControllerTest {
}
}
+ @Test
+ public void testDeployWithoutSourceRevision() {
+ var context = tester.newDeploymentContext();
+ var applicationPackage = new ApplicationPackageBuilder()
+ .upgradePolicy("default")
+ .environment(Environment.prod)
+ .region("us-west-1")
+ .build();
+
+ // Submit without source revision
+ context.submit(applicationPackage, Optional.empty())
+ .deploy();
+ assertEquals("Deployed application", 1, context.instance().deployments().size());
+ }
+
}
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java
index c90abe5ed5c..a918ac51f2d 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java
@@ -8,6 +8,7 @@ import com.yahoo.config.provision.AthenzService;
import com.yahoo.config.provision.ClusterSpec;
import com.yahoo.config.provision.Environment;
import com.yahoo.config.provision.HostName;
+import com.yahoo.config.provision.zone.RoutingMethod;
import com.yahoo.config.provision.zone.ZoneId;
import com.yahoo.security.KeyAlgorithm;
import com.yahoo.security.KeyUtils;
@@ -241,16 +242,16 @@ public class DeploymentContext {
/** Submit given application package for deployment */
public DeploymentContext submit(ApplicationPackage applicationPackage) {
- return submit(applicationPackage, defaultSourceRevision);
+ return submit(applicationPackage, Optional.of(defaultSourceRevision));
}
/** Submit given application package for deployment */
- public DeploymentContext submit(ApplicationPackage applicationPackage, SourceRevision sourceRevision) {
+ public DeploymentContext submit(ApplicationPackage applicationPackage, Optional<SourceRevision> sourceRevision) {
var projectId = tester.controller().applications()
.requireApplication(applicationId)
.projectId()
.orElse(1000); // These are really set through submission, so just pick one if it hasn't been set.
- lastSubmission = jobs.submit(applicationId, Optional.of(sourceRevision), Optional.of("a@b"), Optional.empty(),
+ lastSubmission = jobs.submit(applicationId, sourceRevision, Optional.of("a@b"), Optional.empty(),
Optional.empty(), projectId, applicationPackage, new byte[0]);
return this;
}
@@ -524,7 +525,7 @@ public class DeploymentContext {
/** Sets a single endpoint in the routing layer; this matches that required for the tester */
private DeploymentContext setEndpoints(ZoneId zone, boolean tester) {
- if (isDirectlyRouted(zone)) return this;
+ if (!supportsRoutingMethod(RoutingMethod.shared, zone)) return this;
var id = instanceId;
if (tester) {
id = testerId.id();
@@ -615,11 +616,12 @@ public class DeploymentContext {
/** Returns whether a load balancer is expected to be provisioned in given zone */
private boolean provisionLoadBalancerIn(ZoneId zone) {
- return !deferLoadBalancerProvisioning.contains(zone.environment()) && isDirectlyRouted(zone);
+ return !deferLoadBalancerProvisioning.contains(zone.environment()) &&
+ supportsRoutingMethod(RoutingMethod.exclusive, zone);
}
- private boolean isDirectlyRouted(ZoneId zone) {
- return tester.controller().zoneRegistry().zones().directlyRouted().ids().contains(zone);
+ private boolean supportsRoutingMethod(RoutingMethod method, ZoneId zone) {
+ return tester.controller().zoneRegistry().zones().routingMethod(method).ids().contains(zone);
}
private JobId jobId(JobType type) {
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java
index 830643b215a..cf5d341042c 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java
@@ -8,6 +8,7 @@ import com.yahoo.config.provision.AthenzDomain;
import com.yahoo.config.provision.HostName;
import com.yahoo.config.provision.NodeResources;
import com.yahoo.config.provision.SystemName;
+import com.yahoo.config.provision.zone.RoutingMethod;
import com.yahoo.config.provision.zone.ZoneId;
import com.yahoo.slime.ArrayTraverser;
import com.yahoo.slime.Inspector;
@@ -260,7 +261,7 @@ public class InternalStepRunnerTest {
public void alternativeEndpointsAreDetected() {
var systemTestZone = JobType.systemTest.zone(system());
var stagingZone = JobType.stagingTest.zone(system());
- tester.controllerTester().zoneRegistry().setDirectlyRouted(ZoneApiMock.from(systemTestZone), ZoneApiMock.from(stagingZone));
+ tester.controllerTester().zoneRegistry().exclusiveRoutingIn(ZoneApiMock.from(systemTestZone), ZoneApiMock.from(stagingZone));
app.newRun(JobType.systemTest);
tester.runner().run();;
tester.configServer().convergeServices(app.instanceId(), JobType.systemTest.zone(system()));
@@ -426,9 +427,12 @@ public class InternalStepRunnerTest {
@Test
public void certificateTimeoutAbortsJob() {
tester.controllerTester().zoneRegistry().setSystemName(SystemName.PublicCd);
- tester.controllerTester().zoneRegistry().setZones(ZoneApiMock.fromId("test.aws-us-east-1c"),
- ZoneApiMock.fromId("staging.aws-us-east-1c"),
- ZoneApiMock.fromId("prod.aws-us-east-1c"));
+ var zones = List.of(ZoneApiMock.fromId("test.aws-us-east-1c"),
+ ZoneApiMock.fromId("staging.aws-us-east-1c"),
+ ZoneApiMock.fromId("prod.aws-us-east-1c"));
+ tester.controllerTester().zoneRegistry()
+ .setZones(zones)
+ .setRoutingMethod(zones, RoutingMethod.shared);
tester.configServer().bootstrap(tester.controllerTester().zoneRegistry().zones().all().ids(), SystemApplication.values());
RunId id = app.startSystemTestTests();
@@ -452,6 +456,39 @@ public class InternalStepRunnerTest {
"3554970337.947845\t17491290-v6-1.ostk.bm2.prod.ne1.yahoo.com\t5480\tcontainer\tstderr\twarning\tjava.lang.NullPointerException\\n\\tat org.apache.felix.framework.BundleRevisionImpl.calculateContentPath(BundleRevisionImpl.java:438)\\n\\tat org.apache.felix.framework.BundleRevisionImpl.initializeContentPath(BundleRevisionImpl.java:371)";
@Test
+ public void generates_correct_tester_flavor() {
+ DeploymentSpec spec = DeploymentSpec.fromXml("<deployment version='1.0' athenz-domain='domain' athenz-service='service'>\n" +
+ " <instance id='first'>\n" +
+ " <test tester-flavor=\"d-6-16-100\" />\n" +
+ " <prod>\n" +
+ " <region active=\"true\">us-west-1</region>\n" +
+ " <test>us-west-1</test>\n" +
+ " </prod>\n" +
+ " </instance>\n" +
+ " <instance id='second'>\n" +
+ " <test />\n" +
+ " <staging />\n" +
+ " <prod tester-flavor=\"d-6-16-100\">\n" +
+ " <parallel>\n" +
+ " <region active=\"true\">us-east-3</region>\n" +
+ " <region active=\"true\">us-central-1</region>\n" +
+ " </parallel>\n" +
+ " <region active=\"true\">us-west-1</region>\n" +
+ " <test>us-west-1</test>\n" +
+ " </prod>\n" +
+ " </instance>\n" +
+ "</deployment>\n");
+
+ NodeResources firstResources = InternalStepRunner.testerResourcesFor(ZoneId.from("prod", "us-west-1"), spec.requireInstance("first"));
+ assertEquals(InternalStepRunner.DEFAULT_TESTER_RESOURCES, firstResources);
+
+ NodeResources secondResources = InternalStepRunner.testerResourcesFor(ZoneId.from("prod", "us-west-1"), spec.requireInstance("second"));
+ assertEquals(6, secondResources.vcpu(), 1e-9);
+ assertEquals(16, secondResources.memoryGb(), 1e-9);
+ assertEquals(100, secondResources.diskGb(), 1e-9);
+ }
+
+ @Test
public void generates_correct_services_xml_test() {
assertFile("test_runner_services.xml-cd", new String(InternalStepRunner.servicesXml(AthenzDomain.from("vespa.vespa.cd"),
true,
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/endpointcertificates/EndpointCertificateManagerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/endpointcertificates/EndpointCertificateManagerTest.java
index 808d9ca05c4..3f8e91dec58 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/endpointcertificates/EndpointCertificateManagerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/endpointcertificates/EndpointCertificateManagerTest.java
@@ -3,18 +3,31 @@ package com.yahoo.vespa.hosted.controller.endpointcertificates;
import com.yahoo.config.provision.ApplicationId;
import com.yahoo.config.provision.SystemName;
import com.yahoo.config.provision.zone.ZoneId;
+import com.yahoo.security.KeyAlgorithm;
+import com.yahoo.security.KeyUtils;
+import com.yahoo.security.SignatureAlgorithm;
+import com.yahoo.security.X509CertificateBuilder;
+import com.yahoo.security.X509CertificateUtils;
+import com.yahoo.vespa.flags.Flags;
+import com.yahoo.vespa.flags.InMemoryFlagSource;
import com.yahoo.vespa.hosted.controller.Instance;
-import com.yahoo.vespa.hosted.controller.api.integration.certificates.ApplicationCertificateMock;
+import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificateMock;
import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificateMetadata;
import com.yahoo.vespa.hosted.controller.integration.SecretStoreMock;
import com.yahoo.vespa.hosted.controller.integration.ZoneRegistryMock;
import com.yahoo.vespa.hosted.controller.persistence.MockCuratorDb;
+import org.junit.Before;
import org.junit.Test;
+import javax.security.auth.x500.X500Principal;
+import java.security.KeyPair;
+import java.security.cert.X509Certificate;
import java.time.Clock;
+import java.time.Instant;
+import java.time.temporal.ChronoUnit;
import java.util.Optional;
-import java.util.Set;
+import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
/**
@@ -22,19 +35,74 @@ import static org.junit.Assert.assertTrue;
*/
public class EndpointCertificateManagerTest {
+ private final SecretStoreMock secretStore = new SecretStoreMock();
+ private final ZoneRegistryMock zoneRegistryMock = new ZoneRegistryMock(SystemName.main);
+ private final MockCuratorDb mockCuratorDb = new MockCuratorDb();
+ private final EndpointCertificateMock endpointCertificateMock = new EndpointCertificateMock();
+ private final InMemoryFlagSource inMemoryFlagSource = new InMemoryFlagSource();
+ private final Clock clock = Clock.systemUTC();
+ private final EndpointCertificateManager endpointCertificateManager = new EndpointCertificateManager(zoneRegistryMock, mockCuratorDb, secretStore, endpointCertificateMock, clock, inMemoryFlagSource);
+
+ private static final KeyPair testKeyPair = KeyUtils.generateKeypair(KeyAlgorithm.EC, 192);
+ private static final X509Certificate testCertificate = X509CertificateBuilder
+ .fromKeypair(
+ testKeyPair,
+ new X500Principal("CN=test"),
+ Instant.now(), Instant.now().plus(5, ChronoUnit.MINUTES),
+ SignatureAlgorithm.SHA256_WITH_ECDSA,
+ X509CertificateBuilder.generateRandomSerialNumber())
+ .addSubjectAlternativeName("vt2ktgkqme5zlnp4tj4ttyor7fj3v7q5o.vespa.oath.cloud")
+ .addSubjectAlternativeName("default.default.global.vespa.oath.cloud")
+ .addSubjectAlternativeName("*.default.default.global.vespa.oath.cloud")
+ .addSubjectAlternativeName("default.default.us-east-1.test.vespa.oath.cloud")
+ .addSubjectAlternativeName("*.default.default.us-east-1.test.vespa.oath.cloud")
+ .build();
+
+ private final Instance testInstance = new Instance(ApplicationId.defaultId());
+ private final String testKeyName = "testKeyName";
+ private final String testCertName = "testCertName";
+ private ZoneId testZone;
+
+ @Before
+ public void setUp() {
+ zoneRegistryMock.exclusiveRoutingIn(zoneRegistryMock.zones().all().zones());
+ testZone = zoneRegistryMock.zones().directlyRouted().zones().stream().findFirst().get().getId();
+ }
+
+ @Test
+ public void provisions_new_certificate() {
+ Optional<EndpointCertificateMetadata> endpointCertificateMetadata = endpointCertificateManager.getEndpointCertificateMetadata(testInstance, testZone);
+ assertTrue(endpointCertificateMetadata.isPresent());
+ assertTrue(endpointCertificateMetadata.get().keyName().matches("vespa.tls.default.default.*-key"));
+ assertTrue(endpointCertificateMetadata.get().certName().matches("vespa.tls.default.default.*-cert"));
+ assertEquals(0, endpointCertificateMetadata.get().version());
+ }
+
@Test
- public void getEndpointCertificate() {
- SecretStoreMock secretStore = new SecretStoreMock();
- ZoneRegistryMock zoneRegistryMock = new ZoneRegistryMock(SystemName.main);
- zoneRegistryMock.setDirectlyRouted(Set.copyOf(zoneRegistryMock.zones().all().zones()));
- MockCuratorDb mockCuratorDb = new MockCuratorDb();
- ApplicationCertificateMock applicationCertificateMock = new ApplicationCertificateMock();
- Clock clock = Clock.systemUTC();
- EndpointCertificateManager endpointCertificateManager = new EndpointCertificateManager(zoneRegistryMock, mockCuratorDb, secretStore, applicationCertificateMock, clock);
- ZoneId id = zoneRegistryMock.zones().directlyRouted().zones().stream().findFirst().get().getId();
- Instance instance = new Instance(ApplicationId.defaultId());
- Optional<EndpointCertificateMetadata> endpointCertificateMetadata = endpointCertificateManager.getEndpointCertificateMetadata(instance, id);
+ public void reuses_stored_certificate_metadata() {
+ mockCuratorDb.writeEndpointCertificateMetadata(testInstance.id(), new EndpointCertificateMetadata(testKeyName, testCertName, 7));
+ Optional<EndpointCertificateMetadata> endpointCertificateMetadata = endpointCertificateManager.getEndpointCertificateMetadata(testInstance, testZone);
+ assertTrue(endpointCertificateMetadata.isPresent());
+ assertEquals(testKeyName, endpointCertificateMetadata.get().keyName());
+ assertEquals(testCertName, endpointCertificateMetadata.get().certName());
+ assertEquals(7, endpointCertificateMetadata.get().version());
+ }
+
+ @Test
+ public void uses_refreshed_certificate_when_available_and_valid() {
+ inMemoryFlagSource.withBooleanFlag(Flags.USE_REFRESHED_ENDPOINT_CERTIFICATE.id(), true);
+
+ secretStore.setSecret(testKeyName, "secret-key", 7);
+ secretStore.setSecret(testCertName, "cert", 7);
+ secretStore.setSecret(testKeyName, KeyUtils.toPem(testKeyPair.getPrivate()), 8);
+ secretStore.setSecret(testKeyName, KeyUtils.toPem(testKeyPair.getPrivate()), 9);
+ secretStore.setSecret(testCertName, X509CertificateUtils.toPem(testCertificate)+X509CertificateUtils.toPem(testCertificate), 8);
+ mockCuratorDb.writeEndpointCertificateMetadata(testInstance.id(), new EndpointCertificateMetadata(testKeyName, testCertName, 7));
+ Optional<EndpointCertificateMetadata> endpointCertificateMetadata = endpointCertificateManager.getEndpointCertificateMetadata(testInstance, testZone);
assertTrue(endpointCertificateMetadata.isPresent());
+ assertEquals(testKeyName, endpointCertificateMetadata.get().keyName());
+ assertEquals(testCertName, endpointCertificateMetadata.get().certName());
+ assertEquals(8, endpointCertificateMetadata.get().version());
}
}
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java
index eb8af62a0e6..5d93881129f 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java
@@ -18,6 +18,7 @@ import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId;
import com.yahoo.vespa.hosted.controller.api.identifiers.Hostname;
import com.yahoo.vespa.hosted.controller.api.identifiers.Identifier;
import com.yahoo.vespa.hosted.controller.api.identifiers.TenantId;
+import com.yahoo.vespa.hosted.controller.api.integration.LogEntry;
import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificateMetadata;
import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServer;
import com.yahoo.vespa.hosted.controller.api.integration.configserver.ContainerEndpoint;
@@ -282,6 +283,21 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer
return TesterCloud.Status.SUCCESS;
}
+ @Override
+ public String startTests(DeploymentId deployment, TesterCloud.Suite suite, byte[] config) {
+ return "Tests started";
+ }
+
+ @Override
+ public List<LogEntry> getTesterLog(DeploymentId deployment, long after) {
+ return List.of();
+ }
+
+ @Override
+ public boolean isTesterReady(DeploymentId deployment) {
+ return false;
+ }
+
/** Add any of given loadBalancers that do not already exist to the load balancers in zone */
public void putLoadBalancers(ZoneId zone, List<LoadBalancer> loadBalancers) {
this.loadBalancers.putIfAbsent(zone, new LinkedHashSet<>());
@@ -404,8 +420,7 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer
// Returns a canned example response
@Override
- public Map<?,?> getServiceApiResponse(String tenantName, String applicationName, String instanceName,
- String environment, String region, String serviceName, String restPath) {
+ public Map<?,?> getServiceApiResponse(DeploymentId deployment, String serviceName, String restPath) {
Map<String,List<?>> root = new HashMap<>();
List<Map<?,?>> resources = new ArrayList<>();
Map<String,String> resource = new HashMap<>();
@@ -416,6 +431,11 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer
}
@Override
+ public String getClusterControllerStatus(DeploymentId deployment, String restPath) {
+ return "<h1>OK</h1>";
+ }
+
+ @Override
public void setGlobalRotationStatus(DeploymentId deployment, String upstreamName, EndpointStatus status) {
endpoints.put(upstreamName, status);
}
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/SecretStoreMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/SecretStoreMock.java
index a14b7b82d67..c088965c2ad 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/SecretStoreMock.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/SecretStoreMock.java
@@ -5,6 +5,7 @@ import com.yahoo.component.AbstractComponent;
import com.yahoo.container.jdisc.secretstore.SecretStore;
import java.util.HashMap;
+import java.util.List;
import java.util.Map;
import java.util.TreeMap;
@@ -45,4 +46,8 @@ public class SecretStoreMock extends AbstractComponent implements SecretStore {
return secrets.getOrDefault(key, new TreeMap<>()).get(version);
}
+ @Override
+ public List<Integer> listSecretVersions(String key) {
+ return List.copyOf(secrets.getOrDefault(key, new TreeMap<>()).keySet());
+ }
}
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ServiceRegistryMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ServiceRegistryMock.java
index bd82807342e..323b86be1d3 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ServiceRegistryMock.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ServiceRegistryMock.java
@@ -1,4 +1,4 @@
-// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.hosted.controller.integration;
import com.google.inject.Inject;
@@ -10,7 +10,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.ServiceRegistry;
import com.yahoo.vespa.hosted.controller.api.integration.aws.MockAwsEventFetcher;
import com.yahoo.vespa.hosted.controller.api.integration.aws.MockResourceTagger;
import com.yahoo.vespa.hosted.controller.api.integration.aws.ResourceTagger;
-import com.yahoo.vespa.hosted.controller.api.integration.certificates.ApplicationCertificateMock;
+import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificateMock;
import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServer;
import com.yahoo.vespa.hosted.controller.api.integration.dns.MemoryNameService;
import com.yahoo.vespa.hosted.controller.api.integration.entity.MemoryEntityService;
@@ -42,9 +42,9 @@ public class ServiceRegistryMock extends AbstractComponent implements ServiceReg
private final ConfigServerMock configServerMock;
private final MemoryNameService memoryNameService = new MemoryNameService();
private final MemoryGlobalRoutingService memoryGlobalRoutingService = new MemoryGlobalRoutingService();
- private final RoutingGeneratorMock routingGeneratorMock = new RoutingGeneratorMock(RoutingGeneratorMock.TEST_ENDPOINTS);
+ private final RoutingGeneratorMock routingGeneratorMock;
private final MockMailer mockMailer = new MockMailer();
- private final ApplicationCertificateMock applicationCertificateMock = new ApplicationCertificateMock();
+ private final EndpointCertificateMock endpointCertificateMock = new EndpointCertificateMock();
private final MockMeteringClient mockMeteringClient = new MockMeteringClient();
private final MockContactRetriever mockContactRetriever = new MockContactRetriever();
private final MockIssueHandler mockIssueHandler = new MockIssueHandler();
@@ -64,6 +64,7 @@ public class ServiceRegistryMock extends AbstractComponent implements ServiceReg
public ServiceRegistryMock(SystemName system) {
this.zoneRegistryMock = new ZoneRegistryMock(system);
this.configServerMock = new ConfigServerMock(zoneRegistryMock);
+ this.routingGeneratorMock = new RoutingGeneratorMock(RoutingGeneratorMock.TEST_ENDPOINTS, zoneRegistryMock);
}
@Inject
@@ -101,8 +102,8 @@ public class ServiceRegistryMock extends AbstractComponent implements ServiceReg
}
@Override
- public ApplicationCertificateMock applicationCertificateProvider() {
- return applicationCertificateMock;
+ public EndpointCertificateMock endpointCertificateProvider() {
+ return endpointCertificateMock;
}
@Override
@@ -212,8 +213,8 @@ public class ServiceRegistryMock extends AbstractComponent implements ServiceReg
return artifactRepositoryMock;
}
- public ApplicationCertificateMock applicationCertificateMock() {
- return applicationCertificateMock;
+ public EndpointCertificateMock endpointCertificateMock() {
+ return endpointCertificateMock;
}
}
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneFilterMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneFilterMock.java
index 4cd50625ca3..764c2003c46 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneFilterMock.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneFilterMock.java
@@ -1,9 +1,10 @@
-// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.hosted.controller.integration;
import com.yahoo.config.provision.CloudName;
import com.yahoo.config.provision.Environment;
import com.yahoo.config.provision.RegionName;
+import com.yahoo.config.provision.zone.RoutingMethod;
import com.yahoo.config.provision.zone.ZoneApi;
import com.yahoo.config.provision.zone.ZoneFilter;
import com.yahoo.config.provision.zone.ZoneId;
@@ -11,6 +12,7 @@ import com.yahoo.config.provision.zone.ZoneList;
import java.util.Collection;
import java.util.List;
+import java.util.Map;
import java.util.Set;
import java.util.function.Predicate;
import java.util.stream.Collectors;
@@ -23,22 +25,22 @@ import java.util.stream.Collectors;
public class ZoneFilterMock implements ZoneList {
private final List<ZoneApi> zones;
- private final Set<ZoneApi> directlyRouted;
+ private final Map<ZoneApi, Set<RoutingMethod>> zoneRoutingMethods;
private final boolean negate;
- private ZoneFilterMock(List<ZoneApi> zones, Set<ZoneApi> directlyRouted, boolean negate) {
+ private ZoneFilterMock(List<ZoneApi> zones, Map<ZoneApi, Set<RoutingMethod>> zoneRoutingMethods, boolean negate) {
this.zones = zones;
- this.directlyRouted = directlyRouted;
+ this.zoneRoutingMethods = zoneRoutingMethods;
this.negate = negate;
}
- public static ZoneFilter from(Collection<ZoneApi> zones, Set<ZoneApi> directlyRouted) {
- return new ZoneFilterMock(List.copyOf(zones), Set.copyOf(directlyRouted), false);
+ public static ZoneFilter from(Collection<? extends ZoneApi> zones, Map<ZoneApi, Set<RoutingMethod>> routingMethods) {
+ return new ZoneFilterMock(List.copyOf(zones), Map.copyOf(routingMethods), false);
}
@Override
public ZoneList not() {
- return new ZoneFilterMock(zones, directlyRouted, ! negate);
+ return new ZoneFilterMock(zones, zoneRoutingMethods, ! negate);
}
@Override
@@ -53,7 +55,12 @@ public class ZoneFilterMock implements ZoneList {
@Override
public ZoneList directlyRouted() {
- return filter(directlyRouted::contains);
+ return routingMethod(RoutingMethod.exclusive);
+ }
+
+ @Override
+ public ZoneList routingMethod(RoutingMethod method) {
+ return filter(zone -> zoneRoutingMethods.getOrDefault(zone, Set.of()).contains(method));
}
@Override
@@ -93,7 +100,7 @@ public class ZoneFilterMock implements ZoneList {
condition.negate().test(zone) :
condition.test(zone))
.collect(Collectors.toList()),
- directlyRouted, false);
+ zoneRoutingMethods, false);
}
}
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneRegistryMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneRegistryMock.java
index 3cb7a13dd54..a5e054d6068 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneRegistryMock.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneRegistryMock.java
@@ -1,4 +1,4 @@
-// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.hosted.controller.integration;
import com.yahoo.component.AbstractComponent;
@@ -9,6 +9,7 @@ import com.yahoo.config.provision.Environment;
import com.yahoo.config.provision.NodeType;
import com.yahoo.config.provision.RegionName;
import com.yahoo.config.provision.SystemName;
+import com.yahoo.config.provision.zone.RoutingMethod;
import com.yahoo.config.provision.zone.UpgradePolicy;
import com.yahoo.config.provision.zone.ZoneApi;
import com.yahoo.config.provision.zone.ZoneFilter;
@@ -34,11 +35,12 @@ public class ZoneRegistryMock extends AbstractComponent implements ZoneRegistry
private final Map<ZoneId, Duration> deploymentTimeToLive = new HashMap<>();
private final Map<Environment, RegionName> defaultRegionForEnvironment = new HashMap<>();
- private List<ZoneApi> zones = List.of();
+ private final Map<CloudName, UpgradePolicy> osUpgradePolicies = new HashMap<>();
+ private final Map<ZoneApi, Set<RoutingMethod>> zoneRoutingMethods = new HashMap<>();
+
+ private List<? extends ZoneApi> zones;
private SystemName system;
private UpgradePolicy upgradePolicy = null;
- private Map<CloudName, UpgradePolicy> osUpgradePolicies = new HashMap<>();
- private Set<ZoneApi> directlyRouted = Set.of();
/**
* This sets the default list of zones contained in this. If your test need a particular set of zones, use
@@ -46,20 +48,21 @@ public class ZoneRegistryMock extends AbstractComponent implements ZoneRegistry
*/
public ZoneRegistryMock(SystemName system) {
this.system = system;
- setZones(List.of(
- ZoneApiMock.fromId("test.us-east-1"),
- ZoneApiMock.fromId("staging.us-east-3"),
- ZoneApiMock.fromId("dev.us-east-1"),
- ZoneApiMock.fromId("dev.aws-us-east-2a"),
- ZoneApiMock.fromId("perf.us-east-3"),
- ZoneApiMock.fromId("prod.aws-us-east-1a"),
- ZoneApiMock.fromId("prod.ap-northeast-1"),
- ZoneApiMock.fromId("prod.ap-northeast-2"),
- ZoneApiMock.fromId("prod.ap-southeast-1"),
- ZoneApiMock.fromId("prod.us-east-3"),
- ZoneApiMock.fromId("prod.us-west-1"),
- ZoneApiMock.fromId("prod.us-central-1"),
- ZoneApiMock.fromId("prod.eu-west-1")));
+ this.zones = List.of(ZoneApiMock.fromId("test.us-east-1"),
+ ZoneApiMock.fromId("staging.us-east-3"),
+ ZoneApiMock.fromId("dev.us-east-1"),
+ ZoneApiMock.fromId("dev.aws-us-east-2a"),
+ ZoneApiMock.fromId("perf.us-east-3"),
+ ZoneApiMock.fromId("prod.aws-us-east-1a"),
+ ZoneApiMock.fromId("prod.ap-northeast-1"),
+ ZoneApiMock.fromId("prod.ap-northeast-2"),
+ ZoneApiMock.fromId("prod.ap-southeast-1"),
+ ZoneApiMock.fromId("prod.us-east-3"),
+ ZoneApiMock.fromId("prod.us-west-1"),
+ ZoneApiMock.fromId("prod.us-central-1"),
+ ZoneApiMock.fromId("prod.eu-west-1"));
+ // All zones use a shared routing method by default
+ setRoutingMethod(this.zones, RoutingMethod.shared);
}
public ZoneRegistryMock setDeploymentTimeToLive(ZoneId zone, Duration duration) {
@@ -72,7 +75,7 @@ public class ZoneRegistryMock extends AbstractComponent implements ZoneRegistry
return this;
}
- public ZoneRegistryMock setZones(List<ZoneApi> zones) {
+ public ZoneRegistryMock setZones(List<? extends ZoneApi> zones) {
this.zones = zones;
return this;
}
@@ -96,12 +99,25 @@ public class ZoneRegistryMock extends AbstractComponent implements ZoneRegistry
return this;
}
- public ZoneRegistryMock setDirectlyRouted(ZoneApi... zones) {
- return setDirectlyRouted(Set.of(zones));
+ public ZoneRegistryMock exclusiveRoutingIn(ZoneApi... zones) {
+ return exclusiveRoutingIn(List.of(zones));
+ }
+
+ public ZoneRegistryMock exclusiveRoutingIn(List<? extends ZoneApi> zones) {
+ return setRoutingMethod(zones, RoutingMethod.exclusive);
+ }
+
+ public ZoneRegistryMock setRoutingMethod(ZoneApi zone, RoutingMethod... routingMethods) {
+ return setRoutingMethod(zone, Set.of(routingMethods));
+ }
+
+ public ZoneRegistryMock setRoutingMethod(List<? extends ZoneApi> zones, RoutingMethod... routingMethods) {
+ zones.forEach(zone -> setRoutingMethod(zone, Set.of(routingMethods)));
+ return this;
}
- public ZoneRegistryMock setDirectlyRouted(Set<ZoneApi> zones) {
- directlyRouted = zones;
+ public ZoneRegistryMock setRoutingMethod(ZoneApi zone, Set<RoutingMethod> routingMethods) {
+ this.zoneRoutingMethods.put(zone, Set.copyOf(routingMethods));
return this;
}
@@ -112,7 +128,7 @@ public class ZoneRegistryMock extends AbstractComponent implements ZoneRegistry
@Override
public ZoneFilter zones() {
- return ZoneFilterMock.from(zones, directlyRouted);
+ return ZoneFilterMock.from(zones, zoneRoutingMethods);
}
@Override
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OutstandingChangeDeployerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OutstandingChangeDeployerTest.java
index 442a2fd1853..ceebbb8254f 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OutstandingChangeDeployerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OutstandingChangeDeployerTest.java
@@ -17,6 +17,7 @@ import org.junit.Test;
import java.time.Duration;
import java.util.List;
+import java.util.Optional;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
@@ -47,7 +48,7 @@ public class OutstandingChangeDeployerTest {
assertFalse(app1.deploymentStatus().outstandingChange(app1.instance().name()).hasTargets());
assertEquals(1, app1.application().latestVersion().get().buildNumber().getAsLong());
- app1.submit(applicationPackage, new SourceRevision("repository1", "master", "cafed00d"));
+ app1.submit(applicationPackage, Optional.of(new SourceRevision("repository1", "master", "cafed00d")));
assertTrue(app1.deploymentStatus().outstandingChange(app1.instance().name()).hasTargets());
assertEquals("1.0.2-cafed00d", app1.deploymentStatus().outstandingChange(app1.instance().name()).application().get().id());
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java
index b7adc3064fa..da770c9c023 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java
@@ -65,6 +65,8 @@ public class ControllerContainerTest {
" <item key=\"rotation-id-5\">rotation-fqdn-5</item>\n" +
" </rotations>\n" +
" </config>\n" +
+ " " +
+ "<accesslog type='disabled'/>\n" +
" <component id='com.yahoo.vespa.flags.InMemoryFlagSource'/>\n" +
" <component id='com.yahoo.vespa.configserver.flags.db.FlagsDbImpl'/>\n" +
" <component id='com.yahoo.vespa.curator.mock.MockCurator'/>\n" +
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/RoutingApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/RoutingApiTest.java
index e5670139b0f..635adc73d1d 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/RoutingApiTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/RoutingApiTest.java
@@ -2,6 +2,7 @@
package com.yahoo.vespa.hosted.controller.restapi.routing;
import com.yahoo.application.container.handler.Request;
+import com.yahoo.config.provision.zone.RoutingMethod;
import com.yahoo.config.provision.zone.ZoneId;
import com.yahoo.vespa.hosted.controller.ControllerTester;
import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder;
@@ -34,14 +35,13 @@ public class RoutingApiTest extends ControllerContainerTest {
}
@Test
- public void policy_based_routing() {
+ public void exclusive_routing() {
var context = deploymentTester.newDeploymentContext();
// Zones support direct routing
var westZone = ZoneId.from("prod", "us-west-1");
var eastZone = ZoneId.from("prod", "us-east-3");
- deploymentTester.controllerTester().zoneRegistry().setDirectlyRouted(ZoneApiMock.from(westZone),
- ZoneApiMock.from(eastZone));
-
+ deploymentTester.controllerTester().zoneRegistry().exclusiveRoutingIn(ZoneApiMock.from(westZone),
+ ZoneApiMock.from(eastZone));
// Deploy application
var applicationPackage = new ApplicationPackageBuilder()
.region(westZone.region())
@@ -96,7 +96,7 @@ public class RoutingApiTest extends ControllerContainerTest {
}
@Test
- public void rotation_based_routing() {
+ public void shared_routing() {
// Deploy application
var context = deploymentTester.newDeploymentContext();
var westZone = ZoneId.from("prod", "us-west-1");
@@ -156,10 +156,16 @@ public class RoutingApiTest extends ControllerContainerTest {
// TODO(mpolden): Remove this once a zone supports either of routing policy and rotation
@Test
public void mixed_routing() {
- // Deploy application
- var context = deploymentTester.newDeploymentContext();
var westZone = ZoneId.from("prod", "us-west-1");
var eastZone = ZoneId.from("prod", "us-east-3");
+
+ // One zone supports multiple routing methods
+ deploymentTester.controllerTester().zoneRegistry().setRoutingMethod(ZoneApiMock.from(westZone),
+ RoutingMethod.shared,
+ RoutingMethod.exclusive);
+
+ // Deploy application
+ var context = deploymentTester.newDeploymentContext();
var applicationPackage = new ApplicationPackageBuilder()
.region(westZone.region())
.region(eastZone.region())
@@ -168,7 +174,6 @@ public class RoutingApiTest extends ControllerContainerTest {
context.submit(applicationPackage).deploy();
// Assign policy in one zone
- deploymentTester.controllerTester().zoneRegistry().setDirectlyRouted(ZoneApiMock.from(westZone));
context.addRoutingPolicy(westZone, true);
// GET status with both policy and rotation assigned
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/multi-status-in.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/multi-status-in.json
index b3f3e90a9cd..1c23c6bb569 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/multi-status-in.json
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/multi-status-in.json
@@ -1,7 +1,7 @@
{
"deployments": [
{
- "routingType": "rotation",
+ "routingMethod": "shared",
"instance": "tenant:application:default",
"environment": "prod",
"region": "us-west-1",
@@ -10,7 +10,7 @@
"changedAt": "(ignore)"
},
{
- "routingType": "policy",
+ "routingMethod": "exclusive",
"instance": "tenant:application:default",
"environment": "prod",
"region": "us-west-1",
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/multi-status-initial.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/multi-status-initial.json
index cdf3e093f97..eea78c1b963 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/multi-status-initial.json
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/multi-status-initial.json
@@ -1,7 +1,7 @@
{
"deployments": [
{
- "routingType": "rotation",
+ "routingMethod": "shared",
"instance": "tenant:application:default",
"environment": "prod",
"region": "us-west-1",
@@ -10,7 +10,7 @@
"changedAt": "(ignore)"
},
{
- "routingType": "policy",
+ "routingMethod": "exclusive",
"instance": "tenant:application:default",
"environment": "prod",
"region": "us-west-1",
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/multi-status-out.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/multi-status-out.json
index 373d7076ffc..6cb90bdb673 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/multi-status-out.json
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/multi-status-out.json
@@ -1,7 +1,7 @@
{
"deployments": [
{
- "routingType": "rotation",
+ "routingMethod": "shared",
"instance": "tenant:application:default",
"environment": "prod",
"region": "us-west-1",
@@ -10,7 +10,7 @@
"changedAt": "(ignore)"
},
{
- "routingType": "policy",
+ "routingMethod": "exclusive",
"instance": "tenant:application:default",
"environment": "prod",
"region": "us-west-1",
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/policy/deployment-status-in.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/policy/deployment-status-in.json
index 74c8173d132..59519c33d06 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/policy/deployment-status-in.json
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/policy/deployment-status-in.json
@@ -1,7 +1,7 @@
{
"deployments": [
{
- "routingType": "policy",
+ "routingMethod": "exclusive",
"instance": "tenant:application:default",
"environment": "prod",
"region": "us-west-1",
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/policy/deployment-status-initial.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/policy/deployment-status-initial.json
index 444b6a825ea..e95d9bcdc42 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/policy/deployment-status-initial.json
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/policy/deployment-status-initial.json
@@ -1,7 +1,7 @@
{
"deployments": [
{
- "routingType": "policy",
+ "routingMethod": "exclusive",
"instance": "tenant:application:default",
"environment": "prod",
"region": "us-west-1",
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/policy/deployment-status-out.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/policy/deployment-status-out.json
index 3f5353505df..49b85775e63 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/policy/deployment-status-out.json
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/policy/deployment-status-out.json
@@ -1,7 +1,7 @@
{
"deployments": [
{
- "routingType": "policy",
+ "routingMethod": "exclusive",
"instance": "tenant:application:default",
"environment": "prod",
"region": "us-west-1",
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/policy/zone-status-in.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/policy/zone-status-in.json
index 376b6c9c902..abf0a46ae3e 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/policy/zone-status-in.json
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/policy/zone-status-in.json
@@ -1,5 +1,5 @@
{
- "routingType": "policy",
+ "routingMethod": "exclusive",
"environment": "prod",
"region": "us-west-1",
"status": "in",
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/policy/zone-status-initial.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/policy/zone-status-initial.json
index 482fd920070..8328e1ffab1 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/policy/zone-status-initial.json
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/policy/zone-status-initial.json
@@ -1,5 +1,5 @@
{
- "routingType": "policy",
+ "routingMethod": "exclusive",
"environment": "prod",
"region": "us-west-1",
"status": "in",
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/policy/zone-status-out.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/policy/zone-status-out.json
index 5ab261067bf..d86ca2d56e6 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/policy/zone-status-out.json
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/policy/zone-status-out.json
@@ -1,5 +1,5 @@
{
- "routingType": "policy",
+ "routingMethod": "exclusive",
"environment": "prod",
"region": "us-west-1",
"status": "out",
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/rotation/deployment-status-in.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/rotation/deployment-status-in.json
index fd30a27fba5..5b15b72752c 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/rotation/deployment-status-in.json
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/rotation/deployment-status-in.json
@@ -1,7 +1,7 @@
{
"deployments": [
{
- "routingType": "rotation",
+ "routingMethod": "shared",
"instance": "tenant:application:default",
"environment": "prod",
"region": "us-west-1",
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/rotation/deployment-status-initial.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/rotation/deployment-status-initial.json
index 59ea77cf7ae..90b2317c1b3 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/rotation/deployment-status-initial.json
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/rotation/deployment-status-initial.json
@@ -1,7 +1,7 @@
{
"deployments": [
{
- "routingType": "rotation",
+ "routingMethod": "shared",
"instance": "tenant:application:default",
"environment": "prod",
"region": "us-west-1",
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/rotation/deployment-status-out.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/rotation/deployment-status-out.json
index 0240079f154..85e345c01d0 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/rotation/deployment-status-out.json
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/rotation/deployment-status-out.json
@@ -1,7 +1,7 @@
{
"deployments": [
{
- "routingType": "rotation",
+ "routingMethod": "shared",
"instance": "tenant:application:default",
"environment": "prod",
"region": "us-west-1",
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/rotation/zone-status-in.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/rotation/zone-status-in.json
index a883fd3f342..eb06e9ee11d 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/rotation/zone-status-in.json
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/rotation/zone-status-in.json
@@ -1,5 +1,5 @@
{
- "routingType": "rotation",
+ "routingMethod": "shared",
"environment": "prod",
"region": "us-west-1",
"status": "in",
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/rotation/zone-status-initial.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/rotation/zone-status-initial.json
index a883fd3f342..eb06e9ee11d 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/rotation/zone-status-initial.json
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/rotation/zone-status-initial.json
@@ -1,5 +1,5 @@
{
- "routingType": "rotation",
+ "routingMethod": "shared",
"environment": "prod",
"region": "us-west-1",
"status": "in",
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/rotation/zone-status-out.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/rotation/zone-status-out.json
index 18803f56de3..440b80bc4d0 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/rotation/zone-status-out.json
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/rotation/zone-status-out.json
@@ -1,5 +1,5 @@
{
- "routingType": "rotation",
+ "routingMethod": "shared",
"environment": "prod",
"region": "us-west-1",
"status": "out",
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java
index 2b196b9127f..d81b836cad8 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java
@@ -266,7 +266,7 @@ public class RoutingPoliciesTest {
var zoneApi = ZoneApiMock.from(zone.environment(), zone.region());
tester.controllerTester().serviceRegistry().zoneRegistry()
.setZones(zoneApi)
- .setDirectlyRouted(zoneApi);
+ .exclusiveRoutingIn(zoneApi);
tester.provisionLoadBalancers(1, context.instanceId(), zone);
// Deploy to dev
@@ -289,7 +289,7 @@ public class RoutingPoliciesTest {
var zoneApi = ZoneApiMock.from(zone.environment(), zone.region());
tester.controllerTester().serviceRegistry().zoneRegistry()
.setZones(zoneApi)
- .setDirectlyRouted(zoneApi);
+ .exclusiveRoutingIn(zoneApi);
var prodRecords = Set.of("app1.tenant1.us-central-1.vespa.oath.cloud", "app1.tenant1.us-west-1.vespa.oath.cloud");
assertEquals(prodRecords, tester.recordNames());
@@ -523,7 +523,7 @@ public class RoutingPoliciesTest {
public RoutingPoliciesTester(DeploymentTester tester) {
this.tester = tester;
// Make all zones directly routed
- tester.controllerTester().zoneRegistry().setDirectlyRouted(Set.copyOf(tester.controllerTester().zoneRegistry().zones().all().zones()));
+ tester.controllerTester().zoneRegistry().exclusiveRoutingIn(tester.controllerTester().zoneRegistry().zones().all().zones());
}
private void provisionLoadBalancers(int clustersPerZone, ApplicationId application, ZoneId... zones) {
diff --git a/default_build_settings.cmake b/default_build_settings.cmake
index 9b7c7302a54..fb2500d07be 100644
--- a/default_build_settings.cmake
+++ b/default_build_settings.cmake
@@ -4,12 +4,13 @@ include(VespaExtendedDefaultBuildSettings OPTIONAL)
function(setup_vespa_default_build_settings_rhel_6_10)
message("-- Setting up default build settings for rhel 6.10")
+ set(DEFAULT_EXTRA_INCLUDE_DIRECTORY "${VESPA_DEPS}/include" "/usr/include/openblas" PARENT_SCOPE)
endfunction()
function(setup_vespa_default_build_settings_rhel_7_7)
message("-- Setting up default build settings for rhel 7.7")
set(DEFAULT_EXTRA_LINK_DIRECTORY "${VESPA_DEPS}/lib64" "/usr/lib64/llvm5.0/lib" PARENT_SCOPE)
- set(DEFAULT_EXTRA_INCLUDE_DIRECTORY "${VESPA_DEPS}/include" "/usr/include/llvm5.0" PARENT_SCOPE)
+ set(DEFAULT_EXTRA_INCLUDE_DIRECTORY "${VESPA_DEPS}/include" "/usr/include/llvm5.0" "/usr/include/openblas" PARENT_SCOPE)
set(DEFAULT_VESPA_LLVM_VERSION "5.0" PARENT_SCOPE)
endfunction()
@@ -21,12 +22,13 @@ endfunction()
function(setup_vespa_default_build_settings_centos_7)
message("-- Setting up default build settings for centos 7")
set(DEFAULT_EXTRA_LINK_DIRECTORY "${VESPA_DEPS}/lib64" "/usr/lib64/llvm5.0/lib" PARENT_SCOPE)
- set(DEFAULT_EXTRA_INCLUDE_DIRECTORY "${VESPA_DEPS}/include" "/usr/include/llvm5.0" PARENT_SCOPE)
+ set(DEFAULT_EXTRA_INCLUDE_DIRECTORY "${VESPA_DEPS}/include" "/usr/include/llvm5.0" "/usr/include/openblas" PARENT_SCOPE)
set(DEFAULT_VESPA_LLVM_VERSION "5.0" PARENT_SCOPE)
endfunction()
function(setup_vespa_default_build_settings_centos_8)
message("-- Setting up default build settings for centos 8")
+ set(DEFAULT_EXTRA_INCLUDE_DIRECTORY "${VESPA_DEPS}/include" "/usr/include/openblas" PARENT_SCOPE)
set(DEFAULT_VESPA_LLVM_VERSION "8" PARENT_SCOPE)
endfunction()
@@ -39,14 +41,14 @@ function(setup_vespa_default_build_settings_darwin)
else()
set(DEFAULT_VESPA_LLVM_VERSION "8" PARENT_SCOPE)
endif()
- set(DEFAULT_CMAKE_PREFIX_PATH "${VESPA_DEPS}" "/usr/local/opt/bison" "/usr/local/opt/flex" "/usr/local/opt/openssl@1.1" PARENT_SCOPE)
- set(DEFAULT_EXTRA_LINK_DIRECTORY "${VESPA_DEPS}/lib" "/usr/local/opt/bison/lib" "/usr/local/opt/flex/lib" "/usr/local/opt/icu4c/lib" "/usr/local/opt/openssl@1.1/lib")
+ set(DEFAULT_CMAKE_PREFIX_PATH "${VESPA_DEPS}" "/usr/local/opt/bison" "/usr/local/opt/flex" "/usr/local/opt/openssl@1.1" "/usr/local/opt/openblas" PARENT_SCOPE)
+ set(DEFAULT_EXTRA_LINK_DIRECTORY "${VESPA_DEPS}/lib" "/usr/local/opt/bison/lib" "/usr/local/opt/flex/lib" "/usr/local/opt/icu4c/lib" "/usr/local/opt/openssl@1.1/lib" "/usr/local/opt/openblas/lib")
if(DEFINED DEFAULT_LLVM_LINK_DIRECTORY)
list(APPEND DEFAULT_EXTRA_LINK_DIRECTORY "${DEFAULT_LLVM_LINK_DIRECTORY}")
endif()
list(APPEND DEFAULT_EXTRA_LINK_DIRECTORY "/usr/local/lib")
set(DEFAULT_EXTRA_LINK_DIRECTORY "${DEFAULT_EXTRA_LINK_DIRECTORY}" PARENT_SCOPE)
- set(DEFAULT_EXTRA_INCLUDE_DIRECTORY "${VESPA_DEPS}/include" "/usr/local/opt/flex/include" "/usr/local/opt/icu4c/include" "/usr/local/opt/openssl@1.1/include")
+ set(DEFAULT_EXTRA_INCLUDE_DIRECTORY "${VESPA_DEPS}/include" "/usr/local/opt/flex/include" "/usr/local/opt/icu4c/include" "/usr/local/opt/openssl@1.1/include" "/usr/local/opt/openblas/include")
if(DEFINED DEFAULT_LLVM_INCLUDE_DIRECTORY)
list(APPEND DEFAULT_EXTRA_INCLUDE_DIRECTORY "${DEFAULT_LLVM_INCLUDE_DIRECTORY}")
endif()
@@ -56,21 +58,25 @@ endfunction()
function(setup_vespa_default_build_settings_fedora_29)
message("-- Setting up default build settings for fedora 29")
+ set(DEFAULT_EXTRA_INCLUDE_DIRECTORY "${VESPA_DEPS}/include" "/usr/include/openblas" PARENT_SCOPE)
set(DEFAULT_VESPA_LLVM_VERSION "7" PARENT_SCOPE)
endfunction()
function(setup_vespa_default_build_settings_fedora_30)
message("-- Setting up default build settings for fedora 30")
+ set(DEFAULT_EXTRA_INCLUDE_DIRECTORY "${VESPA_DEPS}/include" "/usr/include/openblas" PARENT_SCOPE)
set(DEFAULT_VESPA_LLVM_VERSION "8" PARENT_SCOPE)
endfunction()
function(setup_vespa_default_build_settings_fedora_31)
message("-- Setting up default build settings for fedora 31")
+ set(DEFAULT_EXTRA_INCLUDE_DIRECTORY "${VESPA_DEPS}/include" "/usr/include/openblas" PARENT_SCOPE)
set(DEFAULT_VESPA_LLVM_VERSION "9" PARENT_SCOPE)
endfunction()
function(setup_vespa_default_build_settings_fedora_32)
message("-- Setting up default build settings for fedora 32")
+ set(DEFAULT_EXTRA_INCLUDE_DIRECTORY "${VESPA_DEPS}/include" "/usr/include/openblas" PARENT_SCOPE)
set(DEFAULT_VESPA_LLVM_VERSION "9" PARENT_SCOPE)
endfunction()
diff --git a/dist/vespa.spec b/dist/vespa.spec
index 833e4798084..3a336496f4c 100644
--- a/dist/vespa.spec
+++ b/dist/vespa.spec
@@ -47,10 +47,11 @@ BuildRequires: vespa-boost-devel >= 1.59.0-6
BuildRequires: vespa-gtest >= 1.8.1-1
BuildRequires: vespa-protobuf-devel >= 3.7.0-4
BuildRequires: vespa-openssl-devel >= 1.1.1c-1
+BuildRequires: vespa-icu-devel >= 65.1.0-1
%endif
%if 0%{?el8}
BuildRequires: cmake >= 3.11.4-3
-BuildRequires: llvm-devel >= 7.0.1-3
+BuildRequires: llvm-devel >= 8.0.1
BuildRequires: boost-devel >= 1.66
BuildRequires: openssl-devel
BuildRequires: vespa-gtest >= 1.8.1-1
@@ -91,7 +92,9 @@ BuildRequires: openblas-devel
BuildRequires: lz4-devel
BuildRequires: libzstd-devel
BuildRequires: zlib-devel
+%if ! 0%{?el7}
BuildRequires: libicu-devel
+%endif
BuildRequires: java-11-openjdk-devel
BuildRequires: rpm-build
BuildRequires: make
@@ -132,25 +135,28 @@ Requires: openblas-serial
Requires: lz4
Requires: libzstd
Requires: zlib
+%if ! 0%{?el7}
Requires: libicu
+%endif
Requires: perf
Requires: gdb
Requires: net-tools
%if 0%{?el7}
Requires: llvm5.0
Requires: vespa-openssl >= 1.1.1c-1
+Requires: vespa-icu >= 65.1.0-1
Requires: vespa-protobuf >= 3.7.0-4
%define _vespa_llvm_version 5.0
%define _extra_link_directory /usr/lib64/llvm5.0/lib;%{_vespa_deps_prefix}/lib64
-%define _extra_include_directory /usr/include/llvm5.0;%{_vespa_deps_prefix}/include
+%define _extra_include_directory /usr/include/llvm5.0;%{_vespa_deps_prefix}/include;/usr/include/openblas
%endif
%if 0%{?el8}
-Requires: llvm-libs >= 7.0.1-3
+Requires: llvm-libs >= 8.0.1
Requires: vespa-protobuf >= 3.7.0-4
Requires: openssl-libs
-%define _vespa_llvm_version 7.0
+%define _vespa_llvm_version 8
%define _extra_link_directory %{_vespa_deps_prefix}/lib64
-%define _extra_include_directory %{_vespa_deps_prefix}/include
+%define _extra_include_directory %{_vespa_deps_prefix}/include;/usr/include/openblas
%endif
%if 0%{?fedora}
Requires: vespa-protobuf >= 3.7.0-4
@@ -172,7 +178,7 @@ Requires: llvm-libs >= 9.0.0
%define _vespa_llvm_version 9
%endif
%define _extra_link_directory %{_vespa_deps_prefix}/lib64
-%define _extra_include_directory %{_vespa_deps_prefix}/include
+%define _extra_include_directory %{_vespa_deps_prefix}/include;/usr/include/openblas
%endif
Requires: java-11-openjdk
Requires(pre): shadow-utils
diff --git a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/ContainerResources.java b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/ContainerResources.java
index ffd5dffece9..c7b9d342043 100644
--- a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/ContainerResources.java
+++ b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/ContainerResources.java
@@ -124,4 +124,8 @@ public class ContainerResources {
public ContainerResources withMemoryBytes(long memoryBytes) {
return new ContainerResources(cpus, cpuShares, memoryBytes);
}
+
+ public ContainerResources withUnlimitedCpus() {
+ return new ContainerResources(0, 0, memoryBytes);
+ }
}
diff --git a/document/abi-spec.json b/document/abi-spec.json
index 9b67eeb97ee..abbf03bc228 100644
--- a/document/abi-spec.json
+++ b/document/abi-spec.json
@@ -427,6 +427,8 @@
"methods": [
"public void <init>(java.lang.String)",
"public void <init>(java.lang.String, com.yahoo.document.StructDataType, com.yahoo.document.StructDataType)",
+ "public void <init>(java.lang.String, com.yahoo.document.StructDataType, com.yahoo.document.StructDataType, java.util.Set)",
+ "public void <init>(java.lang.String, java.util.Set)",
"public com.yahoo.document.DocumentType clone()",
"public com.yahoo.document.Document createFieldValue()",
"public java.lang.Class getValueClass()",
@@ -448,6 +450,8 @@
"public com.yahoo.document.Field getField(int)",
"public boolean hasField(java.lang.String)",
"public int getFieldCount()",
+ "public java.util.Set getImportedFieldNames()",
+ "public boolean hasImportedField(java.lang.String)",
"public com.yahoo.document.Field removeField(java.lang.String)",
"public java.util.Collection getFields()",
"public java.util.Set fieldSet()",
@@ -712,6 +716,8 @@
"public com.yahoo.document.DocumenttypesConfig$Documenttype$Builder fieldsets(java.util.Map)",
"public com.yahoo.document.DocumenttypesConfig$Documenttype$Builder referencetype(com.yahoo.document.DocumenttypesConfig$Documenttype$Referencetype$Builder)",
"public com.yahoo.document.DocumenttypesConfig$Documenttype$Builder referencetype(java.util.List)",
+ "public com.yahoo.document.DocumenttypesConfig$Documenttype$Builder importedfield(com.yahoo.document.DocumenttypesConfig$Documenttype$Importedfield$Builder)",
+ "public com.yahoo.document.DocumenttypesConfig$Documenttype$Builder importedfield(java.util.List)",
"public com.yahoo.document.DocumenttypesConfig$Documenttype build()"
],
"fields": [
@@ -719,7 +725,8 @@
"public java.util.List datatype",
"public java.util.List annotationtype",
"public java.util.Map fieldsets",
- "public java.util.List referencetype"
+ "public java.util.List referencetype",
+ "public java.util.List importedfield"
]
},
"com.yahoo.document.DocumenttypesConfig$Documenttype$Datatype$Annotationref$Annotation$Builder": {
@@ -1264,6 +1271,35 @@
],
"fields": []
},
+ "com.yahoo.document.DocumenttypesConfig$Documenttype$Importedfield$Builder": {
+ "superClass": "java.lang.Object",
+ "interfaces": [
+ "com.yahoo.config.ConfigBuilder"
+ ],
+ "attributes": [
+ "public"
+ ],
+ "methods": [
+ "public void <init>()",
+ "public void <init>(com.yahoo.document.DocumenttypesConfig$Documenttype$Importedfield)",
+ "public com.yahoo.document.DocumenttypesConfig$Documenttype$Importedfield$Builder name(java.lang.String)",
+ "public com.yahoo.document.DocumenttypesConfig$Documenttype$Importedfield build()"
+ ],
+ "fields": []
+ },
+ "com.yahoo.document.DocumenttypesConfig$Documenttype$Importedfield": {
+ "superClass": "com.yahoo.config.InnerNode",
+ "interfaces": [],
+ "attributes": [
+ "public",
+ "final"
+ ],
+ "methods": [
+ "public void <init>(com.yahoo.document.DocumenttypesConfig$Documenttype$Importedfield$Builder)",
+ "public java.lang.String name()"
+ ],
+ "fields": []
+ },
"com.yahoo.document.DocumenttypesConfig$Documenttype$Inherits$Builder": {
"superClass": "java.lang.Object",
"interfaces": [
@@ -1347,7 +1383,9 @@
"public java.util.Map fieldsets()",
"public com.yahoo.document.DocumenttypesConfig$Documenttype$Fieldsets fieldsets(java.lang.String)",
"public java.util.List referencetype()",
- "public com.yahoo.document.DocumenttypesConfig$Documenttype$Referencetype referencetype(int)"
+ "public com.yahoo.document.DocumenttypesConfig$Documenttype$Referencetype referencetype(int)",
+ "public java.util.List importedfield()",
+ "public com.yahoo.document.DocumenttypesConfig$Documenttype$Importedfield importedfield(int)"
],
"fields": []
},
diff --git a/document/src/main/java/com/yahoo/document/DocumentType.java b/document/src/main/java/com/yahoo/document/DocumentType.java
index 42ccca9f477..23559878fbb 100755
--- a/document/src/main/java/com/yahoo/document/DocumentType.java
+++ b/document/src/main/java/com/yahoo/document/DocumentType.java
@@ -41,6 +41,7 @@ public class DocumentType extends StructuredDataType {
private StructDataType bodyType;
private List<DocumentType> inherits = new ArrayList<>(1);
private Map<String, Set<Field>> fieldSets = new HashMap<>();
+ private final Set<String> importedFieldNames;
/**
* Creates a new document type and registers it with the document type manager.
@@ -51,8 +52,7 @@ public class DocumentType extends StructuredDataType {
* @param name The name of the new document type
*/
public DocumentType(String name) {
- this(name, new StructDataType(name + ".header"),
- new StructDataType(name + ".body"));
+ this(name, createHeaderStructType(name), createBodyStructType(name));
}
/**
@@ -65,9 +65,27 @@ public class DocumentType extends StructuredDataType {
* @param bodyType The type of the body struct
*/
public DocumentType(String name, StructDataType headerType, StructDataType bodyType) {
+ this(name, headerType, bodyType, Collections.emptySet());
+ }
+
+ public DocumentType(String name, StructDataType headerType,
+ StructDataType bodyType, Set<String> importedFieldNames) {
super(name);
this.headerType = headerType;
this.bodyType = bodyType;
+ this.importedFieldNames = Collections.unmodifiableSet(importedFieldNames);
+ }
+
+ public DocumentType(String name, Set<String> importedFieldNames) {
+ this(name, createHeaderStructType(name), createBodyStructType(name), importedFieldNames);
+ }
+
+ private static StructDataType createHeaderStructType(String name) {
+ return new StructDataType(name + ".header");
+ }
+
+ private static StructDataType createBodyStructType(String name) {
+ return new StructDataType(name + ".body");
}
@Override
@@ -369,6 +387,14 @@ public class DocumentType extends StructuredDataType {
return headerType.getFieldCount() + bodyType.getFieldCount();
}
+ public Set<String> getImportedFieldNames() {
+ return importedFieldNames;
+ }
+
+ public boolean hasImportedField(String fieldName) {
+ return importedFieldNames.contains(fieldName);
+ }
+
/**
* Removes an field from the DocumentType.
*
diff --git a/document/src/main/java/com/yahoo/document/DocumentTypeManagerConfigurer.java b/document/src/main/java/com/yahoo/document/DocumentTypeManagerConfigurer.java
index 5e698e980ff..21a163aa2b9 100644
--- a/document/src/main/java/com/yahoo/document/DocumentTypeManagerConfigurer.java
+++ b/document/src/main/java/com/yahoo/document/DocumentTypeManagerConfigurer.java
@@ -10,8 +10,10 @@ import com.yahoo.log.LogLevel;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.Map;
import java.util.logging.Logger;
+import java.util.stream.Collectors;
/**
* Configures the Vespa document manager from a config id.
@@ -144,7 +146,10 @@ public class DocumentTypeManagerConfigurer implements ConfigSubscriber.SingleSub
for (Field field : body.getFields()) {
field.setHeader(false);
}
- DocumentType type = new DocumentType(doc.name(), header, body);
+ var importedFields = doc.importedfield().stream()
+ .map(f -> f.name())
+ .collect(Collectors.toUnmodifiableSet());
+ DocumentType type = new DocumentType(doc.name(), header, body, importedFields);
for (Object j : doc.inherits()) {
DocumentmanagerConfig.Datatype.Documenttype.Inherits parent =
(DocumentmanagerConfig.Datatype.Documenttype.Inherits) j;
diff --git a/document/src/main/java/com/yahoo/document/FieldPath.java b/document/src/main/java/com/yahoo/document/FieldPath.java
index 295b1c9dd5c..134681a4028 100755
--- a/document/src/main/java/com/yahoo/document/FieldPath.java
+++ b/document/src/main/java/com/yahoo/document/FieldPath.java
@@ -10,7 +10,7 @@ import java.util.List;
* This class represents a path into a document, that can be used to iterate through the document and extract the field
* values you're interested in.
*
- * @author <a href="mailto:thomasg@yahoo-inc.com">Thomas Gundersen</a>
+ * @author Thomas Gundersen
*/
public class FieldPath implements Iterable<FieldPathEntry> {
diff --git a/document/src/main/java/com/yahoo/document/select/rule/AttributeNode.java b/document/src/main/java/com/yahoo/document/select/rule/AttributeNode.java
index 0cedda7c4f0..9e2759e1590 100644
--- a/document/src/main/java/com/yahoo/document/select/rule/AttributeNode.java
+++ b/document/src/main/java/com/yahoo/document/select/rule/AttributeNode.java
@@ -7,6 +7,7 @@ import com.yahoo.document.Document;
import com.yahoo.document.DocumentGet;
import com.yahoo.document.DocumentPut;
import com.yahoo.document.DocumentRemove;
+import com.yahoo.document.DocumentType;
import com.yahoo.document.DocumentUpdate;
import com.yahoo.document.FieldPath;
import com.yahoo.document.datatypes.FieldPathIteratorHandler;
@@ -131,10 +132,37 @@ public class AttributeNode implements ExpressionNode {
throw new IllegalStateException("Function '" + function + "' is not supported.");
}
- private static Object evaluateFieldPath(String fieldPth, Object value) {
+ private static boolean looksLikeComplexFieldPath(String path) {
+ for (int i = 0; i < path.length(); ++i) {
+ switch (path.charAt(i)) {
+ case '.':
+ case '{':
+ case '[':
+ return true;
+ }
+ }
+ return false;
+ }
+
+ private static boolean isSimpleImportedField(String path, DocumentType documentType) {
+ if (looksLikeComplexFieldPath(path)) {
+ return false;
+ }
+ return documentType.hasImportedField(path);
+ }
+
+ private static Object evaluateFieldPath(String fieldPathStr, Object value) {
if (value instanceof DocumentPut) {
final Document doc = ((DocumentPut) value).getDocument();
- FieldPath fieldPath = doc.getDataType().buildFieldPath(fieldPth);
+ if (isSimpleImportedField(fieldPathStr, doc.getDataType())) {
+ // Imported fields can only be meaningfully evaluated in the backend, so we
+ // explicitly treat them as if they are valid fields with missing values. This
+ // will be treated the same as if it's a normal field by the selection operators.
+ // This avoids any awkward interaction with Invalid values or having to
+ // augment the FieldPath code with knowledge of imported fields.
+ return null;
+ }
+ FieldPath fieldPath = doc.getDataType().buildFieldPath(fieldPathStr);
IteratorHandler handler = new IteratorHandler();
doc.iterateNested(fieldPath, 0, handler);
if (handler.values.isEmpty()) {
diff --git a/document/src/test/document/documentmanager.importedfields.cfg b/document/src/test/document/documentmanager.importedfields.cfg
new file mode 100644
index 00000000000..765e290ae82
--- /dev/null
+++ b/document/src/test/document/documentmanager.importedfields.cfg
@@ -0,0 +1,65 @@
+datatype[7]
+datatype[0].id -1365874599
+datatype[0].arraytype[0]
+datatype[0].weightedsettype[0]
+datatype[0].structtype[1]
+datatype[0].structtype[0].name referenced_type.header
+datatype[0].structtype[0].version 9
+datatype[0].structtype[0].field[0]
+datatype[0].documenttype[0]
+datatype[1].id 278604398
+datatype[1].arraytype[0]
+datatype[1].weightedsettype[0]
+datatype[1].structtype[1]
+datatype[1].structtype[0].name referenced_type.body
+datatype[1].structtype[0].version 9
+datatype[1].structtype[0].field[0]
+datatype[1].documenttype[0]
+datatype[2].id 124647170
+datatype[2].arraytype[0]
+datatype[2].weightedsettype[0]
+datatype[2].structtype[0]
+datatype[2].documenttype[1]
+datatype[2].documenttype[0].name referenced_type
+datatype[2].documenttype[0].version 9
+datatype[2].documenttype[0].inherits[0]
+datatype[2].documenttype[0].headerstruct -1365874599
+datatype[2].documenttype[0].bodystruct 278604398
+datatype[3].id 12345678
+datatype[3].arraytype[0]
+datatype[3].weightedsettype[0]
+datatype[3].structtype[0]
+datatype[3].documenttype[0]
+datatype[3].referencetype[1]
+datatype[3].referencetype[0].target_type_id 124647170
+datatype[4].id 673066331
+datatype[4].arraytype[0]
+datatype[4].weightedsettype[0]
+datatype[4].structtype[1]
+datatype[4].structtype[0].name type_with_ref.header
+datatype[4].structtype[0].version 234
+datatype[4].structtype[0].field[1]
+datatype[4].structtype[0].field[0].name my_ref_field
+datatype[4].structtype[0].field[0].id[0]
+datatype[4].structtype[0].field[0].datatype 12345678
+datatype[4].documenttype[0]
+datatype[5].id -176986064
+datatype[5].arraytype[0]
+datatype[5].weightedsettype[0]
+datatype[5].structtype[1]
+datatype[5].structtype[0].name type_with_ref.body
+datatype[5].structtype[0].version 234
+datatype[5].structtype[0].field[0]
+datatype[5].documenttype[0]
+datatype[6].id -1293964543
+datatype[6].arraytype[0]
+datatype[6].weightedsettype[0]
+datatype[6].structtype[0]
+datatype[6].documenttype[1]
+datatype[6].documenttype[0].name type_with_ref
+datatype[6].documenttype[0].version 234
+datatype[6].documenttype[0].inherits[0]
+datatype[6].documenttype[0].headerstruct 673066331
+datatype[6].documenttype[0].bodystruct -176986064
+datatype[6].documenttype[0].importedfield[0].name "my_cool_imported_field"
+datatype[6].documenttype[0].importedfield[1].name "my_awesome_imported_field" \ No newline at end of file
diff --git a/document/src/test/java/com/yahoo/document/DocumentTypeManagerTestCase.java b/document/src/test/java/com/yahoo/document/DocumentTypeManagerTestCase.java
index cac77dba434..57b36d9758f 100644
--- a/document/src/test/java/com/yahoo/document/DocumentTypeManagerTestCase.java
+++ b/document/src/test/java/com/yahoo/document/DocumentTypeManagerTestCase.java
@@ -8,6 +8,7 @@ import com.yahoo.document.datatypes.StringFieldValue;
import com.yahoo.document.datatypes.StructuredFieldValue;
import org.junit.Test;
+import java.util.HashSet;
import java.util.Iterator;
import static org.junit.Assert.assertEquals;
@@ -555,6 +556,35 @@ search annotationsimplicitstruct {
assertTrue(fieldRefType.getTargetType() == targetDocType);
}
+ @Test
+ public void imported_fields_are_empty_if_no_fields_provided_in_config() {
+ var manager = createConfiguredManager("file:src/test/document/documentmanager.singlereference.cfg");
+ var docType = manager.getDocumentType("type_with_ref");
+
+ assertNotNull(docType.getImportedFieldNames());
+ assertEquals(docType.getImportedFieldNames().size(), 0);
+ assertFalse(docType.hasImportedField("foo"));
+ }
+
+ @Test
+ public void imported_fields_are_populated_from_config() {
+ var manager = createConfiguredManager("file:src/test/document/documentmanager.importedfields.cfg");
+ var docType = manager.getDocumentType("type_with_ref");
+
+ var expectedFields = new HashSet<String>();
+ expectedFields.add("my_cool_imported_field");
+ expectedFields.add("my_awesome_imported_field");
+ assertEquals(docType.getImportedFieldNames(), expectedFields);
+
+ assertTrue(docType.hasImportedField("my_cool_imported_field"));
+ assertTrue(docType.hasImportedField("my_awesome_imported_field"));
+ assertFalse(docType.hasImportedField("a_missing_imported_field"));
+ }
+
+ // TODO test clone(). Also fieldSets not part of clone()..!
+
+ // TODO add imported field to equals()/hashCode() for DocumentType? fieldSets not part of this...
+
// TODO test reference to own doc type
}
diff --git a/document/src/test/java/com/yahoo/document/select/DocumentSelectorTestCase.java b/document/src/test/java/com/yahoo/document/select/DocumentSelectorTestCase.java
index b33d81ffdea..0212be8542e 100644
--- a/document/src/test/java/com/yahoo/document/select/DocumentSelectorTestCase.java
+++ b/document/src/test/java/com/yahoo/document/select/DocumentSelectorTestCase.java
@@ -8,10 +8,13 @@ import com.yahoo.document.select.parser.ParseException;
import com.yahoo.document.select.parser.TokenMgrException;
import com.yahoo.yolean.Exceptions;
import org.junit.Before;
+import org.junit.Rule;
import org.junit.Test;
+import org.junit.rules.ExpectedException;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.HashSet;
import java.util.List;
import java.util.Map;
@@ -26,11 +29,15 @@ import static org.junit.Assert.fail;
*/
public class DocumentSelectorTestCase {
+ @Rule
+ public final ExpectedException exceptionRule = ExpectedException.none();
+
private static DocumentTypeManager manager = new DocumentTypeManager();
@Before
public void setUp() {
- DocumentType type = new DocumentType("test");
+ var importedFields = new HashSet<>(List.of("my_imported_field"));
+ DocumentType type = new DocumentType("test", importedFields);
type.addField("hint", DataType.INT);
type.addField("hfloat", DataType.FLOAT);
type.addField("hstring", DataType.STRING);
@@ -697,6 +704,27 @@ public class DocumentSelectorTestCase {
}
@Test
+ public void imported_field_references_are_treated_as_valid_field_with_missing_value() throws ParseException {
+ var documents = createDocs();
+ assertEquals(Result.TRUE, evaluate("test.my_imported_field == null", documents.get(0)));
+ assertEquals(Result.FALSE, evaluate("test.my_imported_field != null", documents.get(0)));
+ assertEquals(Result.FALSE, evaluate("test.my_imported_field", documents.get(0)));
+ // Only (in)equality operators are well defined for null values; everything else becomes Invalid.
+ assertEquals(Result.INVALID, evaluate("test.my_imported_field > 0", documents.get(0)));
+ }
+
+ @Test
+ public void imported_fields_only_supported_for_simple_expressions() throws ParseException {
+ exceptionRule.expect(IllegalArgumentException.class);
+ // TODO we should probably handle this case specially and give a better exception message
+ exceptionRule.expectMessage("Field 'my_imported_field' not found in type datatype test");
+
+ var documents = createDocs();
+ // Nested field access is NOT considered a simple expression.
+ evaluate("test.my_imported_field.foo", documents.get(0));
+ }
+
+ @Test
public void testTicket1769674() {
assertParseError("music.uri=\"junk",
"Lexical error at line -1, column 17. Encountered: <EOF> after : \"\\\"junk\"");
diff --git a/document/src/tests/documentselectparsertest.cpp b/document/src/tests/documentselectparsertest.cpp
index 79b849c5ba9..3441a336553 100644
--- a/document/src/tests/documentselectparsertest.cpp
+++ b/document/src/tests/documentselectparsertest.cpp
@@ -92,6 +92,10 @@ void DocumentSelectParserTest::SetUp()
builder.document(-1673092522, "usergroup",
Struct("usergroup.header"),
Struct("usergroup.body"));
+ builder.document(1234567, "with_imported",
+ Struct("with_imported.header"),
+ Struct("with_imported.body"))
+ .imported_field("my_imported_field");
_repo = std::make_unique<DocumentTypeRepo>(builder.config());
_parser = std::make_unique<select::Parser>(*_repo, _bucketIdFactory);
@@ -103,7 +107,7 @@ Document::SP DocumentSelectParserTest::createDoc(
uint64_t hlong)
{
const DocumentType* type = _repo->getDocumentType(doctype);
- Document::SP doc(new Document(*type, DocumentId(id)));
+ auto doc = std::make_shared<Document>(*type, DocumentId(id));
doc->setValue(doc->getField("headerval"), IntFieldValue(hint));
if (hlong != 0) {
@@ -1524,4 +1528,26 @@ TEST_F(DocumentSelectParserTest, test_parse_utilities_handle_malformed_input)
check_parse_double("1.79769e+309", true, std::numeric_limits<double>::infinity());
}
+TEST_F(DocumentSelectParserTest, imported_field_references_are_treated_as_valid_field_with_missing_value) {
+ const DocumentType* type = _repo->getDocumentType("with_imported");
+ ASSERT_TRUE(type != nullptr);
+ Document doc(*type, DocumentId("id::with_imported::foo"));
+
+ PARSE("with_imported.my_imported_field == null", doc, True);
+ PARSE("with_imported.my_imported_field != null", doc, False);
+ PARSE("with_imported.my_imported_field", doc, False);
+ // Only (in)equality operators are well defined for null values; everything else becomes Invalid.
+ PARSE("with_imported.my_imported_field > 0", doc, Invalid);
+}
+
+TEST_F(DocumentSelectParserTest, imported_field_references_only_support_for_simple_expressions) {
+ const DocumentType* type = _repo->getDocumentType("with_imported");
+ ASSERT_TRUE(type != nullptr);
+ Document doc(*type, DocumentId("id::with_imported::foo"));
+
+ PARSE("with_imported.my_imported_field.foo", doc, Invalid);
+ PARSE("with_imported.my_imported_field[0]", doc, Invalid);
+ PARSE("with_imported.my_imported_field{foo}", doc, Invalid);
+}
+
} // document
diff --git a/document/src/tests/repo/documenttyperepo_test.cpp b/document/src/tests/repo/documenttyperepo_test.cpp
index b263ad75930..0bc80ebcd16 100644
--- a/document/src/tests/repo/documenttyperepo_test.cpp
+++ b/document/src/tests/repo/documenttyperepo_test.cpp
@@ -532,6 +532,47 @@ TEST("Reference fields are resolved to correct reference type") {
EXPECT_EQUAL(*ref1_type, type->getFieldsType().getField("ref3").getDataType());
}
+TEST("Config with no imported fields has empty imported fields set in DocumentType") {
+ DocumenttypesConfigBuilderHelper builder;
+ builder.document(doc_type_id, type_name,
+ Struct(header_name), Struct(body_name));
+ DocumentTypeRepo repo(builder.config());
+ const auto *type = repo.getDocumentType(doc_type_id);
+ ASSERT_TRUE(type != nullptr);
+ EXPECT_TRUE(type->imported_field_names().empty());
+ EXPECT_FALSE(type->has_imported_field_name("foo"));
+}
+
+TEST("Configured imported field names are available in the DocumentType") {
+ const int type_2_id = doc_type_id + 1;
+ // Note: we cheat a bit by specifying imported field names in types that have no
+ // reference fields. Add to test if we add config read-time validation of this. :)
+ DocumenttypesConfigBuilderHelper builder;
+ // Type with one imported field
+ builder.document(doc_type_id, type_name,
+ Struct(header_name), Struct(body_name))
+ .imported_field("my_cool_field");
+ // Type with two imported fields
+ builder.document(type_2_id, type_name_2,
+ Struct(header_name_2), Struct(body_name_2))
+ .imported_field("my_awesome_field")
+ .imported_field("my_swag_field");
+
+ DocumentTypeRepo repo(builder.config());
+ const auto* type = repo.getDocumentType(doc_type_id);
+ ASSERT_TRUE(type != nullptr);
+ EXPECT_EQUAL(1u, type->imported_field_names().size());
+ EXPECT_TRUE(type->has_imported_field_name("my_cool_field"));
+ EXPECT_FALSE(type->has_imported_field_name("my_awesome_field"));
+
+ type = repo.getDocumentType(type_2_id);
+ ASSERT_TRUE(type != nullptr);
+ EXPECT_EQUAL(2u, type->imported_field_names().size());
+ EXPECT_TRUE(type->has_imported_field_name("my_awesome_field"));
+ EXPECT_TRUE(type->has_imported_field_name("my_swag_field"));
+ EXPECT_FALSE(type->has_imported_field_name("my_cool_field"));
+}
+
namespace {
const TensorDataType &
diff --git a/document/src/vespa/document/config/documentmanager.def b/document/src/vespa/document/config/documentmanager.def
index 1961f67d83c..092a29d9293 100644
--- a/document/src/vespa/document/config/documentmanager.def
+++ b/document/src/vespa/document/config/documentmanager.def
@@ -93,6 +93,9 @@ datatype[].documenttype[].bodystruct int
## Field sets
datatype[].documenttype[].fieldsets{}.fields[] string
+## Imported fields (specified outside the document block in the schema)
+datatype[].documenttype[].importedfield[].name string
+
## Cross-document reference with ID of target document type
datatype[].referencetype[].target_type_id int
diff --git a/document/src/vespa/document/config/documenttypes.def b/document/src/vespa/document/config/documenttypes.def
index fb3fcd8b123..0f0a9e3e37c 100644
--- a/document/src/vespa/document/config/documenttypes.def
+++ b/document/src/vespa/document/config/documenttypes.def
@@ -104,3 +104,6 @@ documenttype[].referencetype[].id int
## Numeric ID of the document type instances of the reference point to.
documenttype[].referencetype[].target_type_id int
+
+## Imported fields (specified outside the document block in the schema)
+documenttype[].importedfield[].name string
diff --git a/document/src/vespa/document/datatype/documenttype.cpp b/document/src/vespa/document/datatype/documenttype.cpp
index ffc250eacba..c2739dc1303 100644
--- a/document/src/vespa/document/datatype/documenttype.cpp
+++ b/document/src/vespa/document/datatype/documenttype.cpp
@@ -23,9 +23,10 @@ DocumentType::DocumentType() = default;
DocumentType::DocumentType(stringref name, int32_t id)
: StructuredDataType(name, id),
_inheritedTypes(),
- _ownedFields(new StructDataType(name + ".header")),
+ _ownedFields(std::make_shared<StructDataType>(name + ".header")),
_fields(_ownedFields.get()),
- _fieldSets()
+ _fieldSets(),
+ _imported_field_names()
{
if (name != "document") {
_inheritedTypes.push_back(DataType::DOCUMENT);
@@ -36,7 +37,8 @@ DocumentType::DocumentType(stringref name, int32_t id, const StructDataType& fie
: StructuredDataType(name, id),
_inheritedTypes(),
_fields(&fields),
- _fieldSets()
+ _fieldSets(),
+ _imported_field_names()
{
if (name != "document") {
_inheritedTypes.push_back(DataType::DOCUMENT);
@@ -46,12 +48,13 @@ DocumentType::DocumentType(stringref name, int32_t id, const StructDataType& fie
DocumentType::DocumentType(stringref name)
: StructuredDataType(name),
_inheritedTypes(),
- _ownedFields(new StructDataType(name + ".header")),
+ _ownedFields(std::make_shared<StructDataType>(name + ".header")),
_fields(_ownedFields.get()),
- _fieldSets()
+ _fieldSets(),
+ _imported_field_names()
{
if (name != "document") {
- _inheritedTypes.push_back(DataType::DOCUMENT);
+ _inheritedTypes.emplace_back(DataType::DOCUMENT);
}
}
@@ -59,27 +62,28 @@ DocumentType::DocumentType(stringref name, const StructDataType& fields)
: StructuredDataType(name),
_inheritedTypes(),
_fields(&fields),
- _fieldSets()
+ _fieldSets(),
+ _imported_field_names()
{
if (name != "document") {
- _inheritedTypes.push_back(DataType::DOCUMENT);
+ _inheritedTypes.emplace_back(DataType::DOCUMENT);
}
}
DocumentType::~DocumentType() = default;
DocumentType &
-DocumentType::addFieldSet(const vespalib::string & name, const FieldSet::Fields & fields)
+DocumentType::addFieldSet(const vespalib::string & name, FieldSet::Fields fields)
{
- _fieldSets[name] = FieldSet(name, fields);
+ _fieldSets[name] = FieldSet(name, std::move(fields));
return *this;
}
const DocumentType::FieldSet *
DocumentType::getFieldSet(const vespalib::string & name) const
{
- FieldSetMap::const_iterator it(_fieldSets.find(name));
- return (it != _fieldSets.end()) ? & it->second : NULL;
+ auto it = _fieldSets.find(name);
+ return (it != _fieldSets.end()) ? & it->second : nullptr;
}
void
@@ -107,37 +111,34 @@ DocumentType::inherit(const DocumentType &docType) {
"Document type " + docType.toString() + " already inherits type "
+ toString() + ". Cannot add cyclic dependencies.", VESPA_STRLOC);
}
- // If we already inherits this type, there is no point in adding it
- // again.
+ // If we already inherits this type, there is no point in adding it again.
if (isA(docType)) {
- // If we already directly inherits it, complain
- for (std::vector<const DocumentType *>::const_iterator
- it = _inheritedTypes.begin(); it != _inheritedTypes.end(); ++it)
- {
- if (**it == docType) {
+ // If we already directly inherits it, complain
+ for (const auto* inherited : _inheritedTypes) {
+ if (*inherited == docType) {
throw IllegalArgumentException(
"DocumentType " + getName() + " already inherits "
"document type " + docType.getName(), VESPA_STRLOC);
}
}
- // Indirectly already inheriting it is oki, as this can happen
- // due to inherited documents inheriting the same type.
+ // Indirectly already inheriting it is oki, as this can happen
+ // due to inherited documents inheriting the same type.
LOG(info, "Document type %s inherits document type %s from multiple "
"types.", getName().c_str(), docType.getName().c_str());
return;
}
- // Add non-conflicting types.
+ // Add non-conflicting types.
Field::Set fs = docType._fields->getFieldSet();
- for (Field::Set::const_iterator it = fs.begin(); it != fs.end(); ++it) {
+ for (const auto* field : fs) {
if (!_ownedFields.get()) {
_ownedFields.reset(_fields->clone());
_fields = _ownedFields.get();
}
- _ownedFields->addInheritedField(**it);
+ _ownedFields->addInheritedField(*field);
}
// If we inherit default document type Document.0, remove that if adding
// another parent, as that has to also inherit Document
- if (_inheritedTypes.size() == 1 && *_inheritedTypes[0] == *DataType::DOCUMENT) {
+ if ((_inheritedTypes.size() == 1) && (*_inheritedTypes[0] == *DataType::DOCUMENT)) {
_inheritedTypes.clear();
}
_inheritedTypes.push_back(&docType);
@@ -168,8 +169,7 @@ DocumentType::print(std::ostream& out, bool verbose, const std::string& indent)
out << ")";
if (verbose) {
if (!_inheritedTypes.empty()) {
- std::vector<const DocumentType *>::const_iterator it(
- _inheritedTypes.begin());
+ auto it = _inheritedTypes.begin();
out << "\n" << indent << " : ";
(*it)->print(out, false, "");
while (++it != _inheritedTypes.end()) {
@@ -188,17 +188,18 @@ DocumentType::operator==(const DataType& other) const
{
if (&other == this) return true;
if (!DataType::operator==(other)) return false;
- const DocumentType* o(dynamic_cast<const DocumentType*>(&other));
- if (o == 0) return false;
+ const auto* o(dynamic_cast<const DocumentType*>(&other));
+ if (o == nullptr) return false;
if (*_fields != *o->_fields) return false;
if (_inheritedTypes.size() != o->_inheritedTypes.size()) return false;
- std::vector<const DocumentType *>::const_iterator it1(_inheritedTypes.begin());
- std::vector<const DocumentType *>::const_iterator it2(o->_inheritedTypes.begin());
+ auto it1 = _inheritedTypes.begin();
+ auto it2 = o->_inheritedTypes.begin();
while (it1 != _inheritedTypes.end()) {
if (**it1 != **it2) return false;
++it1;
++it2;
}
+ // TODO imported fields? like in the Java impl, field sets are not considered either... :I
return true;
}
@@ -228,6 +229,14 @@ DocumentType::getFieldSet() const
return _fields->getFieldSet();
}
+bool DocumentType::has_imported_field_name(const vespalib::string& name) const noexcept {
+ return (_imported_field_names.find(name) != _imported_field_names.end());
+}
+
+void DocumentType::add_imported_field_name(const vespalib::string& name) {
+ _imported_field_names.insert(name);
+}
+
DocumentType *
DocumentType::clone() const {
return new DocumentType(*this);
diff --git a/document/src/vespa/document/datatype/documenttype.h b/document/src/vespa/document/datatype/documenttype.h
index 7fcf5dfa237..ed6e9e66ab5 100644
--- a/document/src/vespa/document/datatype/documenttype.h
+++ b/document/src/vespa/document/datatype/documenttype.h
@@ -12,9 +12,11 @@
#pragma once
#include <vespa/document/datatype/structdatatype.h>
-
+#include <vespa/vespalib/stllike/hash_set.h>
+#include <vespa/vespalib/stllike/string.h>
#include <vector>
#include <map>
+#include <set>
namespace document {
@@ -25,12 +27,17 @@ class DocumentType : public StructuredDataType {
public:
class FieldSet {
public:
- typedef std::set<vespalib::string> Fields;
- FieldSet() : _name(), _fields() {}
- FieldSet(const vespalib::string & name) : _name(name), _fields() {}
- FieldSet(const vespalib::string & name, const Fields & fields) : _name(name), _fields(fields) {}
- const vespalib::string & getName() const { return _name; }
- const Fields & getFields() const { return _fields; }
+ using Fields = std::set<vespalib::string>;
+ FieldSet() = default;
+ explicit FieldSet(const vespalib::string & name) : _name(name), _fields() {}
+ FieldSet(const vespalib::string & name, Fields fields) : _name(name), _fields(std::move(fields)) {}
+ FieldSet(const FieldSet&) = default;
+ FieldSet& operator=(const FieldSet&) = default;
+ FieldSet(FieldSet&&) noexcept = default;
+ FieldSet& operator=(FieldSet&&) noexcept = default;
+
+ const vespalib::string & getName() const noexcept { return _name; }
+ const Fields & getFields() const noexcept { return _fields; }
FieldSet & add(vespalib::string & field) {
_fields.insert(field);
return *this;
@@ -39,28 +46,31 @@ public:
vespalib::string _name;
Fields _fields;
};
- typedef std::map<vespalib::string, FieldSet> FieldSetMap;
+ using FieldSetMap = std::map<vespalib::string, FieldSet>;
+ using ImportedFieldNames = vespalib::hash_set<vespalib::string>;
+
std::vector<const DocumentType *> _inheritedTypes;
- StructDataType::SP _ownedFields;
- const StructDataType* _fields;
- FieldSetMap _fieldSets;
+ StructDataType::SP _ownedFields;
+ const StructDataType* _fields;
+ FieldSetMap _fieldSets;
+ ImportedFieldNames _imported_field_names;
public:
- typedef std::unique_ptr<DocumentType> UP;
- typedef std::shared_ptr<DocumentType> SP;
+ using UP = std::unique_ptr<DocumentType>;
+ using SP = std::shared_ptr<DocumentType>;
DocumentType();
DocumentType(vespalib::stringref name, int32_t id);
DocumentType(vespalib::stringref name, int32_t id,
const StructDataType& fields);
- DocumentType(vespalib::stringref name);
+ explicit DocumentType(vespalib::stringref name);
DocumentType(vespalib::stringref name,
const StructDataType& fields);
- ~DocumentType();
+ ~DocumentType() override;
- const StructDataType& getFieldsType() const { return *_fields; }
+ const StructDataType& getFieldsType() const noexcept { return *_fields; }
void addField(const Field&);
@@ -89,9 +99,16 @@ public:
Field::Set getFieldSet() const override;
DocumentType* clone() const override;
- DocumentType & addFieldSet(const vespalib::string & name, const FieldSet::Fields & fields);
+ DocumentType & addFieldSet(const vespalib::string & name, FieldSet::Fields fields);
const FieldSet * getFieldSet(const vespalib::string & name) const;
+ const ImportedFieldNames& imported_field_names() const noexcept {
+ return _imported_field_names;
+ }
+ bool has_imported_field_name(const vespalib::string& name) const noexcept;
+ // Ideally the type would be immutable, but this is how it's built today.
+ void add_imported_field_name(const vespalib::string& name);
+
DECLARE_IDENTIFIABLE(DocumentType);
};
diff --git a/document/src/vespa/document/repo/configbuilder.h b/document/src/vespa/document/repo/configbuilder.h
index 15ee0da0c79..52ed8d878a2 100644
--- a/document/src/vespa/document/repo/configbuilder.h
+++ b/document/src/vespa/document/repo/configbuilder.h
@@ -38,9 +38,9 @@ struct TypeOrId {
};
struct Struct : DatatypeConfig {
- Struct(const vespalib::string &name) {
+ explicit Struct(vespalib::string name) {
type = Type::STRUCT;
- sstruct.name = name;
+ sstruct.name = std::move(name);
}
Struct &setCompression(Sstruct::Compression::Type t, int32_t level,
int32_t threshold, int32_t min_size) {
@@ -63,7 +63,7 @@ struct Struct : DatatypeConfig {
};
struct Array : DatatypeConfig {
- Array(TypeOrId nested_type) {
+ explicit Array(TypeOrId nested_type) {
addNestedType(nested_type);
type = Type::ARRAY;
array.element.id = nested_type.id;
@@ -71,7 +71,7 @@ struct Array : DatatypeConfig {
};
struct Wset : DatatypeConfig {
- Wset(TypeOrId nested_type) {
+ explicit Wset(TypeOrId nested_type) {
addNestedType(nested_type);
type = Type::WSET;
wset.key.id = nested_type.id;
@@ -94,7 +94,7 @@ struct Map : DatatypeConfig {
};
struct AnnotationRef : DatatypeConfig {
- AnnotationRef(int32_t annotation_type_id) {
+ explicit AnnotationRef(int32_t annotation_type_id) {
type = Type::ANNOTATIONREF;
annotationref.annotation.id = annotation_type_id;
}
@@ -103,7 +103,7 @@ struct AnnotationRef : DatatypeConfig {
struct DocTypeRep {
DocumenttypesConfig::Documenttype &doc_type;
- DocTypeRep(DocumenttypesConfig::Documenttype &type)
+ explicit DocTypeRep(DocumenttypesConfig::Documenttype &type)
: doc_type(type) {}
DocTypeRep &inherit(int32_t id) {
doc_type.inherits.resize(doc_type.inherits.size() + 1);
@@ -127,6 +127,12 @@ struct DocTypeRep {
doc_type.referencetype.back().targetTypeId = target_type_id;
return *this;
}
+
+ DocTypeRep& imported_field(vespalib::string field_name) {
+ doc_type.importedfield.resize(doc_type.importedfield.size() + 1);
+ doc_type.importedfield.back().name = std::move(field_name);
+ return *this;
+ }
};
class DocumenttypesConfigBuilderHelper {
diff --git a/document/src/vespa/document/repo/documenttyperepo.cpp b/document/src/vespa/document/repo/documenttyperepo.cpp
index da59f527115..58dbd16beb6 100644
--- a/document/src/vespa/document/repo/documenttyperepo.cpp
+++ b/document/src/vespa/document/repo/documenttyperepo.cpp
@@ -441,10 +441,10 @@ void addFieldSet(const DocumenttypesConfig::Documenttype::FieldsetsMap & fsv, Do
for (const auto & entry : fsv) {
const DocumenttypesConfig::Documenttype::Fieldsets & fs(entry.second);
DocumentType::FieldSet::Fields fields;
- for (size_t j(0); j < fs.fields.size(); j++) {
- fields.insert(fs.fields[j]);
+ for (const auto& f : fs.fields) {
+ fields.insert(f);
}
- doc_type.addFieldSet(entry.first, fields);
+ doc_type.addFieldSet(entry.first, std::move(fields));
}
}
@@ -457,6 +457,14 @@ void addReferenceTypes(const vector<DocumenttypesConfig::Documenttype::Reference
}
}
+void add_imported_fields(const DocumenttypesConfig::Documenttype::ImportedfieldVector& imported_fields,
+ DocumentType& doc_type)
+{
+ for (const auto& imported : imported_fields) {
+ doc_type.add_imported_field_name(imported.name);
+ }
+}
+
void configureDataTypeRepo(const DocumenttypesConfig::Documenttype &doc_type, DocumentTypeMap &type_map) {
DataTypeRepo *data_types = type_map[doc_type.id];
inheritAnnotationTypes(doc_type.inherits, type_map, data_types->annotations);
@@ -467,6 +475,7 @@ void configureDataTypeRepo(const DocumenttypesConfig::Documenttype &doc_type, Do
setAnnotationDataTypes(doc_type.annotationtype, data_types->annotations, data_types->repo);
inheritDocumentTypes(doc_type.inherits, type_map, *data_types->doc_type);
addFieldSet(doc_type.fieldsets, *data_types->doc_type);
+ add_imported_fields(doc_type.importedfield, *data_types->doc_type);
}
void addDataTypeRepo(DataTypeRepo::UP data_types, DocumentTypeMap &doc_types) {
diff --git a/document/src/vespa/document/select/valuenodes.cpp b/document/src/vespa/document/select/valuenodes.cpp
index 5c7820b76d7..95cf2f4e7e5 100644
--- a/document/src/vespa/document/select/valuenodes.cpp
+++ b/document/src/vespa/document/select/valuenodes.cpp
@@ -340,10 +340,34 @@ FieldValueNode::initFieldPath(const DocumentType& type) const {
}
}
+namespace {
+
+bool looks_like_complex_field_path(const vespalib::string& expr) {
+ for (const char c : expr) {
+ switch (c) {
+ case '.':
+ case '[':
+ case '{':
+ return true;
+ default: continue;
+ }
+ }
+ return false;
+}
+
+bool is_simple_imported_field(const vespalib::string& expr, const DocumentType& doc_type) {
+ if (looks_like_complex_field_path(expr)) {
+ return false;
+ }
+ return (doc_type.has_imported_field_name(expr));
+}
+
+}
+
std::unique_ptr<Value>
FieldValueNode::getValue(const Context& context) const
{
- if (context._doc == NULL) {
+ if (context._doc == nullptr) {
return std::make_unique<InvalidValue>();
}
@@ -352,7 +376,17 @@ FieldValueNode::getValue(const Context& context) const
if (!documentTypeEqualsName(doc.getType(), _doctype)) {
return std::make_unique<InvalidValue>();
}
- try{
+ // Imported fields can only be meaningfully evaluated inside Proton, so we
+ // explicitly treat them as if they are valid fields with missing values. This
+ // will be treated the same as if it's a normal field by the selection operators.
+ // This avoids any awkward interaction with Invalid values or having to
+ // augment the FieldPath code with knowledge of imported fields.
+ // When a selection is running inside Proton, it will patch FieldValueNodes for
+ // imported fields, which removes this check entirely.
+ if (is_simple_imported_field(_fieldExpression, doc.getType())) {
+ return std::make_unique<NullValue>();
+ }
+ try {
initFieldPath(doc.getType());
IteratorHandler handler;
@@ -363,7 +397,7 @@ FieldValueNode::getValue(const Context& context) const
} else {
const std::vector<ArrayValue::VariableValue>& values = handler.getValues();
- if (values.size() == 0) {
+ if (values.empty()) {
return std::make_unique<NullValue>();
} else {
return std::make_unique<ArrayValue>(handler.getValues());
@@ -399,7 +433,7 @@ FieldValueNode::print(std::ostream& out, bool verbose,
std::unique_ptr<Value>
FieldValueNode::traceValue(const Context &context, std::ostream& out) const
{
- if (context._doc == NULL) {
+ if (context._doc == nullptr) {
return defaultTrace(getValue(context), out);
}
const Document &doc(*context._doc);
@@ -408,7 +442,12 @@ FieldValueNode::traceValue(const Context &context, std::ostream& out) const
<< _doctype << " document, thus resolving invalid.\n";
return std::make_unique<InvalidValue>();
}
- try{
+ if (is_simple_imported_field(_fieldExpression, doc.getType())) {
+ out << "Field '" << _fieldExpression << "' refers to an imported field; "
+ << "returning NullValue to treat this as an unset field value.\n";
+ return std::make_unique<NullValue>();
+ }
+ try {
initFieldPath(doc.getType());
IteratorHandler handler;
diff --git a/eval/src/tests/tensor/dense_xw_product_function/dense_xw_product_function_test.cpp b/eval/src/tests/tensor/dense_xw_product_function/dense_xw_product_function_test.cpp
index 36609c04219..0b924451907 100644
--- a/eval/src/tests/tensor/dense_xw_product_function/dense_xw_product_function_test.cpp
+++ b/eval/src/tests/tensor/dense_xw_product_function/dense_xw_product_function_test.cpp
@@ -78,9 +78,9 @@ void verify_optimized(const vespalib::string &expr, size_t vec_size, size_t res_
auto info = fixture.find_all<DenseXWProductFunction>();
ASSERT_EQUAL(info.size(), 1u);
EXPECT_TRUE(info[0]->result_is_mutable());
- EXPECT_EQUAL(info[0]->vectorSize(), vec_size);
- EXPECT_EQUAL(info[0]->resultSize(), res_size);
- EXPECT_EQUAL(info[0]->matrixHasCommonDimensionInnermost(), happy);
+ EXPECT_EQUAL(info[0]->vector_size(), vec_size);
+ EXPECT_EQUAL(info[0]->result_size(), res_size);
+ EXPECT_EQUAL(info[0]->common_inner(), happy);
}
vespalib::string make_expr(const vespalib::string &a, const vespalib::string &b, const vespalib::string &common,
diff --git a/eval/src/vespa/eval/eval/test/eval_fixture.cpp b/eval/src/vespa/eval/eval/test/eval_fixture.cpp
index e578b28da18..1e17c8284cb 100644
--- a/eval/src/vespa/eval/eval/test/eval_fixture.cpp
+++ b/eval/src/vespa/eval/eval/test/eval_fixture.cpp
@@ -83,6 +83,19 @@ std::vector<Value::CREF> get_refs(const std::vector<Value::UP> &values) {
} // namespace vespalib::eval::test
+void
+EvalFixture::detect_param_tampering(const ParamRepo &param_repo, bool allow_mutable) const
+{
+ for (size_t i = 0; i < _function->num_params(); ++i) {
+ auto pos = param_repo.map.find(_function->param_name(i));
+ ASSERT_TRUE(pos != param_repo.map.end());
+ bool allow_tampering = allow_mutable && pos->second.is_mutable;
+ if (!allow_tampering) {
+ ASSERT_EQUAL(pos->second.value, _engine.to_spec(*_param_values[i]));
+ }
+ }
+}
+
EvalFixture::EvalFixture(const TensorEngine &engine,
const vespalib::string &expr,
const ParamRepo &param_repo,
@@ -104,6 +117,7 @@ EvalFixture::EvalFixture(const TensorEngine &engine,
{
auto result_type = ValueType::from_spec(_result.type());
ASSERT_TRUE(!result_type.is_error());
+ TEST_DO(detect_param_tampering(param_repo, allow_mutable));
}
const TensorSpec
diff --git a/eval/src/vespa/eval/eval/test/eval_fixture.h b/eval/src/vespa/eval/eval/test/eval_fixture.h
index 48f6a7e5d2e..1d39dc52cba 100644
--- a/eval/src/vespa/eval/eval/test/eval_fixture.h
+++ b/eval/src/vespa/eval/eval/test/eval_fixture.h
@@ -67,6 +67,8 @@ private:
}
}
+ void detect_param_tampering(const ParamRepo &param_repo, bool allow_mutable) const;
+
public:
EvalFixture(const TensorEngine &engine, const vespalib::string &expr, const ParamRepo &param_repo,
bool optimized = true, bool allow_mutable = false);
diff --git a/eval/src/vespa/eval/tensor/dense/dense_dot_product_function.cpp b/eval/src/vespa/eval/tensor/dense/dense_dot_product_function.cpp
index 8bcaddba3b4..f2c374ea02d 100644
--- a/eval/src/vespa/eval/tensor/dense/dense_dot_product_function.cpp
+++ b/eval/src/vespa/eval/tensor/dense/dense_dot_product_function.cpp
@@ -7,6 +7,8 @@
#include <vespa/eval/eval/value.h>
#include <vespa/eval/tensor/tensor.h>
+#include <cblas.h>
+
namespace vespalib::tensor {
using eval::ValueType;
@@ -19,32 +21,29 @@ using namespace eval::operation;
namespace {
template <typename LCT, typename RCT>
-struct HWSupport {
- static double call(hwaccelrated::IAccelrated *, const ConstArrayRef<LCT> &lhs, const ConstArrayRef<RCT> &rhs) {
- double result = 0.0;
- for (size_t i = 0; i < lhs.size(); ++i) {
- result += (lhs[i] * rhs[i]);
- }
- return result;
+void my_dot_product_op(eval::InterpretedFunction::State &state, uint64_t) {
+ auto lhs_cells = DenseTensorView::typify_cells<LCT>(state.peek(1));
+ auto rhs_cells = DenseTensorView::typify_cells<RCT>(state.peek(0));
+ double result = 0.0;
+ const LCT *lhs = lhs_cells.cbegin();
+ const RCT *rhs = rhs_cells.cbegin();
+ for (size_t i = 0; i < lhs_cells.size(); ++i) {
+ result += ((*lhs++) * (*rhs++));
}
-};
-template <> struct HWSupport<float, float> {
- static double call(hwaccelrated::IAccelrated *hw, const ConstArrayRef<float> &lhs, const ConstArrayRef<float> &rhs) {
- return hw->dotProduct(lhs.cbegin(), rhs.cbegin(), lhs.size());
- }
-};
-template <> struct HWSupport<double, double> {
- static double call(hwaccelrated::IAccelrated *hw, const ConstArrayRef<double> &lhs, const ConstArrayRef<double> &rhs) {
- return hw->dotProduct(lhs.cbegin(), rhs.cbegin(), lhs.size());
- }
-};
+ state.pop_pop_push(state.stash.create<eval::DoubleValue>(result));
+}
-template <typename LCT, typename RCT>
-void my_dot_product_op(eval::InterpretedFunction::State &state, uint64_t param) {
- auto *hw = (hwaccelrated::IAccelrated *)(param);
- auto lhs = DenseTensorView::typify_cells<LCT>(state.peek(1));
- auto rhs = DenseTensorView::typify_cells<RCT>(state.peek(0));
- double result = HWSupport<LCT,RCT>::call(hw, lhs, rhs);
+void my_cblas_double_dot_product_op(eval::InterpretedFunction::State &state, uint64_t) {
+ auto lhs_cells = DenseTensorView::typify_cells<double>(state.peek(1));
+ auto rhs_cells = DenseTensorView::typify_cells<double>(state.peek(0));
+ double result = cblas_ddot(lhs_cells.size(), lhs_cells.cbegin(), 1, rhs_cells.cbegin(), 1);
+ state.pop_pop_push(state.stash.create<eval::DoubleValue>(result));
+}
+
+void my_cblas_float_dot_product_op(eval::InterpretedFunction::State &state, uint64_t) {
+ auto lhs_cells = DenseTensorView::typify_cells<float>(state.peek(1));
+ auto rhs_cells = DenseTensorView::typify_cells<float>(state.peek(0));
+ double result = cblas_sdot(lhs_cells.size(), lhs_cells.cbegin(), 1, rhs_cells.cbegin(), 1);
state.pop_pop_push(state.stash.create<eval::DoubleValue>(result));
}
@@ -53,21 +52,31 @@ struct MyDotProductOp {
static auto get_fun() { return my_dot_product_op<LCT,RCT>; }
};
+eval::InterpretedFunction::op_function my_select(CellType lct, CellType rct) {
+ if (lct == rct) {
+ if (lct == ValueType::CellType::DOUBLE) {
+ return my_cblas_double_dot_product_op;
+ }
+ if (lct == ValueType::CellType::FLOAT) {
+ return my_cblas_float_dot_product_op;
+ }
+ }
+ return select_2<MyDotProductOp>(lct, rct);
+}
+
} // namespace vespalib::tensor::<unnamed>
DenseDotProductFunction::DenseDotProductFunction(const eval::TensorFunction &lhs_in,
const eval::TensorFunction &rhs_in)
- : eval::tensor_function::Op2(eval::ValueType::double_type(), lhs_in, rhs_in),
- _hwAccelerator(hwaccelrated::IAccelrated::getAccelrator())
+ : eval::tensor_function::Op2(eval::ValueType::double_type(), lhs_in, rhs_in)
{
}
eval::InterpretedFunction::Instruction
DenseDotProductFunction::compile_self(Stash &) const
{
- auto op = select_2<MyDotProductOp>(lhs().result_type().cell_type(),
- rhs().result_type().cell_type());
- return eval::InterpretedFunction::Instruction(op, (uint64_t)(_hwAccelerator.get()));
+ auto op = my_select(lhs().result_type().cell_type(), rhs().result_type().cell_type());
+ return eval::InterpretedFunction::Instruction(op);
}
bool
diff --git a/eval/src/vespa/eval/tensor/dense/dense_dot_product_function.h b/eval/src/vespa/eval/tensor/dense/dense_dot_product_function.h
index d6181d33887..1d8f749689b 100644
--- a/eval/src/vespa/eval/tensor/dense/dense_dot_product_function.h
+++ b/eval/src/vespa/eval/tensor/dense/dense_dot_product_function.h
@@ -3,7 +3,6 @@
#pragma once
#include <vespa/eval/eval/tensor_function.h>
-#include <vespa/vespalib/hwaccelrated/iaccelrated.h>
namespace vespalib::tensor {
@@ -13,7 +12,6 @@ namespace vespalib::tensor {
class DenseDotProductFunction : public eval::tensor_function::Op2
{
private:
- hwaccelrated::IAccelrated::UP _hwAccelerator;
using ValueType = eval::ValueType;
public:
DenseDotProductFunction(const eval::TensorFunction &lhs_in,
diff --git a/eval/src/vespa/eval/tensor/dense/dense_matmul_function.cpp b/eval/src/vespa/eval/tensor/dense/dense_matmul_function.cpp
index ce0d4230c23..272ceb319c8 100644
--- a/eval/src/vespa/eval/tensor/dense/dense_matmul_function.cpp
+++ b/eval/src/vespa/eval/tensor/dense/dense_matmul_function.cpp
@@ -10,7 +10,7 @@
#include <vespa/eval/tensor/tensor.h>
#include <assert.h>
-#include <openblas/cblas.h>
+#include <cblas.h>
namespace vespalib::tensor {
diff --git a/eval/src/vespa/eval/tensor/dense/dense_xw_product_function.cpp b/eval/src/vespa/eval/tensor/dense/dense_xw_product_function.cpp
index 2db5b4e8f92..7b03f45e9cc 100644
--- a/eval/src/vespa/eval/tensor/dense/dense_xw_product_function.cpp
+++ b/eval/src/vespa/eval/tensor/dense/dense_xw_product_function.cpp
@@ -10,6 +10,8 @@
#include <vespa/vespalib/util/exceptions.h>
#include <assert.h>
+#include <cblas.h>
+
namespace vespalib::tensor {
using eval::ValueType;
@@ -21,76 +23,59 @@ using namespace eval::operation;
namespace {
-template <typename LCT, typename RCT>
-struct HWSupport {
- static double call(hwaccelrated::IAccelrated *, const LCT *lhs, const RCT *rhs, size_t len) {
- double result = 0.0;
- for (size_t i = 0; i < len; ++i) {
- result += (lhs[i] * rhs[i]);
- }
- return result;
+template <typename LCT, typename RCT, bool common_inner>
+double my_dot_product(const LCT *lhs, const RCT *rhs, size_t vector_size, size_t result_size) {
+ double result = 0.0;
+ for (size_t i = 0; i < vector_size; ++i) {
+ result += ((*lhs) * (*rhs));
+ ++lhs;
+ rhs += (common_inner ? 1 : result_size);
}
-};
-template <> struct HWSupport<float, float> {
- static double call(hwaccelrated::IAccelrated *hw, const float *lhs, const float *rhs, size_t len) {
- return hw->dotProduct(lhs, rhs, len);
- }
-};
-template <> struct HWSupport<double, double> {
- static double call(hwaccelrated::IAccelrated *hw, const double *lhs, const double *rhs, size_t len) {
- return hw->dotProduct(lhs, rhs, len);
- }
-};
-
-template <typename LCT, typename RCT, typename OCT>
-void multiDotProduct(const DenseXWProductFunction::Self &self,
- const ConstArrayRef<LCT> &vectorCells, const ConstArrayRef<RCT> &matrixCells, ArrayRef<OCT> &result)
-{
- OCT *out = result.begin();
- const RCT *matrixP = matrixCells.cbegin();
- const LCT * const vectorP = vectorCells.cbegin();
- for (size_t row = 0; row < self._resultSize; ++row) {
- double cell = HWSupport<LCT,RCT>::call(self._hwAccelerator.get(), vectorP, matrixP, self._vectorSize);
- *out++ = cell;
- matrixP += self._vectorSize;
- }
- assert(out == result.end());
- assert(matrixP == matrixCells.cend());
-}
-
-template <typename LCT, typename RCT, typename OCT>
-void transposedProduct(const DenseXWProductFunction::Self &self,
- const ConstArrayRef<LCT> &vectorCells, const ConstArrayRef<RCT> &matrixCells, ArrayRef<OCT> &result)
-{
- OCT *out = result.begin();
- const RCT * const matrixP = matrixCells.cbegin();
- const LCT * const vectorP = vectorCells.cbegin();
- for (size_t row = 0; row < self._resultSize; ++row) {
- double cell = 0;
- for (size_t col = 0; col < self._vectorSize; ++col) {
- cell += matrixP[col*self._resultSize + row] * vectorP[col];
- }
- *out++ = cell;
- }
- assert(out == result.end());
+ return result;
}
-template <typename LCT, typename RCT, bool commonDimensionInnermost>
+template <typename LCT, typename RCT, bool common_inner>
void my_xw_product_op(eval::InterpretedFunction::State &state, uint64_t param) {
- DenseXWProductFunction::Self *self = (DenseXWProductFunction::Self *)(param);
-
+ const DenseXWProductFunction::Self &self = *((const DenseXWProductFunction::Self *)(param));
using OCT = typename eval::UnifyCellTypes<LCT,RCT>::type;
- auto vectorCells = DenseTensorView::typify_cells<LCT>(state.peek(1));
- auto matrixCells = DenseTensorView::typify_cells<RCT>(state.peek(0));
- auto outputCells = state.stash.create_array<OCT>(self->_resultSize);
-
- if (commonDimensionInnermost) {
- multiDotProduct(*self, vectorCells, matrixCells, outputCells);
- } else {
- transposedProduct(*self, vectorCells, matrixCells, outputCells);
+ auto vector_cells = DenseTensorView::typify_cells<LCT>(state.peek(1));
+ auto matrix_cells = DenseTensorView::typify_cells<RCT>(state.peek(0));
+ auto dst_cells = state.stash.create_array<OCT>(self.result_size);
+ OCT *dst = dst_cells.begin();
+ const RCT *matrix = matrix_cells.cbegin();
+ for (size_t i = 0; i < self.result_size; ++i) {
+ *dst++ = my_dot_product<LCT,RCT,common_inner>(vector_cells.cbegin(), matrix, self.vector_size, self.result_size);
+ matrix += (common_inner ? self.vector_size : 1);
}
+ state.pop_pop_push(state.stash.create<DenseTensorView>(self.result_type, TypedCells(dst_cells)));
+}
+
+template <bool common_inner>
+void my_cblas_double_xw_product_op(eval::InterpretedFunction::State &state, uint64_t param) {
+ const DenseXWProductFunction::Self &self = *((const DenseXWProductFunction::Self *)(param));
+ auto vector_cells = DenseTensorView::typify_cells<double>(state.peek(1));
+ auto matrix_cells = DenseTensorView::typify_cells<double>(state.peek(0));
+ auto dst_cells = state.stash.create_array<double>(self.result_size);
+ cblas_dgemv(CblasRowMajor, common_inner ? CblasNoTrans : CblasTrans,
+ common_inner ? self.result_size : self.vector_size,
+ common_inner ? self.vector_size : self.result_size,
+ 1.0, matrix_cells.cbegin(), common_inner ? self.vector_size : self.result_size, vector_cells.cbegin(), 1,
+ 0.0, dst_cells.begin(), 1);
+ state.pop_pop_push(state.stash.create<DenseTensorView>(self.result_type, TypedCells(dst_cells)));
+}
- state.pop_pop_push(state.stash.create<DenseTensorView>(self->_resultType, TypedCells(outputCells)));
+template <bool common_inner>
+void my_cblas_float_xw_product_op(eval::InterpretedFunction::State &state, uint64_t param) {
+ const DenseXWProductFunction::Self &self = *((const DenseXWProductFunction::Self *)(param));
+ auto vector_cells = DenseTensorView::typify_cells<float>(state.peek(1));
+ auto matrix_cells = DenseTensorView::typify_cells<float>(state.peek(0));
+ auto dst_cells = state.stash.create_array<float>(self.result_size);
+ cblas_sgemv(CblasRowMajor, common_inner ? CblasNoTrans : CblasTrans,
+ common_inner ? self.result_size : self.vector_size,
+ common_inner ? self.vector_size : self.result_size,
+ 1.0, matrix_cells.cbegin(), common_inner ? self.vector_size : self.result_size, vector_cells.cbegin(), 1,
+ 0.0, dst_cells.begin(), 1);
+ state.pop_pop_push(state.stash.create<DenseTensorView>(self.result_type, TypedCells(dst_cells)));
}
template <bool common_inner>
@@ -99,11 +84,24 @@ struct MyXWProductOp {
static auto get_fun() { return my_xw_product_op<LCT,RCT,common_inner>; }
};
-eval::InterpretedFunction::op_function my_select(CellType lct, CellType rct, bool common_innermost) {
- if (common_innermost) {
- return select_2<MyXWProductOp<true> >(lct, rct);
+template <bool common_inner>
+eval::InterpretedFunction::op_function my_select2(CellType lct, CellType rct) {
+ if (lct == rct) {
+ if (lct == ValueType::CellType::DOUBLE) {
+ return my_cblas_double_xw_product_op<common_inner>;
+ }
+ if (lct == ValueType::CellType::FLOAT) {
+ return my_cblas_float_xw_product_op<common_inner>;
+ }
+ }
+ return select_2<MyXWProductOp<common_inner>>(lct, rct);
+}
+
+eval::InterpretedFunction::op_function my_select(CellType lct, CellType rct, bool common_inner) {
+ if (common_inner) {
+ return my_select2<true>(lct, rct);
} else {
- return select_2<MyXWProductOp<false> >(lct, rct);
+ return my_select2<false>(lct, rct);
}
}
@@ -129,42 +127,43 @@ bool isDenseXWProduct(const ValueType &res, const ValueType &vec, const ValueTyp
}
const TensorFunction &createDenseXWProduct(const ValueType &res, const TensorFunction &vec, const TensorFunction &mat, Stash &stash) {
- bool common_is_inner = (mat.result_type().dimension_index(vec.result_type().dimensions()[0].name) == 1);
+ bool common_inner = (mat.result_type().dimension_index(vec.result_type().dimensions()[0].name) == 1);
return stash.create<DenseXWProductFunction>(res, vec, mat,
vec.result_type().dimensions()[0].size,
res.dimensions()[0].size,
- common_is_inner);
+ common_inner);
}
} // namespace vespalib::tensor::<unnamed>
-DenseXWProductFunction::Self::Self(const eval::ValueType &resultType,
- size_t vectorSize,
- size_t resultSize)
- : _resultType(resultType),
- _vectorSize(vectorSize),
- _resultSize(resultSize),
- _hwAccelerator(hwaccelrated::IAccelrated::getAccelrator())
-{}
+DenseXWProductFunction::Self::Self(const eval::ValueType &result_type_in,
+ size_t vector_size_in, size_t result_size_in)
+ : result_type(result_type_in),
+ vector_size(vector_size_in),
+ result_size(result_size_in)
+{
+}
+DenseXWProductFunction::Self::~Self() = default;
-DenseXWProductFunction::DenseXWProductFunction(const eval::ValueType &resultType,
+DenseXWProductFunction::DenseXWProductFunction(const eval::ValueType &result_type,
const eval::TensorFunction &vector_in,
const eval::TensorFunction &matrix_in,
- size_t vectorSize,
- size_t resultSize,
- bool matrixHasCommonDimensionInnermost)
- : eval::tensor_function::Op2(resultType, vector_in, matrix_in),
- _vectorSize(vectorSize),
- _resultSize(resultSize),
- _commonDimensionInnermost(matrixHasCommonDimensionInnermost)
-{}
+ size_t vector_size,
+ size_t result_size,
+ bool common_inner)
+ : eval::tensor_function::Op2(result_type, vector_in, matrix_in),
+ _vector_size(vector_size),
+ _result_size(result_size),
+ _common_inner(common_inner)
+{
+}
eval::InterpretedFunction::Instruction
DenseXWProductFunction::compile_self(Stash &stash) const
{
- Self &self = stash.create<Self>(result_type(), _vectorSize, _resultSize);
+ Self &self = stash.create<Self>(result_type(), _vector_size, _result_size);
auto op = my_select(lhs().result_type().cell_type(),
- rhs().result_type().cell_type(), _commonDimensionInnermost);
+ rhs().result_type().cell_type(), _common_inner);
return eval::InterpretedFunction::Instruction(op, (uint64_t)(&self));
}
@@ -172,9 +171,9 @@ void
DenseXWProductFunction::visit_self(vespalib::ObjectVisitor &visitor) const
{
Super::visit_self(visitor);
- visitor.visitInt("vector_size", _vectorSize);
- visitor.visitInt("result_size", _resultSize);
- visitor.visitBool("common_dimension_innermost", _commonDimensionInnermost);
+ visitor.visitInt("vector_size", _vector_size);
+ visitor.visitInt("result_size", _result_size);
+ visitor.visitBool("common_inner", _common_inner);
}
const TensorFunction &
diff --git a/eval/src/vespa/eval/tensor/dense/dense_xw_product_function.h b/eval/src/vespa/eval/tensor/dense/dense_xw_product_function.h
index f2f4d67c0f0..d7c39fa45a2 100644
--- a/eval/src/vespa/eval/tensor/dense/dense_xw_product_function.h
+++ b/eval/src/vespa/eval/tensor/dense/dense_xw_product_function.h
@@ -4,7 +4,6 @@
#include <vespa/eval/eval/tensor_function.h>
#include "dense_tensor_view.h"
-#include <vespa/vespalib/hwaccelrated/iaccelrated.h>
namespace vespalib::tensor {
@@ -16,37 +15,34 @@ class DenseXWProductFunction : public eval::tensor_function::Op2
using Super = eval::tensor_function::Op2;
public:
struct Self {
- const eval::ValueType _resultType;
- const size_t _vectorSize;
- const size_t _resultSize;
- hwaccelrated::IAccelrated::UP _hwAccelerator;
- Self(const eval::ValueType &resultType,
- size_t vectorSize,
- size_t resultSize);
- ~Self() {}
+ eval::ValueType result_type;
+ size_t vector_size;
+ size_t result_size;
+ Self(const eval::ValueType &result_type_in,
+ size_t vector_size_in, size_t result_size_in);
+ ~Self();
};
private:
- const size_t _vectorSize;
- const size_t _resultSize;
- bool _commonDimensionInnermost;
+ size_t _vector_size;
+ size_t _result_size;
+ bool _common_inner;
public:
- DenseXWProductFunction(const eval::ValueType &resultType,
+ DenseXWProductFunction(const eval::ValueType &result_type,
const eval::TensorFunction &vector_in,
const eval::TensorFunction &matrix_in,
- size_t vectorSize,
- size_t resultSize,
- bool matrixHasCommonDimensionInnermost);
+ size_t vector_size,
+ size_t result_size,
+ bool common_inner);
~DenseXWProductFunction() {}
bool result_is_mutable() const override { return true; }
- size_t vectorSize() const { return _vectorSize; }
- size_t resultSize() const { return _resultSize; }
-
- bool matrixHasCommonDimensionInnermost() const { return _commonDimensionInnermost; }
+ size_t vector_size() const { return _vector_size; }
+ size_t result_size() const { return _result_size; }
+ bool common_inner() const { return _common_inner; }
eval::InterpretedFunction::Instruction compile_self(Stash &stash) const override;
void visit_self(vespalib::ObjectVisitor &visitor) const override;
diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java
index 5edab66e6b3..5bda3559ccf 100644
--- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java
+++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java
@@ -191,12 +191,6 @@ public class Flags {
"Takes effect immediately",
ZONE_ID);
- public static final UnboundBooleanFlag INSTALL_L4_ROUTING_SUPPORT = defineFeatureFlag(
- "install-l4-routing-support", false,
- "Whether routing nodes should install package supporting L4 routing",
- "Takes effect immediately",
- ZONE_ID, HOSTNAME);
-
public static final UnboundBooleanFlag GENERATE_L4_ROUTING_CONFIG = defineFeatureFlag(
"generate-l4-routing-config", false,
"Whether routing nodes should generate L4 routing config",
@@ -215,6 +209,12 @@ public class Flags {
"Takes effect on the next deployment of the application",
APPLICATION_ID);
+ public static final UnboundBooleanFlag USE_NEW_ATHENZ_FILTER = defineFeatureFlag(
+ "use-new-athenz-filter", false,
+ "Use new Athenz filter that supports access-tokens",
+ "Takes effect at redeployment",
+ APPLICATION_ID);
+
/** WARNING: public for testing: All flags should be defined in {@link Flags}. */
public static UnboundBooleanFlag defineFeatureFlag(String flagId, boolean defaultValue, String description,
diff --git a/fnet/src/vespa/fnet/connection.cpp b/fnet/src/vespa/fnet/connection.cpp
index 74a7d19387c..5f7adb32af0 100644
--- a/fnet/src/vespa/fnet/connection.cpp
+++ b/fnet/src/vespa/fnet/connection.cpp
@@ -13,6 +13,8 @@
#include <vespa/log/log.h>
LOG_SETUP(".fnet");
+std::atomic<uint64_t> FNET_Connection::_num_connections = 0;
+
namespace {
class SyncPacket : public FNET_DummyPacket {
private:
@@ -489,6 +491,7 @@ FNET_Connection::FNET_Connection(FNET_TransportThread *owner,
_cleanup(nullptr)
{
assert(_socket && (_socket->get_fd() >= 0));
+ _num_connections.fetch_add(1, std::memory_order_relaxed);
}
@@ -526,6 +529,7 @@ FNET_Connection::FNET_Connection(FNET_TransportThread *owner,
_adminChannel = admin.get();
_channels.Register(admin.release());
}
+ _num_connections.fetch_add(1, std::memory_order_relaxed);
}
@@ -536,6 +540,7 @@ FNET_Connection::~FNET_Connection()
delete _adminChannel;
}
assert(_cleanup == nullptr);
+ _num_connections.fetch_sub(1, std::memory_order_relaxed);
}
diff --git a/fnet/src/vespa/fnet/connection.h b/fnet/src/vespa/fnet/connection.h
index 760b1b96d4d..c9c49c5151a 100644
--- a/fnet/src/vespa/fnet/connection.h
+++ b/fnet/src/vespa/fnet/connection.h
@@ -10,6 +10,7 @@
#include <vespa/vespalib/net/socket_handle.h>
#include <vespa/vespalib/net/async_resolver.h>
#include <vespa/vespalib/net/crypto_socket.h>
+#include <atomic>
class FNET_IPacketStreamer;
class FNET_IServerAdapter;
@@ -111,6 +112,8 @@ private:
FNET_IConnectionCleanupHandler *_cleanup; // cleanup handler
+ static std::atomic<uint64_t> _num_connections; // total number of connections
+
FNET_Connection(const FNET_Connection &);
FNET_Connection &operator=(const FNET_Connection &);
@@ -517,5 +520,10 @@ public:
*/
uint32_t getInputBufferSize() const { return _input.GetBufSize(); }
+ /**
+ * @return the total number of connection objects
+ **/
+ static uint64_t get_num_connections() {
+ return _num_connections.load(std::memory_order_relaxed);
+ }
};
-
diff --git a/jdisc-security-filters/src/main/java/com/yahoo/jdisc/http/filter/security/athenz/RequestResourceMapper.java b/jdisc-security-filters/src/main/java/com/yahoo/jdisc/http/filter/security/athenz/RequestResourceMapper.java
index 77709975cba..0bf000efc00 100644
--- a/jdisc-security-filters/src/main/java/com/yahoo/jdisc/http/filter/security/athenz/RequestResourceMapper.java
+++ b/jdisc-security-filters/src/main/java/com/yahoo/jdisc/http/filter/security/athenz/RequestResourceMapper.java
@@ -33,5 +33,13 @@ public interface RequestResourceMapper {
public String action() {
return action;
}
+
+ @Override
+ public String toString() {
+ return "ResourceNameAndAction{" +
+ "resourceName=" + resourceName +
+ ", action='" + action + '\'' +
+ '}';
+ }
}
}
diff --git a/jdisc-security-filters/src/main/java/com/yahoo/jdisc/http/filter/security/base/JsonSecurityRequestFilterBase.java b/jdisc-security-filters/src/main/java/com/yahoo/jdisc/http/filter/security/base/JsonSecurityRequestFilterBase.java
index ec8a93019b0..e654182d665 100644
--- a/jdisc-security-filters/src/main/java/com/yahoo/jdisc/http/filter/security/base/JsonSecurityRequestFilterBase.java
+++ b/jdisc-security-filters/src/main/java/com/yahoo/jdisc/http/filter/security/base/JsonSecurityRequestFilterBase.java
@@ -10,9 +10,11 @@ import com.yahoo.jdisc.handler.ResponseDispatch;
import com.yahoo.jdisc.handler.ResponseHandler;
import com.yahoo.jdisc.http.filter.DiscFilterRequest;
import com.yahoo.jdisc.http.filter.SecurityRequestFilter;
+import com.yahoo.log.LogLevel;
import java.io.UncheckedIOException;
import java.util.Optional;
+import java.util.logging.Logger;
/**
* A base class for {@link SecurityRequestFilter} implementations that renders an error response as JSON.
@@ -21,22 +23,25 @@ import java.util.Optional;
*/
public abstract class JsonSecurityRequestFilterBase implements SecurityRequestFilter {
+ private static final Logger log = Logger.getLogger(JsonSecurityRequestFilterBase.class.getName());
private static final ObjectMapper mapper = new ObjectMapper();
@Override
public final void filter(DiscFilterRequest request, ResponseHandler handler) {
filter(request)
- .ifPresent(errorResponse -> writeResponse(errorResponse, handler));
+ .ifPresent(errorResponse -> writeResponse(request, errorResponse, handler));
}
protected abstract Optional<ErrorResponse> filter(DiscFilterRequest request);
- private void writeResponse(ErrorResponse error, ResponseHandler responseHandler) {
+ private void writeResponse(DiscFilterRequest request, ErrorResponse error, ResponseHandler responseHandler) {
ObjectNode errorMessage = mapper.createObjectNode();
errorMessage.put("code", error.errorCode);
errorMessage.put("message", error.message);
error.response.headers().put("Content-Type", "application/json"); // Note: Overwrites header if already exists
error.response.headers().put("Cache-Control", "must-revalidate,no-cache,no-store");
+ log.log(LogLevel.DEBUG, () -> String.format("Error response for '%s': statusCode=%d, errorCode=%d, message='%s'",
+ request, error.response.getStatus(), error.errorCode, error.message));
try (FastContentWriter writer = ResponseDispatch.newInstance(error.response).connectFastWriter(responseHandler)) {
writer.write(mapper.writerWithDefaultPrettyPrinter().writeValueAsString(errorMessage));
} catch (JsonProcessingException e) {
diff --git a/jdisc_http_service/abi-spec.json b/jdisc_http_service/abi-spec.json
index 1615cf7e686..508a6a84974 100644
--- a/jdisc_http_service/abi-spec.json
+++ b/jdisc_http_service/abi-spec.json
@@ -119,9 +119,16 @@
"public com.yahoo.jdisc.http.ConnectorConfig$Ssl$Builder caCertificateFile(java.lang.String)",
"public com.yahoo.jdisc.http.ConnectorConfig$Ssl$Builder caCertificate(java.lang.String)",
"public com.yahoo.jdisc.http.ConnectorConfig$Ssl$Builder clientAuth(com.yahoo.jdisc.http.ConnectorConfig$Ssl$ClientAuth$Enum)",
+ "public com.yahoo.jdisc.http.ConnectorConfig$Ssl$Builder enabledCipherSuites(java.lang.String)",
+ "public com.yahoo.jdisc.http.ConnectorConfig$Ssl$Builder enabledCipherSuites(java.util.Collection)",
+ "public com.yahoo.jdisc.http.ConnectorConfig$Ssl$Builder enabledProtocols(java.lang.String)",
+ "public com.yahoo.jdisc.http.ConnectorConfig$Ssl$Builder enabledProtocols(java.util.Collection)",
"public com.yahoo.jdisc.http.ConnectorConfig$Ssl build()"
],
- "fields": []
+ "fields": [
+ "public java.util.List enabledCipherSuites",
+ "public java.util.List enabledProtocols"
+ ]
},
"com.yahoo.jdisc.http.ConnectorConfig$Ssl$ClientAuth$Enum": {
"superClass": "java.lang.Enum",
@@ -174,7 +181,11 @@
"public java.lang.String certificate()",
"public java.lang.String caCertificateFile()",
"public java.lang.String caCertificate()",
- "public com.yahoo.jdisc.http.ConnectorConfig$Ssl$ClientAuth$Enum clientAuth()"
+ "public com.yahoo.jdisc.http.ConnectorConfig$Ssl$ClientAuth$Enum clientAuth()",
+ "public java.util.List enabledCipherSuites()",
+ "public java.lang.String enabledCipherSuites(int)",
+ "public java.util.List enabledProtocols()",
+ "public java.lang.String enabledProtocols(int)"
],
"fields": []
},
diff --git a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/impl/ConfiguredSslContextFactoryProvider.java b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/impl/ConfiguredSslContextFactoryProvider.java
index 48a7c246500..90848f1dfd4 100644
--- a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/impl/ConfiguredSslContextFactoryProvider.java
+++ b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/impl/ConfiguredSslContextFactoryProvider.java
@@ -2,14 +2,16 @@
package com.yahoo.jdisc.http.ssl.impl;
import com.yahoo.jdisc.http.ConnectorConfig;
+import com.yahoo.jdisc.http.ConnectorConfig.Ssl.ClientAuth;
import com.yahoo.jdisc.http.ssl.SslContextFactoryProvider;
import com.yahoo.security.KeyUtils;
+import com.yahoo.security.SslContextBuilder;
import com.yahoo.security.X509CertificateUtils;
-import com.yahoo.security.tls.DefaultTlsContext;
-import com.yahoo.security.tls.PeerAuthentication;
+import com.yahoo.security.tls.AutoReloadingX509KeyManager;
import com.yahoo.security.tls.TlsContext;
import org.eclipse.jetty.util.ssl.SslContextFactory;
+import javax.net.ssl.SSLContext;
import java.io.IOException;
import java.io.UncheckedIOException;
import java.nio.charset.StandardCharsets;
@@ -17,16 +19,21 @@ import java.nio.file.Files;
import java.nio.file.Paths;
import java.security.PrivateKey;
import java.security.cert.X509Certificate;
+import java.util.ArrayList;
import java.util.List;
import java.util.Optional;
+import static com.yahoo.jdisc.http.ssl.impl.SslContextFactoryUtils.setEnabledCipherSuites;
+import static com.yahoo.jdisc.http.ssl.impl.SslContextFactoryUtils.setEnabledProtocols;
+
/**
* An implementation of {@link SslContextFactoryProvider} that uses the {@link ConnectorConfig} to construct a {@link SslContextFactory}.
*
* @author bjorncs
*/
-public class ConfiguredSslContextFactoryProvider extends TlsContextBasedProvider {
+public class ConfiguredSslContextFactoryProvider implements SslContextFactoryProvider {
+ private volatile AutoReloadingX509KeyManager keyManager;
private final ConnectorConfig connectorConfig;
public ConfiguredSslContextFactoryProvider(ConnectorConfig connectorConfig) {
@@ -35,17 +42,50 @@ public class ConfiguredSslContextFactoryProvider extends TlsContextBasedProvider
}
@Override
- protected TlsContext getTlsContext(String containerId, int port) {
+ public SslContextFactory getInstance(String containerId, int port) {
ConnectorConfig.Ssl sslConfig = connectorConfig.ssl();
if (!sslConfig.enabled()) throw new IllegalStateException();
- PrivateKey privateKey = KeyUtils.fromPemEncodedPrivateKey(getPrivateKey(sslConfig));
- List<X509Certificate> certificates = X509CertificateUtils.certificateListFromPem(getCertificate(sslConfig));
+ SslContextBuilder builder = new SslContextBuilder();
+ if (sslConfig.certificateFile().isBlank() || sslConfig.privateKeyFile().isBlank()) {
+ PrivateKey privateKey = KeyUtils.fromPemEncodedPrivateKey(getPrivateKey(sslConfig));
+ List<X509Certificate> certificates = X509CertificateUtils.certificateListFromPem(getCertificate(sslConfig));
+ builder.withKeyStore(privateKey, certificates);
+ } else {
+ keyManager = AutoReloadingX509KeyManager.fromPemFiles(Paths.get(sslConfig.privateKeyFile()), Paths.get(sslConfig.certificateFile()));
+ builder.withKeyManager(keyManager);
+ }
List<X509Certificate> caCertificates = getCaCertificates(sslConfig)
.map(X509CertificateUtils::certificateListFromPem)
.orElse(List.of());
- PeerAuthentication peerAuthentication = toPeerAuthentication(sslConfig.clientAuth());
- return new DefaultTlsContext(certificates, privateKey, caCertificates, null, null, peerAuthentication);
+ builder.withTrustStore(caCertificates);
+
+ SSLContext sslContext = builder.build();
+
+ SslContextFactory.Server factory = new SslContextFactory.Server();
+ factory.setSslContext(sslContext);
+
+ factory.setNeedClientAuth(sslConfig.clientAuth() == ClientAuth.Enum.NEED_AUTH);
+ factory.setWantClientAuth(sslConfig.clientAuth() == ClientAuth.Enum.WANT_AUTH);
+
+ List<String> protocols = !sslConfig.enabledProtocols().isEmpty()
+ ? sslConfig.enabledProtocols()
+ : new ArrayList<>(TlsContext.getAllowedProtocols(sslContext));
+ setEnabledProtocols(factory, sslContext, protocols);
+
+ List<String> ciphers = !sslConfig.enabledCipherSuites().isEmpty()
+ ? sslConfig.enabledCipherSuites()
+ : new ArrayList<>(TlsContext.getAllowedCipherSuites(sslContext));
+ setEnabledCipherSuites(factory, sslContext, ciphers);
+
+ return factory;
+ }
+
+ @Override
+ public void close() {
+ if (keyManager != null) {
+ keyManager.close();
+ }
}
private static void validateConfig(ConnectorConfig.Ssl config) {
@@ -64,19 +104,6 @@ public class ConfiguredSslContextFactoryProvider extends TlsContextBasedProvider
throw new IllegalArgumentException("Specified neither private key or private key file.");
}
- private static PeerAuthentication toPeerAuthentication(ConnectorConfig.Ssl.ClientAuth.Enum clientAuth) {
- switch (clientAuth) {
- case DISABLED:
- return PeerAuthentication.DISABLED;
- case NEED_AUTH:
- return PeerAuthentication.NEED;
- case WANT_AUTH:
- return PeerAuthentication.WANT;
- default:
- throw new IllegalArgumentException("Unknown client auth: " + clientAuth);
- }
- }
-
private static boolean hasBoth(String a, String b) { return !a.isBlank() && !b.isBlank(); }
private static boolean hasNeither(String a, String b) { return a.isBlank() && b.isBlank(); }
diff --git a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/impl/SslContextFactoryUtils.java b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/impl/SslContextFactoryUtils.java
new file mode 100644
index 00000000000..a0172668cbb
--- /dev/null
+++ b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/impl/SslContextFactoryUtils.java
@@ -0,0 +1,32 @@
+// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.jdisc.http.ssl.impl;
+
+import org.eclipse.jetty.util.ssl.SslContextFactory;
+
+import javax.net.ssl.SSLContext;
+import java.util.Arrays;
+import java.util.List;
+
+/**
+ * @author bjorncs
+ */
+class SslContextFactoryUtils {
+
+ static void setEnabledCipherSuites(SslContextFactory factory, SSLContext sslContext, List<String> enabledCiphers) {
+ String[] supportedCiphers = sslContext.getSupportedSSLParameters().getCipherSuites();
+ factory.setIncludeCipherSuites(enabledCiphers.toArray(String[]::new));
+ factory.setExcludeCipherSuites(createExclusionList(enabledCiphers, supportedCiphers));
+ }
+
+ static void setEnabledProtocols(SslContextFactory factory, SSLContext sslContext, List<String> enabledProtocols) {
+ String[] supportedProtocols = sslContext.getSupportedSSLParameters().getProtocols();
+ factory.setIncludeProtocols(enabledProtocols.toArray(String[]::new));
+ factory.setExcludeProtocols(createExclusionList(enabledProtocols, supportedProtocols));
+ }
+
+ private static String[] createExclusionList(List<String> enabledValues, String[] supportedValues) {
+ return Arrays.stream(supportedValues)
+ .filter(supportedValue -> !enabledValues.contains(supportedValue))
+ .toArray(String[]::new);
+ }
+}
diff --git a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/impl/TlsContextBasedProvider.java b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/impl/TlsContextBasedProvider.java
index e8ae13e48be..93d4f1dca3f 100644
--- a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/impl/TlsContextBasedProvider.java
+++ b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/impl/TlsContextBasedProvider.java
@@ -8,7 +8,10 @@ import org.eclipse.jetty.util.ssl.SslContextFactory;
import javax.net.ssl.SSLContext;
import javax.net.ssl.SSLParameters;
-import java.util.Arrays;
+import java.util.List;
+
+import static com.yahoo.jdisc.http.ssl.impl.SslContextFactoryUtils.setEnabledCipherSuites;
+import static com.yahoo.jdisc.http.ssl.impl.SslContextFactoryUtils.setEnabledProtocols;
/**
* A {@link SslContextFactoryProvider} that creates {@link SslContextFactory} instances from {@link TlsContext} instances.
@@ -31,24 +34,9 @@ public abstract class TlsContextBasedProvider extends AbstractComponent implemen
sslContextFactory.setNeedClientAuth(parameters.getNeedClientAuth());
sslContextFactory.setWantClientAuth(parameters.getWantClientAuth());
- String[] enabledProtocols = parameters.getProtocols();
- sslContextFactory.setIncludeProtocols(enabledProtocols);
- String[] supportedProtocols = sslContext.getSupportedSSLParameters().getProtocols();
- sslContextFactory.setExcludeProtocols(createExclusionList(enabledProtocols, supportedProtocols));
+ setEnabledProtocols(sslContextFactory, sslContext, List.of(parameters.getProtocols()));
+ setEnabledCipherSuites(sslContextFactory, sslContext, List.of(parameters.getCipherSuites()));
- String[] enabledCiphers = parameters.getCipherSuites();
- String[] supportedCiphers = sslContext.getSupportedSSLParameters().getCipherSuites();
- sslContextFactory.setIncludeCipherSuites(enabledCiphers);
- sslContextFactory.setExcludeCipherSuites(createExclusionList(enabledCiphers, supportedCiphers));
return sslContextFactory;
}
-
- private static String[] createExclusionList(String[] enabledValues, String[] supportedValues) {
- return Arrays.stream(supportedValues)
- .filter(supportedValue ->
- Arrays.stream(enabledValues)
- .noneMatch(enabledValue -> enabledValue.equals(supportedValue)))
- .toArray(String[]::new);
- }
-
}
diff --git a/jdisc_http_service/src/main/resources/configdefinitions/jdisc.http.connector.def b/jdisc_http_service/src/main/resources/configdefinitions/jdisc.http.connector.def
index 1122b1db3a9..fe79ec2ffa3 100644
--- a/jdisc_http_service/src/main/resources/configdefinitions/jdisc.http.connector.def
+++ b/jdisc_http_service/src/main/resources/configdefinitions/jdisc.http.connector.def
@@ -81,6 +81,12 @@ ssl.caCertificate string default=""
# Client authentication mode. See SSLEngine.getNeedClientAuth()/getWantClientAuth() for details.
ssl.clientAuth enum { DISABLED, WANT_AUTH, NEED_AUTH } default=DISABLED
+# List of enabled cipher suites. JDisc will use Vespa default if empty.
+ssl.enabledCipherSuites[] string
+
+# List of enabled TLS protocol versions. JDisc will use Vespa default if empty.
+ssl.enabledProtocols[] string
+
# Enforce TLS client authentication for https requests at the http layer.
# Intended to be used with connectors with optional client authentication enabled.
# 401 status code is returned for requests from non-authenticated clients.
diff --git a/model-evaluation/abi-spec.json b/model-evaluation/abi-spec.json
index 5a75c8b31ea..d465464de7f 100644
--- a/model-evaluation/abi-spec.json
+++ b/model-evaluation/abi-spec.json
@@ -8,6 +8,7 @@
"methods": [
"public ai.vespa.models.evaluation.FunctionEvaluator bind(java.lang.String, com.yahoo.tensor.Tensor)",
"public ai.vespa.models.evaluation.FunctionEvaluator bind(java.lang.String, double)",
+ "public ai.vespa.models.evaluation.FunctionEvaluator bind(java.lang.String, java.lang.String)",
"public ai.vespa.models.evaluation.FunctionEvaluator setMissingValue(com.yahoo.tensor.Tensor)",
"public ai.vespa.models.evaluation.FunctionEvaluator setMissingValue(double)",
"public com.yahoo.tensor.Tensor evaluate()",
diff --git a/model-evaluation/src/main/java/ai/vespa/models/evaluation/FunctionEvaluator.java b/model-evaluation/src/main/java/ai/vespa/models/evaluation/FunctionEvaluator.java
index 994f6dd9b64..e373a54bcd1 100644
--- a/model-evaluation/src/main/java/ai/vespa/models/evaluation/FunctionEvaluator.java
+++ b/model-evaluation/src/main/java/ai/vespa/models/evaluation/FunctionEvaluator.java
@@ -2,6 +2,7 @@
package ai.vespa.models.evaluation;
import com.yahoo.searchlib.rankingexpression.ExpressionFunction;
+import com.yahoo.searchlib.rankingexpression.evaluation.StringValue;
import com.yahoo.searchlib.rankingexpression.evaluation.TensorValue;
import com.yahoo.tensor.Tensor;
import com.yahoo.tensor.TensorType;
@@ -61,6 +62,21 @@ public class FunctionEvaluator {
}
/**
+ * Binds the given variable referred in this expression to the given value.
+ * String values are not yet supported in tensors.
+ *
+ * @param name the variable to bind
+ * @param value the value this becomes bound to
+ * @return this for chaining
+ */
+ public FunctionEvaluator bind(String name, String value) {
+ if (evaluated)
+ throw new IllegalStateException("Cannot bind a new value in a used evaluator");
+ context.put(name, new StringValue(value));
+ return this;
+ }
+
+ /**
* Sets the default value to use for variables which are not bound
*
* @param value the default value
diff --git a/model-evaluation/src/main/java/ai/vespa/models/handler/ModelsEvaluationHandler.java b/model-evaluation/src/main/java/ai/vespa/models/handler/ModelsEvaluationHandler.java
index 5c353fcdf35..de23a8c6526 100644
--- a/model-evaluation/src/main/java/ai/vespa/models/handler/ModelsEvaluationHandler.java
+++ b/model-evaluation/src/main/java/ai/vespa/models/handler/ModelsEvaluationHandler.java
@@ -77,8 +77,14 @@ public class ModelsEvaluationHandler extends ThreadedHttpRequestHandler {
property(request, missingValueKey).ifPresent(missingValue -> evaluator.setMissingValue(Tensor.from(missingValue)));
for (Map.Entry<String, TensorType> argument : evaluator.function().argumentTypes().entrySet()) {
- property(request, argument.getKey()).ifPresent(value -> evaluator.bind(argument.getKey(),
- Tensor.from(argument.getValue(), value)));
+ Optional<String> value = property(request, argument.getKey());
+ if (value.isPresent()) {
+ try {
+ evaluator.bind(argument.getKey(), Tensor.from(argument.getValue(), value.get()));
+ } catch (IllegalArgumentException e) {
+ evaluator.bind(argument.getKey(), value.get()); // since we don't yet support tensors with string values
+ }
+ }
}
Tensor result = evaluator.evaluate();
return new Response(200, JsonFormat.encode(result));
diff --git a/model-evaluation/src/test/java/ai/vespa/models/evaluation/MlModelsImportingTest.java b/model-evaluation/src/test/java/ai/vespa/models/evaluation/MlModelsImportingTest.java
index 9320ac3fad8..0d13b7d4660 100644
--- a/model-evaluation/src/test/java/ai/vespa/models/evaluation/MlModelsImportingTest.java
+++ b/model-evaluation/src/test/java/ai/vespa/models/evaluation/MlModelsImportingTest.java
@@ -25,7 +25,7 @@ public class MlModelsImportingTest {
public void testImportingModels() {
ModelTester tester = new ModelTester("src/test/resources/config/models/");
- assertEquals(4, tester.models().size());
+ assertEquals(5, tester.models().size());
// TODO: When we get type information in Models, replace the evaluator.context().names() check below by that
{
@@ -47,7 +47,24 @@ public class MlModelsImportingTest {
}
{
+ Model lightgbm = tester.models().get("lightgbm_regression");
+ // Function
+ assertEquals(1, lightgbm.functions().size());
+ ExpressionFunction function = tester.assertFunction("lightgbm_regression",
+ "(optimized sum of condition trees of size 480 bytes)",
+ lightgbm);
+ assertEquals("tensor()", function.returnType().get().toString());
+ assertEquals("categorical_1, categorical_2, numerical_1, numerical_2", commaSeparated(function.arguments()));
+ function.arguments().forEach(arg -> assertEquals(TensorType.empty, function.argumentTypes().get(arg)));
+
+ // Evaluator
+ FunctionEvaluator evaluator = lightgbm.evaluatorOf();
+ assertEquals("categorical_1, categorical_2, numerical_1, numerical_2", evaluator.context().names().stream().sorted().collect(Collectors.joining(", ")));
+ assertEquals(1.91300868202, evaluator.evaluate().sum().asDouble(), delta);
+ }
+
+ {
Model onnxMnistSoftmax = tester.models().get("mnist_softmax");
// Function
diff --git a/model-evaluation/src/test/java/ai/vespa/models/handler/ModelsEvaluationHandlerTest.java b/model-evaluation/src/test/java/ai/vespa/models/handler/ModelsEvaluationHandlerTest.java
index 629c82f410a..c9e49d3be02 100644
--- a/model-evaluation/src/test/java/ai/vespa/models/handler/ModelsEvaluationHandlerTest.java
+++ b/model-evaluation/src/test/java/ai/vespa/models/handler/ModelsEvaluationHandlerTest.java
@@ -56,7 +56,7 @@ public class ModelsEvaluationHandlerTest {
public void testListModels() {
String url = "http://localhost/model-evaluation/v1";
String expected =
- "{\"mnist_softmax\":\"http://localhost/model-evaluation/v1/mnist_softmax\",\"mnist_saved\":\"http://localhost/model-evaluation/v1/mnist_saved\",\"mnist_softmax_saved\":\"http://localhost/model-evaluation/v1/mnist_softmax_saved\",\"xgboost_2_2\":\"http://localhost/model-evaluation/v1/xgboost_2_2\"}";
+ "{\"mnist_softmax\":\"http://localhost/model-evaluation/v1/mnist_softmax\",\"mnist_saved\":\"http://localhost/model-evaluation/v1/mnist_saved\",\"mnist_softmax_saved\":\"http://localhost/model-evaluation/v1/mnist_softmax_saved\",\"xgboost_2_2\":\"http://localhost/model-evaluation/v1/xgboost_2_2\",\"lightgbm_regression\":\"http://localhost/model-evaluation/v1/lightgbm_regression\"}";
assertResponse(url, 200, expected);
}
@@ -94,6 +94,39 @@ public class ModelsEvaluationHandlerTest {
}
@Test
+ public void testLightGBMEvaluationWithoutBindings() {
+ String url = "http://localhost/model-evaluation/v1/lightgbm_regression/eval";
+ String expected = "{\"cells\":[{\"address\":{},\"value\":1.9130086820218188}]}";
+ assertResponse(url, 200, expected);
+ }
+
+ @Test
+ public void testLightGBMEvaluationWithBindings() {
+ Map<String, String> properties = new HashMap<>();
+ properties.put("numerical_1", "0.1");
+ properties.put("numerical_2", "0.2");
+ properties.put("categorical_1", "a");
+ properties.put("categorical_2", "i");
+ properties.put("non-existing-binding", "-1");
+ String url = "http://localhost/model-evaluation/v1/lightgbm_regression/eval";
+ String expected = "{\"cells\":[{\"address\":{},\"value\":2.054697758469921}]}";
+ assertResponse(url, properties, 200, expected);
+ }
+
+ @Test
+ public void testLightGBMEvaluationWithMissingValue() {
+ Map<String, String> properties = new HashMap<>();
+ properties.put("missing-value", "-1.0");
+ properties.put("numerical_2", "0.5");
+ properties.put("categorical_1", "b");
+ properties.put("categorical_2", "j");
+ properties.put("non-existing-binding", "-1");
+ String url = "http://localhost/model-evaluation/v1/lightgbm_regression/eval";
+ String expected = "{\"cells\":[{\"address\":{},\"value\":2.0745534018208094}]}";
+ assertResponse(url, properties, 200, expected);
+ }
+
+ @Test
public void testMnistSoftmaxDetails() {
String url = "http://localhost:8080/model-evaluation/v1/mnist_softmax";
String expected = "{\"model\":\"mnist_softmax\",\"functions\":[{\"function\":\"default.add\",\"info\":\"http://localhost:8080/model-evaluation/v1/mnist_softmax/default.add\",\"eval\":\"http://localhost:8080/model-evaluation/v1/mnist_softmax/default.add/eval\",\"arguments\":[{\"name\":\"Placeholder\",\"type\":\"tensor(d0[],d1[784])\"}]}]}";
diff --git a/model-evaluation/src/test/resources/config/models/rank-profiles.cfg b/model-evaluation/src/test/resources/config/models/rank-profiles.cfg
index c25c5ba555b..385115b7cd4 100644
--- a/model-evaluation/src/test/resources/config/models/rank-profiles.cfg
+++ b/model-evaluation/src/test/resources/config/models/rank-profiles.cfg
@@ -26,3 +26,6 @@ rankprofile[3].fef.property[3].name "rankingExpression(serving_default.y).input.
rankprofile[3].fef.property[3].value "tensor(d0[],d1[784])"
rankprofile[3].fef.property[4].name "rankingExpression(serving_default.y).type"
rankprofile[3].fef.property[4].value "tensor(d1[10])"
+rankprofile[4].name "lightgbm_regression"
+rankprofile[4].fef.property[0].name "rankingExpression(lightgbm_regression).rankingScript"
+rankprofile[4].fef.property[0].value "if (!(numerical_2 >= 0.46643291586559305), 2.1594397038037663, if (categorical_2 in ["k", "l", "m"], 2.235297305276056, 2.1792953471546546)) + if (categorical_1 in ["d", "e"], 0.03070842919354316, if (!(numerical_1 >= 0.5102250691730842), -0.04439151147520909, 0.005117411709368601)) + if (!(numerical_2 >= 0.668665477622446), if (!(numerical_2 >= 0.008118820676863816), -0.15361238490967524, -0.01192330846157292), 0.03499044894987518) + if (!(numerical_1 >= 0.5201391072644542), -0.02141000620783247, if (categorical_1 in ["a", "b"], -0.004121485787596721, 0.04534090904886873)) + if (categorical_2 in ["k", "l", "m"], if (!(numerical_2 >= 0.27283279016959255), -0.01924803254356527, 0.03643772842347651), -0.02701711918923075)"
diff --git a/model-integration/src/main/config/model-integration.xml b/model-integration/src/main/config/model-integration.xml
index 90ec7d7275e..34f5f0ce31a 100644
--- a/model-integration/src/main/config/model-integration.xml
+++ b/model-integration/src/main/config/model-integration.xml
@@ -1,11 +1,12 @@
<!-- Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -->
<!-- Component which can import some ml model.
This is included into the config server services.xml to enable it to translate
- model pseudofeatures in ranking expressions during config mddel building.
+ model pseudo features in ranking expressions during config model building.
It is provided as separate bundles instead of being included in the config model
because some of these (TensorFlow) includes
JNI code, and so can only exist in one instance in the server. -->
<component id="ai.vespa.rankingexpression.importer.onnx.OnnxImporter" bundle="model-integration" />
<component id="ai.vespa.rankingexpression.importer.tensorflow.TensorFlowImporter" bundle="model-integration" />
<component id="ai.vespa.rankingexpression.importer.xgboost.XGBoostImporter" bundle="model-integration" />
+<component id="ai.vespa.rankingexpression.importer.lightgbm.LightGBMImporter" bundle="model-integration" />
<component id="ai.vespa.rankingexpression.importer.vespa.VespaImporter" bundle="model-integration" />
diff --git a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/lightgbm/LightGBMImporter.java b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/lightgbm/LightGBMImporter.java
new file mode 100644
index 00000000000..76caa652ad2
--- /dev/null
+++ b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/lightgbm/LightGBMImporter.java
@@ -0,0 +1,54 @@
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package ai.vespa.rankingexpression.importer.lightgbm;
+
+import ai.vespa.rankingexpression.importer.ImportedModel;
+import ai.vespa.rankingexpression.importer.ModelImporter;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.yahoo.searchlib.rankingexpression.RankingExpression;
+import com.yahoo.searchlib.rankingexpression.parser.ParseException;
+
+import java.io.File;
+import java.io.IOException;
+
+/**
+ * Converts a LightGBM model into a ranking expression.
+ *
+ * @author lesters
+ */
+public class LightGBMImporter extends ModelImporter {
+
+ @Override
+ public boolean canImport(String modelPath) {
+ File modelFile = new File(modelPath);
+ if ( ! modelFile.isFile()) return false;
+ return modelFile.toString().endsWith(".json") && probe(modelFile);
+ }
+
+ /**
+ * Returns true if the give file looks like a LightGBM json file.
+ * Currently, we just check if the json has an element called "tree_info"
+ */
+ private boolean probe(File modelFile) {
+ try {
+ return new ObjectMapper().readTree(modelFile).has("tree_info");
+ } catch (IOException e) {
+ throw new IllegalArgumentException("Could not read '" + modelFile + "'", e);
+ }
+ }
+
+ @Override
+ public ImportedModel importModel(String modelName, String modelPath) {
+ try {
+ ImportedModel model = new ImportedModel(modelName, modelPath);
+ LightGBMParser parser = new LightGBMParser(modelPath);
+ RankingExpression expression = new RankingExpression(parser.toRankingExpression());
+ model.expression(modelName, expression);
+ return model;
+ } catch (IOException e) {
+ throw new IllegalArgumentException("Could not import LightGBM model from '" + modelPath + "'", e);
+ } catch (ParseException e) {
+ throw new IllegalArgumentException("Could not parse ranking expression resulting from '" + modelPath + "'", e);
+ }
+ }
+
+}
diff --git a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/lightgbm/LightGBMNode.java b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/lightgbm/LightGBMNode.java
new file mode 100644
index 00000000000..dc76ed8cb6f
--- /dev/null
+++ b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/lightgbm/LightGBMNode.java
@@ -0,0 +1,67 @@
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package ai.vespa.rankingexpression.importer.lightgbm;
+
+/**
+ * @author lesters
+ */
+public class LightGBMNode {
+
+ // split nodes
+ private int split_feature;
+ private String threshold; // double for numerical, string for categorical
+ private String decision_type;
+ private boolean default_left;
+ private String missing_type;
+ private int internal_count;
+ private LightGBMNode left_child;
+ private LightGBMNode right_child;
+
+ // leaf nodes
+ private double leaf_value;
+ private int leaf_count;
+
+ public int getSplit_feature() {
+ return split_feature;
+ }
+
+ public String getThreshold() {
+ return threshold;
+ }
+
+ public String getDecision_type() {
+ return decision_type;
+ }
+
+ public boolean isDefault_left() {
+ return default_left;
+ }
+
+ public String getMissing_type() {
+ return missing_type;
+ }
+
+ public int getInternal_count() {
+ return internal_count;
+ }
+
+ public LightGBMNode getLeft_child() {
+ return left_child;
+ }
+
+ public LightGBMNode getRight_child() {
+ return right_child;
+ }
+
+ public double getLeaf_value() {
+ return leaf_value;
+ }
+
+ public int getLeaf_count() {
+ return leaf_count;
+ }
+
+ public boolean isLeaf() {
+ return left_child == null && right_child == null;
+ }
+
+} \ No newline at end of file
diff --git a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/lightgbm/LightGBMParser.java b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/lightgbm/LightGBMParser.java
new file mode 100644
index 00000000000..996343674ce
--- /dev/null
+++ b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/lightgbm/LightGBMParser.java
@@ -0,0 +1,146 @@
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package ai.vespa.rankingexpression.importer.lightgbm;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.DeserializationFeature;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeSet;
+import java.util.stream.Collectors;
+
+/**
+ * @author lesters
+ */
+class LightGBMParser {
+
+ private final String objective;
+ private final List<LightGBMNode> nodes;
+ private final List<String> featureNames;
+ private final Map<Integer, List<String>> categoryValues; // pr feature index
+
+ LightGBMParser(String filePath) throws JsonProcessingException, IOException {
+ ObjectMapper mapper = new ObjectMapper().configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
+ JsonNode root = mapper.readTree(new File(filePath));
+
+ objective = root.get("objective").asText("regression");
+ featureNames = parseFeatureNames(root);
+ nodes = parseTrees(mapper, root);
+ categoryValues = parseCategoryValues(root);
+ }
+
+ private List<String> parseFeatureNames(JsonNode root) {
+ List<String> features = new ArrayList<>();
+ for (JsonNode name : root.get("feature_names")) {
+ features.add(name.textValue());
+ }
+ return features;
+ }
+
+ private List<LightGBMNode> parseTrees(ObjectMapper mapper, JsonNode root) throws JsonProcessingException {
+ List<LightGBMNode> nodes = new ArrayList<>();
+ for (JsonNode treeNode : root.get("tree_info")) {
+ nodes.add(mapper.treeToValue(treeNode.get("tree_structure"), LightGBMNode.class));
+ }
+ return nodes;
+ }
+
+ private Map<Integer, List<String>> parseCategoryValues(JsonNode root) {
+ Map<Integer, List<String>> categoryValues = new HashMap<>();
+
+ // Since the JSON format does not explicitly tell which features are
+ // categorical, we traverse the decision tree looking for categorical
+ // decisions and use that to determine which categorical features.
+ Set<Integer> categoricalFeatures = new TreeSet<>();
+ nodes.forEach(node -> findCategoricalFeatures(node, categoricalFeatures));
+
+ // Again, the LightGBM JSON format does not explicitly tell which
+ // categorical values map to each categorical feature. The assumption
+ // here is that the order they appear in the "pandas_categorical"
+ // structure is the same order as the "feature_names".
+ var pandasFeatureIterator = root.get("pandas_categorical").iterator();
+ var categoricalFeatureIterator = categoricalFeatures.iterator();
+ while (pandasFeatureIterator.hasNext() && categoricalFeatureIterator.hasNext()) {
+ List<String> values = new ArrayList<>();
+ pandasFeatureIterator.next().forEach(value -> values.add(value.textValue()));
+ categoryValues.put(categoricalFeatureIterator.next(), values);
+ }
+
+ return categoryValues;
+ }
+
+ private void findCategoricalFeatures(LightGBMNode node, Set<Integer> categoricalFeatures) {
+ if (node == null || node.isLeaf()) {
+ return;
+ }
+ if (node.getDecision_type().equals("==")) {
+ categoricalFeatures.add(node.getSplit_feature());
+ }
+ findCategoricalFeatures(node.getLeft_child(), categoricalFeatures);
+ findCategoricalFeatures(node.getRight_child(), categoricalFeatures);
+ }
+
+ String toRankingExpression() {
+ return applyObjective(nodes.stream().map(this::nodeToRankingExpression).collect(Collectors.joining(" + \n")));
+ }
+
+ // See https://lightgbm.readthedocs.io/en/latest/Parameters.html#objective
+ private String applyObjective(String expression) {
+ if (objective.startsWith("binary") || objective.equals("cross_entropy")) {
+ return "sigmoid(" + expression + ")";
+ }
+ if (objective.equals("poisson") || objective.equals("gamma") || objective.equals("tweedie")) {
+ return "exp(" + expression + ")";
+ }
+ return expression; // else: use expression directly
+ }
+
+ private String nodeToRankingExpression(LightGBMNode node) {
+ if (node.isLeaf()) {
+ return Double.toString(node.getLeaf_value());
+ } else {
+ String condition;
+ String feature = featureNames.get(node.getSplit_feature());
+ if (node.getDecision_type().equals("==")) {
+ String values = transformCategoryIndexesToValues(node);
+ if (node.isDefault_left()) { // means go left (true) when isNan
+ condition = "isNan(" + feature + ") || (" + feature + " in [ " + values + "])";
+ } else {
+ condition = feature + " in [" + values + "]";
+ }
+ } else { // assumption: all other decision types are <=
+ double value = Double.parseDouble(node.getThreshold());
+ if (node.isDefault_left()) {
+ condition = "!(" + feature + " >= " + value + ")";
+ } else {
+ condition = feature + " < " + value;
+ }
+ }
+ String left = nodeToRankingExpression(node.getLeft_child());
+ String right = nodeToRankingExpression(node.getRight_child());
+ return "if (" + condition + ", " + left + ", " + right + ")";
+ }
+ }
+
+ private String transformCategoryIndexesToValues(LightGBMNode node) {
+ return Arrays.stream(node.getThreshold().split("\\|\\|"))
+ .map(index -> "\"" + transformCategoryIndexToValue(node.getSplit_feature(), index) + "\"")
+ .collect(Collectors.joining(","));
+ }
+
+ private String transformCategoryIndexToValue(int featureIndex, String valueIndex) {
+ if ( ! categoryValues.containsKey(featureIndex) ) {
+ return valueIndex; // We don't have a pandas categorical lookup table
+ }
+ return categoryValues.get(featureIndex).get(Integer.parseInt(valueIndex));
+ }
+
+} \ No newline at end of file
diff --git a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/lightgbm/package-info.java b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/lightgbm/package-info.java
new file mode 100644
index 00000000000..b29145ee21b
--- /dev/null
+++ b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/lightgbm/package-info.java
@@ -0,0 +1,5 @@
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+@ExportPackage
+package ai.vespa.rankingexpression.importer.lightgbm;
+
+import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/model-integration/src/test/java/ai/vespa/rankingexpression/importer/lightgbm/LightGBMImportEvaluationTestCase.java b/model-integration/src/test/java/ai/vespa/rankingexpression/importer/lightgbm/LightGBMImportEvaluationTestCase.java
new file mode 100644
index 00000000000..d2ef4b984ff
--- /dev/null
+++ b/model-integration/src/test/java/ai/vespa/rankingexpression/importer/lightgbm/LightGBMImportEvaluationTestCase.java
@@ -0,0 +1,49 @@
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package ai.vespa.rankingexpression.importer.lightgbm;
+
+import com.yahoo.searchlib.rankingexpression.RankingExpression;
+import com.yahoo.searchlib.rankingexpression.evaluation.ArrayContext;
+import com.yahoo.searchlib.rankingexpression.evaluation.ContextIndex;
+import com.yahoo.searchlib.rankingexpression.evaluation.DoubleValue;
+import com.yahoo.searchlib.rankingexpression.evaluation.ExpressionOptimizer;
+import com.yahoo.searchlib.rankingexpression.evaluation.gbdtoptimization.GBDTForestNode;
+import org.junit.Test;
+
+import static org.junit.Assert.assertTrue;
+
+/**
+ * @author lesters
+ */
+public class LightGBMImportEvaluationTestCase extends LightGBMTestBase {
+
+ @Test
+ public void testRegression() {
+ RankingExpression expression = importModel("src/test/models/lightgbm/regression.json");
+ ArrayContext context = new ArrayContext(expression, true, DoubleValue.NaN);
+
+ assertEvaluation(1.91300868, expression, features(context));
+ assertEvaluation(2.05469776, expression, features(context).add("numerical_1", 0.1).add("numerical_2", 0.2).add("categorical_1", "a").add("categorical_2", "i"));
+ assertEvaluation(2.0745534, expression, features(context).add("numerical_2", 0.5).add("categorical_1", "b").add("categorical_2", "j"));
+ assertEvaluation(2.3571838, expression, features(context).add("numerical_1", 0.7).add("numerical_2", 0.8).add("categorical_2", "m"));
+
+ ExpressionOptimizer optimizer = new ExpressionOptimizer();
+ optimizer.optimize(expression, (ContextIndex)context);
+ assertTrue(expression.getRoot() instanceof GBDTForestNode);
+
+ assertEvaluation(1.91300868, expression, features(context));
+ assertEvaluation(2.05469776, expression, features(context).add("numerical_1", 0.1).add("numerical_2", 0.2).add("categorical_1", "a").add("categorical_2", "i"));
+ assertEvaluation(2.0745534, expression, features(context).add("numerical_2", 0.5).add("categorical_1", "b").add("categorical_2", "j"));
+ assertEvaluation(2.3571838, expression, features(context).add("numerical_1", 0.7).add("numerical_2", 0.8).add("categorical_2", "m"));
+ }
+
+ @Test
+ public void testClassification() {
+ RankingExpression expression = importModel("src/test/models/lightgbm/classification.json");
+ ArrayContext context = new ArrayContext(expression, DoubleValue.NaN);
+ assertEvaluation(0.37464997, expression, features(context));
+ assertEvaluation(0.37464997, expression, features(context).add("numerical_1", 0.1).add("numerical_2", 0.2).add("categorical_1", "a").add("categorical_2", "i"));
+ assertEvaluation(0.38730827, expression, features(context).add("numerical_2", 0.5).add("categorical_1", "b").add("categorical_2", "j"));
+ assertEvaluation(0.5647872, expression, features(context).add("numerical_1", 0.7).add("numerical_2", 0.8).add("categorical_2", "m"));
+ }
+
+}
diff --git a/model-integration/src/test/java/ai/vespa/rankingexpression/importer/lightgbm/LightGBMTestBase.java b/model-integration/src/test/java/ai/vespa/rankingexpression/importer/lightgbm/LightGBMTestBase.java
new file mode 100644
index 00000000000..80c2ce68394
--- /dev/null
+++ b/model-integration/src/test/java/ai/vespa/rankingexpression/importer/lightgbm/LightGBMTestBase.java
@@ -0,0 +1,42 @@
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package ai.vespa.rankingexpression.importer.lightgbm;
+
+import com.yahoo.searchlib.rankingexpression.RankingExpression;
+import com.yahoo.searchlib.rankingexpression.evaluation.ArrayContext;
+import com.yahoo.searchlib.rankingexpression.evaluation.StringValue;
+
+import static org.junit.Assert.assertEquals;
+
+/**
+ * @author lesters
+ */
+class LightGBMTestBase {
+
+ RankingExpression importModel(String path) {
+ return new LightGBMImporter().importModel("lightgbm", path).expressions().get("lightgbm");
+ }
+
+ void assertEvaluation(double expected, RankingExpression expr, TestFeatures features) {
+ assertEquals(expected, expr.evaluate(features.context).asDouble(), 1e-6);
+ }
+
+ TestFeatures features(ArrayContext context) {
+ return new TestFeatures(context.clone());
+ }
+
+ static class TestFeatures {
+ private final ArrayContext context;
+ TestFeatures(ArrayContext context) {
+ this.context = context;
+ }
+ TestFeatures add(String name, double value) {
+ context.put(name, value);
+ return this;
+ }
+ TestFeatures add(String name, String value) {
+ context.put(name, new StringValue(value));
+ return this;
+ }
+ }
+
+}
diff --git a/model-integration/src/test/models/lightgbm/classification.json b/model-integration/src/test/models/lightgbm/classification.json
new file mode 100644
index 00000000000..1087446519d
--- /dev/null
+++ b/model-integration/src/test/models/lightgbm/classification.json
@@ -0,0 +1,275 @@
+{
+ "name": "tree",
+ "version": "v3",
+ "num_class": 1,
+ "num_tree_per_iteration": 1,
+ "label_index": 0,
+ "max_feature_idx": 3,
+ "average_output": false,
+ "objective": "binary sigmoid:1",
+ "feature_names": [
+ "numerical_1",
+ "numerical_2",
+ "categorical_1",
+ "categorical_2"
+ ],
+ "monotone_constraints": [],
+ "tree_info": [
+ {
+ "tree_index": 0,
+ "num_leaves": 3,
+ "num_cat": 2,
+ "shrinkage": 1,
+ "tree_structure": {
+ "split_index": 0,
+ "split_feature": 3,
+ "split_gain": 13080.099609375,
+ "threshold": "2||3||4",
+ "decision_type": "==",
+ "default_left": false,
+ "missing_type": "NaN",
+ "internal_value": 0,
+ "internal_weight": 0,
+ "internal_count": 100000,
+ "left_child": {
+ "split_index": 1,
+ "split_feature": 2,
+ "split_gain": 8303.599609375,
+ "threshold": "2||3||4",
+ "decision_type": "==",
+ "default_left": false,
+ "missing_type": "NaN",
+ "internal_value": 0.598248,
+ "internal_weight": 14841.2,
+ "internal_count": 59371,
+ "left_child": {
+ "leaf_index": 0,
+ "leaf_value": 0.10149588882231209,
+ "leaf_weight": 8812.104370772839,
+ "leaf_count": 35252
+ },
+ "right_child": {
+ "leaf_index": 2,
+ "leaf_value": -0.05076009488472203,
+ "leaf_weight": 6029.137221112847,
+ "leaf_count": 24119
+ }
+ },
+ "right_child": {
+ "leaf_index": 1,
+ "leaf_value": -0.1075553310531564,
+ "leaf_weight": 10156.217760130763,
+ "leaf_count": 40629
+ }
+ }
+ },
+ {
+ "tree_index": 1,
+ "num_leaves": 3,
+ "num_cat": 0,
+ "shrinkage": 0.1,
+ "tree_structure": {
+ "split_index": 0,
+ "split_feature": 1,
+ "split_gain": 12144.5,
+ "threshold": 0.4932456977560694,
+ "decision_type": "<=",
+ "default_left": true,
+ "missing_type": "NaN",
+ "internal_value": 0,
+ "internal_weight": 0,
+ "internal_count": 100000,
+ "left_child": {
+ "leaf_index": 0,
+ "leaf_value": -0.07039230856418545,
+ "leaf_weight": 12362.572675153613,
+ "leaf_count": 49561
+ },
+ "right_child": {
+ "split_index": 1,
+ "split_feature": 0,
+ "split_gain": 6445.509765625,
+ "threshold": 0.4026061210695467,
+ "decision_type": "<=",
+ "default_left": true,
+ "missing_type": "NaN",
+ "internal_value": 0.691647,
+ "internal_weight": 12581.6,
+ "internal_count": 50439,
+ "left_child": {
+ "leaf_index": 1,
+ "leaf_value": -0.016713933964828474,
+ "leaf_weight": 5157.183633238077,
+ "leaf_count": 20675
+ },
+ "right_child": {
+ "leaf_index": 2,
+ "leaf_value": 0.12881836794307533,
+ "leaf_weight": 7424.385220557451,
+ "leaf_count": 29764
+ }
+ }
+ }
+ },
+ {
+ "tree_index": 2,
+ "num_leaves": 3,
+ "num_cat": 2,
+ "shrinkage": 0.1,
+ "tree_structure": {
+ "split_index": 0,
+ "split_feature": 2,
+ "split_gain": 11470.099609375,
+ "threshold": "3||4",
+ "decision_type": "==",
+ "default_left": false,
+ "missing_type": "NaN",
+ "internal_value": 0,
+ "internal_weight": 0,
+ "internal_count": 100000,
+ "left_child": {
+ "leaf_index": 0,
+ "leaf_value": 0.0837843210726433,
+ "leaf_weight": 9858.360527098179,
+ "leaf_count": 39612
+ },
+ "right_child": {
+ "split_index": 1,
+ "split_feature": 3,
+ "split_gain": 8077.8701171875,
+ "threshold": "3||4",
+ "decision_type": "==",
+ "default_left": false,
+ "missing_type": "NaN",
+ "internal_value": -0.549408,
+ "internal_weight": 15039.7,
+ "internal_count": 60388,
+ "left_child": {
+ "leaf_index": 1,
+ "leaf_value": 0.035561394754096094,
+ "leaf_weight": 5955.117423638701,
+ "leaf_count": 23896
+ },
+ "right_child": {
+ "leaf_index": 2,
+ "leaf_value": -0.11424082565448186,
+ "leaf_weight": 9084.538012728095,
+ "leaf_count": 36492
+ }
+ }
+ }
+ },
+ {
+ "tree_index": 3,
+ "num_leaves": 3,
+ "num_cat": 0,
+ "shrinkage": 0.1,
+ "tree_structure": {
+ "split_index": 0,
+ "split_feature": 0,
+ "split_gain": 11022.599609375,
+ "threshold": 0.5135386524711826,
+ "decision_type": "<=",
+ "default_left": true,
+ "missing_type": "NaN",
+ "internal_value": 0,
+ "internal_weight": 0,
+ "internal_count": 100000,
+ "left_child": {
+ "split_index": 1,
+ "split_feature": 1,
+ "split_gain": 5789.919921875,
+ "threshold": 0.6237474076885036,
+ "decision_type": "<=",
+ "default_left": true,
+ "missing_type": "NaN",
+ "internal_value": -0.641438,
+ "internal_weight": 12881.9,
+ "internal_count": 51907,
+ "left_child": {
+ "leaf_index": 0,
+ "leaf_value": -0.11613056205533928,
+ "leaf_weight": 8044.6355674266815,
+ "leaf_count": 32426
+ },
+ "right_child": {
+ "leaf_index": 2,
+ "leaf_value": 0.022313103333779363,
+ "leaf_weight": 4837.266924858093,
+ "leaf_count": 19481
+ }
+ },
+ "right_child": {
+ "leaf_index": 1,
+ "leaf_value": 0.06927713686880098,
+ "leaf_weight": 11923.512641906738,
+ "leaf_count": 48093
+ }
+ }
+ },
+ {
+ "tree_index": 4,
+ "num_leaves": 3,
+ "num_cat": 2,
+ "shrinkage": 0.1,
+ "tree_structure": {
+ "split_index": 0,
+ "split_feature": 3,
+ "split_gain": 9828.9501953125,
+ "threshold": "3||4",
+ "decision_type": "==",
+ "default_left": false,
+ "missing_type": "NaN",
+ "internal_value": 0,
+ "internal_weight": 0,
+ "internal_count": 100000,
+ "left_child": {
+ "leaf_index": 0,
+ "leaf_value": 0.07771712562582928,
+ "leaf_weight": 9804.427681803703,
+ "leaf_count": 39586
+ },
+ "right_child": {
+ "split_index": 1,
+ "split_feature": 2,
+ "split_gain": 6332.2900390625,
+ "threshold": "3||4",
+ "decision_type": "==",
+ "default_left": false,
+ "missing_type": "NaN",
+ "internal_value": -0.51112,
+ "internal_weight": 14922.7,
+ "internal_count": 60414,
+ "left_child": {
+ "leaf_index": 1,
+ "leaf_value": 0.029062142260340918,
+ "leaf_weight": 5933.120021238923,
+ "leaf_count": 23922
+ },
+ "right_child": {
+ "leaf_index": 2,
+ "leaf_value": -0.10400033924773491,
+ "leaf_weight": 8989.602796778083,
+ "leaf_count": 36492
+ }
+ }
+ }
+ }
+ ],
+ "pandas_categorical": [
+ [
+ "a",
+ "b",
+ "c",
+ "d",
+ "e"
+ ],
+ [
+ "i",
+ "j",
+ "k",
+ "l",
+ "m"
+ ]
+ ]
+} \ No newline at end of file
diff --git a/model-integration/src/test/models/lightgbm/regression.json b/model-integration/src/test/models/lightgbm/regression.json
new file mode 100644
index 00000000000..cf0488ecd8b
--- /dev/null
+++ b/model-integration/src/test/models/lightgbm/regression.json
@@ -0,0 +1,275 @@
+{
+ "name": "tree",
+ "version": "v3",
+ "num_class": 1,
+ "num_tree_per_iteration": 1,
+ "label_index": 0,
+ "max_feature_idx": 3,
+ "average_output": false,
+ "objective": "regression",
+ "feature_names": [
+ "numerical_1",
+ "numerical_2",
+ "categorical_1",
+ "categorical_2"
+ ],
+ "monotone_constraints": [],
+ "tree_info": [
+ {
+ "tree_index": 0,
+ "num_leaves": 3,
+ "num_cat": 1,
+ "shrinkage": 1,
+ "tree_structure": {
+ "split_index": 0,
+ "split_feature": 1,
+ "split_gain": 68.5353012084961,
+ "threshold": 0.46643291586559305,
+ "decision_type": "<=",
+ "default_left": true,
+ "missing_type": "NaN",
+ "internal_value": 0,
+ "internal_weight": 0,
+ "internal_count": 1000,
+ "left_child": {
+ "leaf_index": 0,
+ "leaf_value": 2.1594397038037663,
+ "leaf_weight": 469,
+ "leaf_count": 469
+ },
+ "right_child": {
+ "split_index": 1,
+ "split_feature": 3,
+ "split_gain": 41.27640151977539,
+ "threshold": "2||3||4",
+ "decision_type": "==",
+ "default_left": false,
+ "missing_type": "NaN",
+ "internal_value": 0.246035,
+ "internal_weight": 531,
+ "internal_count": 531,
+ "left_child": {
+ "leaf_index": 1,
+ "leaf_value": 2.235297305276056,
+ "leaf_weight": 302,
+ "leaf_count": 302
+ },
+ "right_child": {
+ "leaf_index": 2,
+ "leaf_value": 2.1792953471546546,
+ "leaf_weight": 229,
+ "leaf_count": 229
+ }
+ }
+ }
+ },
+ {
+ "tree_index": 1,
+ "num_leaves": 3,
+ "num_cat": 1,
+ "shrinkage": 0.1,
+ "tree_structure": {
+ "split_index": 0,
+ "split_feature": 2,
+ "split_gain": 64.22250366210938,
+ "threshold": "3||4",
+ "decision_type": "==",
+ "default_left": false,
+ "missing_type": "NaN",
+ "internal_value": 0,
+ "internal_weight": 0,
+ "internal_count": 1000,
+ "left_child": {
+ "leaf_index": 0,
+ "leaf_value": 0.03070842919354316,
+ "leaf_weight": 399,
+ "leaf_count": 399
+ },
+ "right_child": {
+ "split_index": 1,
+ "split_feature": 0,
+ "split_gain": 36.74250030517578,
+ "threshold": 0.5102250691730842,
+ "decision_type": "<=",
+ "default_left": true,
+ "missing_type": "NaN",
+ "internal_value": -0.204906,
+ "internal_weight": 601,
+ "internal_count": 601,
+ "left_child": {
+ "leaf_index": 1,
+ "leaf_value": -0.04439151147520909,
+ "leaf_weight": 315,
+ "leaf_count": 315
+ },
+ "right_child": {
+ "leaf_index": 2,
+ "leaf_value": 0.005117411709368601,
+ "leaf_weight": 286,
+ "leaf_count": 286
+ }
+ }
+ }
+ },
+ {
+ "tree_index": 2,
+ "num_leaves": 3,
+ "num_cat": 0,
+ "shrinkage": 0.1,
+ "tree_structure": {
+ "split_index": 0,
+ "split_feature": 1,
+ "split_gain": 57.1327018737793,
+ "threshold": 0.668665477622446,
+ "decision_type": "<=",
+ "default_left": true,
+ "missing_type": "NaN",
+ "internal_value": 0,
+ "internal_weight": 0,
+ "internal_count": 1000,
+ "left_child": {
+ "split_index": 1,
+ "split_feature": 1,
+ "split_gain": 40.859100341796875,
+ "threshold": 0.008118820676863816,
+ "decision_type": "<=",
+ "default_left": true,
+ "missing_type": "NaN",
+ "internal_value": -0.162926,
+ "internal_weight": 681,
+ "internal_count": 681,
+ "left_child": {
+ "leaf_index": 0,
+ "leaf_value": -0.15361238490967524,
+ "leaf_weight": 21,
+ "leaf_count": 21
+ },
+ "right_child": {
+ "leaf_index": 2,
+ "leaf_value": -0.01192330846157292,
+ "leaf_weight": 660,
+ "leaf_count": 660
+ }
+ },
+ "right_child": {
+ "leaf_index": 1,
+ "leaf_value": 0.03499044894987518,
+ "leaf_weight": 319,
+ "leaf_count": 319
+ }
+ }
+ },
+ {
+ "tree_index": 3,
+ "num_leaves": 3,
+ "num_cat": 1,
+ "shrinkage": 0.1,
+ "tree_structure": {
+ "split_index": 0,
+ "split_feature": 0,
+ "split_gain": 54.77090072631836,
+ "threshold": 0.5201391072644542,
+ "decision_type": "<=",
+ "default_left": true,
+ "missing_type": "NaN",
+ "internal_value": 0,
+ "internal_weight": 0,
+ "internal_count": 1000,
+ "left_child": {
+ "leaf_index": 0,
+ "leaf_value": -0.02141000620783247,
+ "leaf_weight": 543,
+ "leaf_count": 543
+ },
+ "right_child": {
+ "split_index": 1,
+ "split_feature": 2,
+ "split_gain": 27.200700759887695,
+ "threshold": "0||1",
+ "decision_type": "==",
+ "default_left": false,
+ "missing_type": "NaN",
+ "internal_value": 0.255704,
+ "internal_weight": 457,
+ "internal_count": 457,
+ "left_child": {
+ "leaf_index": 1,
+ "leaf_value": -0.004121485787596721,
+ "leaf_weight": 191,
+ "leaf_count": 191
+ },
+ "right_child": {
+ "leaf_index": 2,
+ "leaf_value": 0.04534090904886873,
+ "leaf_weight": 266,
+ "leaf_count": 266
+ }
+ }
+ }
+ },
+ {
+ "tree_index": 4,
+ "num_leaves": 3,
+ "num_cat": 1,
+ "shrinkage": 0.1,
+ "tree_structure": {
+ "split_index": 0,
+ "split_feature": 3,
+ "split_gain": 51.84349822998047,
+ "threshold": "2||3||4",
+ "decision_type": "==",
+ "default_left": false,
+ "missing_type": "NaN",
+ "internal_value": 0,
+ "internal_weight": 0,
+ "internal_count": 1000,
+ "left_child": {
+ "split_index": 1,
+ "split_feature": 1,
+ "split_gain": 39.352699279785156,
+ "threshold": 0.27283279016959255,
+ "decision_type": "<=",
+ "default_left": true,
+ "missing_type": "NaN",
+ "internal_value": 0.188414,
+ "internal_weight": 593,
+ "internal_count": 593,
+ "left_child": {
+ "leaf_index": 0,
+ "leaf_value": -0.01924803254356527,
+ "leaf_weight": 184,
+ "leaf_count": 184
+ },
+ "right_child": {
+ "leaf_index": 2,
+ "leaf_value": 0.03643772842347651,
+ "leaf_weight": 409,
+ "leaf_count": 409
+ }
+ },
+ "right_child": {
+ "leaf_index": 1,
+ "leaf_value": -0.02701711918923075,
+ "leaf_weight": 407,
+ "leaf_count": 407
+ }
+ }
+ }
+ ],
+ "pandas_categorical": [
+ [
+ "a",
+ "b",
+ "c",
+ "d",
+ "e"
+ ],
+ [
+ "i",
+ "j",
+ "k",
+ "l",
+ "m"
+ ]
+ ]
+} \ No newline at end of file
diff --git a/model-integration/src/test/models/lightgbm/train_lightgbm_classification.py b/model-integration/src/test/models/lightgbm/train_lightgbm_classification.py
new file mode 100755
index 00000000000..ac00437d192
--- /dev/null
+++ b/model-integration/src/test/models/lightgbm/train_lightgbm_classification.py
@@ -0,0 +1,54 @@
+#! /usr/bin/env python3
+# coding: utf-8
+
+import json
+import random
+
+import lightgbm as lgb
+import numpy as np
+import pandas as pd
+
+
+def category_value(arr):
+ values = { np.NaN: 0, "a":1, "b":2, "c":3, "d":4, "e":5, "i":1, "j":2, "k":3, "l":4, "m":5 }
+ return np.array([ 0.21 * values[i] for i in arr ])
+
+# Create training set
+num_examples = 100000
+missing_prob = 0.01
+features = pd.DataFrame({
+ "numerical_1": np.random.random(num_examples),
+ "numerical_2": np.random.random(num_examples),
+ "categorical_1": pd.Series(np.random.permutation(["a", "b", "c", "d", "e"] * int(num_examples/5)), dtype="category"),
+ "categorical_2": pd.Series(np.random.permutation(["i", "j", "k", "l", "m"] * int(num_examples/5)), dtype="category"),
+ })
+
+# randomly insert missing values
+for i in range(int(num_examples * len(features.columns) * missing_prob)):
+ features.loc[random.randint(0, num_examples-1), features.columns[random.randint(0, len(features.columns)-1)]] = None
+
+# create targets (with 0.0 as default for missing values)
+target = features["numerical_1"] + features["numerical_2"] + category_value(features["categorical_1"]) + category_value(features["categorical_2"])
+target = (target > 2.24) * 1.0
+lgb_train = lgb.Dataset(features, target)
+
+# Train model
+params = {
+ 'objective': 'binary',
+ 'metric': 'binary_logloss',
+ 'num_leaves': 3,
+}
+model = lgb.train(params, lgb_train, num_boost_round=5)
+
+# Save model
+with open("classification.json", "w") as f:
+ json.dump(model.dump_model(), f, indent=2)
+
+# Predict (for comparison with Vespa evaluation)
+predict_features = pd.DataFrame({
+ "numerical_1": pd.Series([ None, 0.1, None, 0.7]),
+ "numerical_2": pd.Series([np.NaN, 0.2, 0.5, 0.8]),
+ "categorical_1": pd.Series([ None, "a", "b", None], dtype="category"),
+ "categorical_2": pd.Series([ None, "i", "j", "m"], dtype="category"),
+ })
+print(model.predict(predict_features))
diff --git a/model-integration/src/test/models/lightgbm/train_lightgbm_regression.py b/model-integration/src/test/models/lightgbm/train_lightgbm_regression.py
new file mode 100755
index 00000000000..3e74e38da35
--- /dev/null
+++ b/model-integration/src/test/models/lightgbm/train_lightgbm_regression.py
@@ -0,0 +1,53 @@
+#! /usr/bin/env python3
+# coding: utf-8
+
+import json
+import random
+
+import lightgbm as lgb
+import numpy as np
+import pandas as pd
+
+
+def category_value(arr):
+ values = { np.NaN: 0, "a":1, "b":2, "c":3, "d":4, "e":5, "i":1, "j":2, "k":3, "l":4, "m":5 }
+ return np.array([ 0.21 * values[i] for i in arr ])
+
+# Create training set
+num_examples = 100000
+missing_prob = 0.01
+features = pd.DataFrame({
+ "numerical_1": np.random.random(num_examples),
+ "numerical_2": np.random.random(num_examples),
+ "categorical_1": pd.Series(np.random.permutation(["a", "b", "c", "d", "e"] * int(num_examples/5)), dtype="category"),
+ "categorical_2": pd.Series(np.random.permutation(["i", "j", "k", "l", "m"] * int(num_examples/5)), dtype="category"),
+ })
+
+# randomly insert missing values
+for i in range(int(num_examples * len(features.columns) * missing_prob)):
+ features.loc[random.randint(0, num_examples-1), features.columns[random.randint(0, len(features.columns)-1)]] = None
+
+# create targets (with 0.0 as default for missing values)
+target = features["numerical_1"] + features["numerical_2"] + category_value(features["categorical_1"]) + category_value(features["categorical_2"])
+lgb_train = lgb.Dataset(features, target)
+
+# Train model
+params = {
+ 'objective': 'mse',
+ 'metric': {'l2', 'l1'},
+ 'num_leaves': 3,
+}
+model = lgb.train(params, lgb_train, num_boost_round=2)
+
+# Save model
+with open("regression.json", "w") as f:
+ json.dump(model.dump_model(), f, indent=2)
+
+# Predict (for comparison with Vespa evaluation)
+predict_features = pd.DataFrame({
+ "numerical_1": pd.Series([ None, 0.1, None, 0.7]),
+ "numerical_2": pd.Series([np.NaN, 0.2, 0.5, 0.8]),
+ "categorical_1": pd.Series([ None, "a", "b", None], dtype="category"),
+ "categorical_2": pd.Series([ None, "i", "j", "m"], dtype="category"),
+ })
+print(model.predict(predict_features))
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java
index 68178418e62..790bbd03e45 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java
@@ -26,6 +26,9 @@ import com.yahoo.vespa.hosted.node.admin.maintenance.identity.CredentialsMaintai
import com.yahoo.vespa.hosted.node.admin.nodeadmin.ConvergenceException;
import java.nio.file.Path;
+import java.time.Clock;
+import java.time.Duration;
+import java.time.Instant;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
@@ -47,11 +50,10 @@ public class NodeAgentImpl implements NodeAgent {
// This is used as a definition of 1 GB when comparing flavor specs in node-repo
private static final long BYTES_IN_GB = 1_000_000_000L;
- private static final Logger logger = Logger.getLogger(NodeAgentImpl.class.getName());
+ // Container is started with uncapped CPU and is kept that way until the first successful health check + this duration
+ private static final Duration DEFAULT_WARM_UP_DURATION = Duration.ofSeconds(30);
- private final AtomicBoolean terminated = new AtomicBoolean(false);
- private boolean hasResumedNode = false;
- private boolean hasStartedServices = true;
+ private static final Logger logger = Logger.getLogger(NodeAgentImpl.class.getName());
private final NodeAgentContextSupplier contextSupplier;
private final NodeRepository nodeRepository;
@@ -61,12 +63,19 @@ public class NodeAgentImpl implements NodeAgent {
private final Optional<CredentialsMaintainer> credentialsMaintainer;
private final Optional<AclMaintainer> aclMaintainer;
private final Optional<HealthChecker> healthChecker;
+ private final Clock clock;
+ private final Duration warmUpDuration;
private final DoubleFlag containerCpuCap;
private Thread loopThread;
private ContainerState containerState = UNKNOWN;
private NodeSpec lastNode;
+ private final AtomicBoolean terminated = new AtomicBoolean(false);
+ private boolean hasResumedNode = false;
+ private boolean hasStartedServices = true;
+ private Optional<Instant> firstSuccessfulHealthCheckInstant = Optional.empty();
+
private int numberOfUnhandledException = 0;
private long currentRebootGeneration = 0;
private Optional<Long> currentRestartGeneration = Optional.empty();
@@ -87,16 +96,18 @@ public class NodeAgentImpl implements NodeAgent {
// Created in NodeAdminImpl
- public NodeAgentImpl(
- final NodeAgentContextSupplier contextSupplier,
- final NodeRepository nodeRepository,
- final Orchestrator orchestrator,
- final DockerOperations dockerOperations,
- final StorageMaintainer storageMaintainer,
- final FlagSource flagSource,
- final Optional<CredentialsMaintainer> credentialsMaintainer,
- final Optional<AclMaintainer> aclMaintainer,
- final Optional<HealthChecker> healthChecker) {
+ public NodeAgentImpl(NodeAgentContextSupplier contextSupplier, NodeRepository nodeRepository,
+ Orchestrator orchestrator, DockerOperations dockerOperations, StorageMaintainer storageMaintainer,
+ FlagSource flagSource, Optional<CredentialsMaintainer> credentialsMaintainer,
+ Optional<AclMaintainer> aclMaintainer, Optional<HealthChecker> healthChecker, Clock clock) {
+ this(contextSupplier, nodeRepository, orchestrator, dockerOperations, storageMaintainer, flagSource, credentialsMaintainer,
+ aclMaintainer, healthChecker, clock, DEFAULT_WARM_UP_DURATION);
+ }
+
+ public NodeAgentImpl(NodeAgentContextSupplier contextSupplier, NodeRepository nodeRepository,
+ Orchestrator orchestrator, DockerOperations dockerOperations, StorageMaintainer storageMaintainer,
+ FlagSource flagSource, Optional<CredentialsMaintainer> credentialsMaintainer,
+ Optional<AclMaintainer> aclMaintainer, Optional<HealthChecker> healthChecker, Clock clock, Duration warmUpDuration) {
this.contextSupplier = contextSupplier;
this.nodeRepository = nodeRepository;
this.orchestrator = orchestrator;
@@ -105,6 +116,8 @@ public class NodeAgentImpl implements NodeAgent {
this.credentialsMaintainer = credentialsMaintainer;
this.aclMaintainer = aclMaintainer;
this.healthChecker = healthChecker;
+ this.clock = clock;
+ this.warmUpDuration = warmUpDuration;
this.containerCpuCap = Flags.CONTAINER_CPU_CAP.bindTo(flagSource);
}
@@ -194,9 +207,11 @@ public class NodeAgentImpl implements NodeAgent {
}
}
- private void startContainer(NodeAgentContext context) {
+ private Container startContainer(NodeAgentContext context) {
ContainerData containerData = createContainerData(context);
- dockerOperations.createContainer(context, containerData, getContainerResources(context));
+ ContainerResources wantedResources = warmUpDuration.isNegative() ?
+ getContainerResources(context) : getContainerResources(context).withUnlimitedCpus();
+ dockerOperations.createContainer(context, containerData, wantedResources);
dockerOperations.startContainer(context);
currentRebootGeneration = context.node().wantedRebootGeneration();
@@ -204,6 +219,8 @@ public class NodeAgentImpl implements NodeAgent {
hasStartedServices = true; // Automatically started with the container
hasResumedNode = false;
context.log(logger, "Container successfully started, new containerState is " + containerState);
+ return dockerOperations.getContainer(context).orElseThrow(() ->
+ new ConvergenceException("Did not find container that was just started"));
}
private Optional<Container> removeContainerIfNeededUpdateContainerState(
@@ -250,6 +267,7 @@ public class NodeAgentImpl implements NodeAgent {
if (containerState == ABSENT) return;
try {
hasStartedServices = hasResumedNode = false;
+ firstSuccessfulHealthCheckInstant = Optional.empty();
dockerOperations.stopServices(context);
} catch (ContainerNotFoundException e) {
containerState = ABSENT;
@@ -333,9 +351,15 @@ public class NodeAgentImpl implements NodeAgent {
}
- private void updateContainerIfNeeded(NodeAgentContext context, Container existingContainer) {
+ private Container updateContainerIfNeeded(NodeAgentContext context, Container existingContainer) {
ContainerResources wantedContainerResources = getContainerResources(context);
- if (wantedContainerResources.equalsCpu(existingContainer.resources)) return;
+
+ if (healthChecker.isPresent() && firstSuccessfulHealthCheckInstant
+ .map(clock.instant().minus(warmUpDuration)::isBefore)
+ .orElse(true))
+ return existingContainer;
+
+ if (wantedContainerResources.equalsCpu(existingContainer.resources)) return existingContainer;
context.log(logger, "Container should be running with different CPU allocation, wanted: %s, current: %s",
wantedContainerResources.toStringCpu(), existingContainer.resources.toStringCpu());
@@ -343,6 +367,8 @@ public class NodeAgentImpl implements NodeAgent {
// Only update CPU resources
dockerOperations.updateContainer(context, wantedContainerResources.withMemoryBytes(existingContainer.resources.memoryBytes()));
+ return dockerOperations.getContainer(context).orElseThrow(() ->
+ new ConvergenceException("Did not find container that was just updated"));
}
private ContainerResources getContainerResources(NodeAgentContext context) {
@@ -430,16 +456,25 @@ public class NodeAgentImpl implements NodeAgent {
credentialsMaintainer.ifPresent(maintainer -> maintainer.converge(context));
if (container.isEmpty()) {
containerState = STARTING;
- startContainer(context);
+ container = Optional.of(startContainer(context));
containerState = UNKNOWN;
} else {
- updateContainerIfNeeded(context, container.get());
+ container = Optional.of(updateContainerIfNeeded(context, container.get()));
}
aclMaintainer.ifPresent(maintainer -> maintainer.converge(context));
startServicesIfNeeded(context);
resumeNodeIfNeeded(context);
- healthChecker.ifPresent(checker -> checker.verifyHealth(context));
+ if (healthChecker.isPresent()) {
+ healthChecker.get().verifyHealth(context);
+ if (firstSuccessfulHealthCheckInstant.isEmpty())
+ firstSuccessfulHealthCheckInstant = Optional.of(clock.instant());
+
+ Duration timeLeft = Duration.between(clock.instant(), firstSuccessfulHealthCheckInstant.get().plus(warmUpDuration));
+ if (!container.get().resources.equalsCpu(getContainerResources(context)))
+ throw new ConvergenceException("Refusing to resume until warm up period ends (" +
+ (timeLeft.isNegative() ? " next tick" : "in " + timeLeft) + ")");
+ }
// Because it's more important to stop a bad release from rolling out in prod,
// we put the resume call last. So if we fail after updating the node repo attributes
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerTester.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerTester.java
index 653a6f7f091..ea41ab21b9d 100644
--- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerTester.java
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerTester.java
@@ -86,18 +86,19 @@ public class DockerTester implements AutoCloseable {
.build();
nodeRepository.updateNodeRepositoryNode(hostSpec);
+ Clock clock = Clock.systemUTC();
+ Metrics metrics = new Metrics();
FileSystem fileSystem = TestFileSystem.create();
DockerOperations dockerOperations = new DockerOperationsImpl(docker, terminal, ipAddresses, flagSource);
- Metrics metrics = new Metrics();
NodeAgentFactory nodeAgentFactory = (contextSupplier, nodeContext) -> new NodeAgentImpl(
contextSupplier, nodeRepository, orchestrator, dockerOperations, storageMaintainer, flagSource,
- Optional.empty(), Optional.empty(), Optional.empty());
- nodeAdmin = new NodeAdminImpl(nodeAgentFactory, metrics, Clock.systemUTC(), Duration.ofMillis(10), Duration.ZERO);
+ Optional.empty(), Optional.empty(), Optional.empty(), clock, Duration.ofSeconds(-1));
+ nodeAdmin = new NodeAdminImpl(nodeAgentFactory, metrics, clock, Duration.ofMillis(10), Duration.ZERO);
NodeAgentContextFactory nodeAgentContextFactory = (nodeSpec, acl) ->
new NodeAgentContextImpl.Builder(nodeSpec).acl(acl).pathToContainerStorageFromFileSystem(fileSystem).build();
nodeAdminStateUpdater = new NodeAdminStateUpdater(nodeAgentContextFactory, nodeRepository, orchestrator,
- nodeAdmin, HOST_HOSTNAME, Clock.systemUTC());
+ nodeAdmin, HOST_HOSTNAME, clock);
loopThread = new Thread(() -> {
nodeAdminStateUpdater.start();
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java
index 9e1f84f07a1..f65ad379a63 100644
--- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java
@@ -6,6 +6,7 @@ import com.yahoo.config.provision.ApplicationId;
import com.yahoo.config.provision.DockerImage;
import com.yahoo.config.provision.NodeResources;
import com.yahoo.config.provision.NodeType;
+import com.yahoo.test.ManualClock;
import com.yahoo.vespa.flags.Flags;
import com.yahoo.vespa.flags.InMemoryFlagSource;
import com.yahoo.vespa.hosted.dockerapi.Container;
@@ -26,12 +27,16 @@ import com.yahoo.vespa.hosted.node.admin.nodeadmin.ConvergenceException;
import org.junit.Test;
import org.mockito.InOrder;
+import java.time.Duration;
+import java.time.Instant;
import java.util.Optional;
+import static org.junit.Assert.assertEquals;
import static org.junit.Assert.fail;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.doAnswer;
+import static org.mockito.Mockito.doNothing;
import static org.mockito.Mockito.doThrow;
import static org.mockito.Mockito.inOrder;
import static org.mockito.Mockito.mock;
@@ -65,6 +70,7 @@ public class NodeAgentImplTest {
private final HealthChecker healthChecker = mock(HealthChecker.class);
private final CredentialsMaintainer credentialsMaintainer = mock(CredentialsMaintainer.class);
private final InMemoryFlagSource flagSource = new InMemoryFlagSource();
+ private final ManualClock clock = new ManualClock(Instant.now());
@Test
@@ -641,6 +647,86 @@ public class NodeAgentImplTest {
verifyThatContainerIsStopped(NodeState.inactive, Optional.of(ApplicationId.defaultId()));
}
+ @Test
+ public void initial_cpu_cap_test() {
+ NodeSpec.Builder specBuilder = nodeBuilder
+ .state(NodeState.active)
+ .wantedDockerImage(dockerImage).currentDockerImage(dockerImage)
+ .wantedVespaVersion(vespaVersion).currentVespaVersion(vespaVersion)
+ .wantedRestartGeneration(1).currentRestartGeneration(1);
+
+ NodeAgentContext context = createContext(specBuilder.build());
+ NodeAgentImpl nodeAgent = makeNodeAgent(null, false, Duration.ofSeconds(30));
+
+ when(storageMaintainer.getDiskUsageFor(any())).thenReturn(Optional.of(201326592000L));
+
+ InOrder inOrder = inOrder(orchestrator, dockerOperations);
+
+ ConvergenceException healthCheckException = new ConvergenceException("Not yet up");
+ doThrow(healthCheckException).when(healthChecker).verifyHealth(any());
+ for (int i = 0; i < 3; i++) {
+ try {
+ nodeAgent.doConverge(context);
+ fail("Expected to fail with health check exception");
+ } catch (ConvergenceException e) {
+ assertEquals(healthCheckException, e);
+ }
+ clock.advance(Duration.ofSeconds(30));
+ }
+
+ doNothing().when(healthChecker).verifyHealth(any());
+ try {
+ nodeAgent.doConverge(context);
+ fail("Expected to fail due to warm up period not yet done");
+ } catch (ConvergenceException e) {
+ assertEquals("Refusing to resume until warm up period ends (in PT30S)", e.getMessage());
+ }
+ inOrder.verify(orchestrator, never()).resume(any());
+ inOrder.verify(orchestrator, never()).suspend(any());
+ inOrder.verify(dockerOperations, never()).updateContainer(any(), any());
+
+
+ clock.advance(Duration.ofSeconds(31));
+ nodeAgent.doConverge(context);
+
+ inOrder.verify(dockerOperations).updateContainer(eq(context), eq(ContainerResources.from(0, 2, 16)));
+ inOrder.verify(dockerOperations, never()).removeContainer(any(), any());
+ inOrder.verify(dockerOperations, never()).startContainer(any());
+ inOrder.verify(orchestrator).resume(any(String.class));
+
+ // No changes
+ nodeAgent.converge(context);
+ inOrder.verify(orchestrator, never()).suspend(any(String.class));
+ inOrder.verify(dockerOperations, never()).updateContainer(eq(context), any());
+ inOrder.verify(dockerOperations, never()).removeContainer(any(), any());
+ inOrder.verify(orchestrator).resume(any(String.class));
+ }
+
+ @Test
+ public void resumes_normally_if_container_is_already_capped_on_start() {
+ NodeSpec.Builder specBuilder = nodeBuilder
+ .state(NodeState.active)
+ .wantedDockerImage(dockerImage).currentDockerImage(dockerImage)
+ .wantedVespaVersion(vespaVersion).currentVespaVersion(vespaVersion)
+ .wantedRestartGeneration(1).currentRestartGeneration(1);
+
+ NodeAgentContext context = createContext(specBuilder.build());
+ NodeAgentImpl nodeAgent = makeNodeAgent(dockerImage, true, Duration.ofSeconds(30));
+ mockGetContainer(dockerImage, ContainerResources.from(0, 2, 16), true);
+
+ when(storageMaintainer.getDiskUsageFor(any())).thenReturn(Optional.of(201326592000L));
+
+ InOrder inOrder = inOrder(orchestrator, dockerOperations);
+
+ nodeAgent.doConverge(context);
+
+ nodeAgent.converge(context);
+ inOrder.verify(orchestrator, never()).suspend(any(String.class));
+ inOrder.verify(dockerOperations, never()).updateContainer(eq(context), any());
+ inOrder.verify(dockerOperations, never()).removeContainer(any(), any());
+ inOrder.verify(orchestrator).resume(any(String.class));
+ }
+
private void verifyThatContainerIsStopped(NodeState nodeState, Optional<ApplicationId> owner) {
NodeSpec.Builder nodeBuilder = new NodeSpec.Builder()
.resources(resources)
@@ -672,11 +758,28 @@ public class NodeAgentImplTest {
}
private NodeAgentImpl makeNodeAgent(DockerImage dockerImage, boolean isRunning) {
+ return makeNodeAgent(dockerImage, isRunning, Duration.ofSeconds(-1));
+ }
+
+ private NodeAgentImpl makeNodeAgent(DockerImage dockerImage, boolean isRunning, Duration warmUpDuration) {
mockGetContainer(dockerImage, isRunning);
+ doAnswer(invoc -> {
+ NodeAgentContext context = invoc.getArgument(0, NodeAgentContext.class);
+ ContainerResources resources = invoc.getArgument(2, ContainerResources.class);
+ mockGetContainer(context.node().wantedDockerImage().get(), resources, true);
+ return null;
+ }).when(dockerOperations).createContainer(any(), any(), any());
+
+ doAnswer(invoc -> {
+ NodeAgentContext context = invoc.getArgument(0, NodeAgentContext.class);
+ ContainerResources resources = invoc.getArgument(1, ContainerResources.class);
+ mockGetContainer(context.node().wantedDockerImage().get(), resources, true);
+ return null;
+ }).when(dockerOperations).updateContainer(any(), any());
return new NodeAgentImpl(contextSupplier, nodeRepository, orchestrator, dockerOperations,
storageMaintainer, flagSource, Optional.of(credentialsMaintainer), Optional.of(aclMaintainer),
- Optional.of(healthChecker));
+ Optional.of(healthChecker), clock, warmUpDuration);
}
private void mockGetContainer(DockerImage dockerImage, boolean isRunning) {
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporter.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporter.java
index 970972a5fe6..ed6e7cc71ef 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporter.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporter.java
@@ -51,7 +51,7 @@ public class MetricsReporter extends Maintainer {
Clock clock) {
super(nodeRepository, interval);
this.metric = metric;
- this.orchestrator = orchestrator.getNodeStatuses();
+ this.orchestrator = orchestrator.getHostResolver();
this.serviceMonitor = serviceMonitor;
this.pendingRedeploymentsSupplier = pendingRedeploymentsSupplier;
this.clock = clock;
@@ -129,17 +129,15 @@ public class MetricsReporter extends Maintainer {
metric.set("wantToDeprovision", node.status().wantToDeprovision() ? 1 : 0, context);
metric.set("failReport", NodeFailer.reasonsToFailParentHost(node).isEmpty() ? 0 : 1, context);
- orchestrator.apply(new HostName(node.hostname()))
- .ifPresent(info -> {
- int suspended = info.status().isSuspended() ? 1 : 0;
- metric.set("suspended", suspended, context);
- metric.set("allowedToBeDown", suspended, context); // remove summer 2020.
-
- info.suspendedSince().ifPresent(suspendedSince -> {
- Duration duration = Duration.between(suspendedSince, clock.instant());
- metric.set("suspendedSeconds", duration.getSeconds(), context);
- });
- });
+ orchestrator.apply(new HostName(node.hostname())).ifPresent(info -> {
+ int suspended = info.status().isSuspended() ? 1 : 0;
+ metric.set("suspended", suspended, context);
+ metric.set("allowedToBeDown", suspended, context); // remove summer 2020.
+ long suspendedSeconds = info.suspendedSince()
+ .map(suspendedSince -> Duration.between(suspendedSince, clock.instant()).getSeconds())
+ .orElse(0L);
+ metric.set("suspendedSeconds", suspendedSeconds, context);
+ });
long numberOfServices;
HostName hostName = new HostName(node.hostname());
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java
index b80446b06da..ca53b215237 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java
@@ -265,7 +265,7 @@ public class NodeFailer extends Maintainer {
private boolean nodeSuspended(Node node) {
try {
- return orchestrator.getNodeStatus(new HostName(node.hostname())) == HostStatus.ALLOWED_TO_BE_DOWN;
+ return orchestrator.getNodeStatus(new HostName(node.hostname())).isSuspended();
} catch (HostNameNotFoundException e) {
// Treat it as not suspended
return false;
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesResponse.java
index 8a2156b0ce3..8ca8dfc26f6 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesResponse.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesResponse.java
@@ -16,12 +16,10 @@ import com.yahoo.slime.Slime;
import com.yahoo.vespa.applicationmodel.HostName;
import com.yahoo.vespa.hosted.provision.Node;
import com.yahoo.vespa.hosted.provision.NodeRepository;
-import com.yahoo.vespa.hosted.provision.node.Agent;
import com.yahoo.vespa.hosted.provision.node.History;
import com.yahoo.vespa.hosted.provision.node.filter.NodeFilter;
import com.yahoo.vespa.orchestrator.Orchestrator;
import com.yahoo.vespa.orchestrator.status.HostInfo;
-import com.yahoo.vespa.orchestrator.status.HostStatus;
import java.io.IOException;
import java.io.OutputStream;
@@ -59,7 +57,7 @@ class NodesResponse extends HttpResponse {
this.nodeParentUrl = toNodeParentUrl(request);
filter = NodesApiHandler.toNodeFilter(request);
this.recursive = request.getBooleanProperty("recursive");
- this.orchestrator = orchestrator.getNodeStatuses();
+ this.orchestrator = orchestrator.getHostResolver();
this.nodeRepository = nodeRepository;
slime = new Slime();
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/ContainerConfig.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/ContainerConfig.java
index 888b8835e49..49d0ba5cf70 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/ContainerConfig.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/ContainerConfig.java
@@ -13,7 +13,8 @@ public class ContainerConfig {
return "<container version='1.0'>\n" +
" <config name=\"container.handler.threadpool\">\n" +
" <maxthreads>20</maxthreads>\n" +
- " </config> \n" +
+ " </config>\n" +
+ " <accesslog type='disabled'/>\n" +
" <component id='com.yahoo.test.ManualClock'/>\n" +
" <component id='com.yahoo.vespa.curator.mock.MockCurator'/>\n" +
" <component id='com.yahoo.vespa.hosted.provision.testutils.OrchestratorMock'/>\n" +
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/OrchestratorMock.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/OrchestratorMock.java
index 3e671f03adf..d183e62c96c 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/OrchestratorMock.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/OrchestratorMock.java
@@ -12,8 +12,10 @@ import com.yahoo.vespa.orchestrator.status.HostStatus;
import java.time.Instant;
import java.util.Collections;
+import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
+import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.function.Function;
@@ -23,7 +25,7 @@ import java.util.function.Function;
*/
public class OrchestratorMock extends AbstractComponent implements Orchestrator {
- private final Set<HostName> suspendedHosts = new HashSet<>();
+ private final Map<HostName, HostInfo> suspendedHosts = new HashMap<>();
private final Set<ApplicationId> suspendedApplications = new HashSet<>();
@Override
@@ -33,14 +35,13 @@ public class OrchestratorMock extends AbstractComponent implements Orchestrator
@Override
public HostStatus getNodeStatus(HostName hostName) {
- return suspendedHosts.contains(hostName) ? HostStatus.ALLOWED_TO_BE_DOWN : HostStatus.NO_REMARKS;
+ HostInfo hostInfo = suspendedHosts.get(hostName);
+ return hostInfo == null ? HostStatus.NO_REMARKS : hostInfo.status();
}
@Override
- public Function<HostName, Optional<HostInfo>> getNodeStatuses() {
- return hostName -> Optional.of(getNodeStatus(hostName))
- .map(status -> status.isSuspended() ? HostInfo.createSuspended(status, Instant.EPOCH)
- : HostInfo.createNoRemarks());
+ public Function<HostName, Optional<HostInfo>> getHostResolver() {
+ return hostName -> Optional.of(suspendedHosts.getOrDefault(hostName, HostInfo.createNoRemarks()));
}
@Override
@@ -53,7 +54,7 @@ public class OrchestratorMock extends AbstractComponent implements Orchestrator
@Override
public void suspend(HostName hostName) {
- suspendedHosts.add(hostName);
+ suspendedHosts.put(hostName, HostInfo.createSuspended(HostStatus.ALLOWED_TO_BE_DOWN, Instant.EPOCH));
}
@Override
@@ -78,7 +79,7 @@ public class OrchestratorMock extends AbstractComponent implements Orchestrator
}
@Override
- public void acquirePermissionToRemove(HostName hostName) {}
+ public void acquirePermissionToRemove(HostName hostName) { }
@Override
public void suspendAll(HostName parentHostname, List<HostName> hostNames) {
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporterTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporterTest.java
index 287842e56f0..672709a2f8f 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporterTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporterTest.java
@@ -93,7 +93,7 @@ public class MetricsReporterTest {
ManualClock clock = new ManualClock(Instant.ofEpochSecond(124));
Orchestrator orchestrator = mock(Orchestrator.class);
ServiceMonitor serviceMonitor = mock(ServiceMonitor.class);
- when(orchestrator.getNodeStatuses()).thenReturn(hostName ->
+ when(orchestrator.getHostResolver()).thenReturn(hostName ->
Optional.of(HostInfo.createSuspended(HostStatus.ALLOWED_TO_BE_DOWN, Instant.ofEpochSecond(1)))
);
ServiceModel serviceModel = mock(ServiceModel.class);
@@ -144,7 +144,7 @@ public class MetricsReporterTest {
Orchestrator orchestrator = mock(Orchestrator.class);
ServiceMonitor serviceMonitor = mock(ServiceMonitor.class);
- when(orchestrator.getNodeStatuses()).thenReturn(hostName -> Optional.of(HostInfo.createNoRemarks()));
+ when(orchestrator.getHostResolver()).thenReturn(hostName -> Optional.of(HostInfo.createNoRemarks()));
ServiceModel serviceModel = mock(ServiceModel.class);
when(serviceMonitor.getServiceModelSnapshot()).thenReturn(serviceModel);
when(serviceModel.getServiceInstancesByHostName()).thenReturn(Map.of());
diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/Orchestrator.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/Orchestrator.java
index d1331b70251..10fa10f1150 100644
--- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/Orchestrator.java
+++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/Orchestrator.java
@@ -50,12 +50,15 @@ public interface Orchestrator {
HostStatus getNodeStatus(HostName hostName) throws HostNameNotFoundException;
/**
- * Returns a not necessarily consistent mapping from host names to their statuses, for hosts known by this.
+ * Returns a lambda, which when invoked with a hostname, returns its current host info.
+ *
+ * <p>When invoked multiple times, the hostname/host info mapping is not necessarily consistent.
+ * Prefer this to {@link #getNodeStatus(HostName)} when consistency is not required,
+ * and when doing bulk reads.</p>
*
- * Prefer this to {@link #getNodeStatus(HostName)} when consistency is not required, and when doing bulk reads.
* @return a mapping from host names to their statuses. Unknown hosts map to {@code Optional.empty()}.
*/
- Function<HostName, Optional<HostInfo>> getNodeStatuses();
+ Function<HostName, Optional<HostInfo>> getHostResolver();
void setNodeStatus(HostName hostName, HostStatus state) throws OrchestrationException;
diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorImpl.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorImpl.java
index f0b7c2eead1..05268232119 100644
--- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorImpl.java
+++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorImpl.java
@@ -28,6 +28,7 @@ import com.yahoo.vespa.orchestrator.policy.HostedVespaPolicy;
import com.yahoo.vespa.orchestrator.policy.Policy;
import com.yahoo.vespa.orchestrator.status.ApplicationInstanceStatus;
import com.yahoo.vespa.orchestrator.status.HostInfo;
+import com.yahoo.vespa.orchestrator.status.HostInfos;
import com.yahoo.vespa.orchestrator.status.HostStatus;
import com.yahoo.vespa.orchestrator.status.MutableStatusRegistry;
import com.yahoo.vespa.orchestrator.status.StatusService;
@@ -113,7 +114,7 @@ public class OrchestratorImpl implements Orchestrator {
}
@Override
- public Function<HostName, Optional<HostInfo>> getNodeStatuses() {
+ public Function<HostName, Optional<HostInfo>> getHostResolver() {
return hostName -> instanceLookupService.findInstanceByHost(hostName)
.map(application -> statusService.getHostInfo(application.reference(), hostName));
}
@@ -253,6 +254,8 @@ public class OrchestratorImpl implements Orchestrator {
suspendGroup(context.createSubcontextForSingleAppOp(probe), nodeGroup);
} catch (HostStateChangeDeniedException e) {
throw new BatchHostStateChangeDeniedException(parentHostname, nodeGroup, e);
+ } catch (UncheckedTimeoutException e) {
+ throw e;
} catch (RuntimeException e) {
throw new BatchInternalErrorException(parentHostname, nodeGroup, e);
}
@@ -334,9 +337,15 @@ public class OrchestratorImpl implements Orchestrator {
if (status == ApplicationInstanceStatus.ALLOWED_TO_BE_DOWN) {
ApplicationInstance application = getApplicationInstance(appRef);
+ HostInfos hostInfosSnapshot = statusRegistry.getHostInfos();
+
// Mark it allowed to be down before we manipulate the clustercontroller
OrchestratorUtil.getHostsUsedByApplicationInstance(application)
- .forEach(h -> statusRegistry.setHostState(h, HostStatus.ALLOWED_TO_BE_DOWN));
+ .stream()
+ // This filter also ensures host status is not modified if a suspended host
+ // has status != ALLOWED_TO_BE_DOWN.
+ .filter(hostname -> !hostInfosSnapshot.getOrNoRemarks(hostname).status().isSuspended())
+ .forEach(hostname -> statusRegistry.setHostState(hostname, HostStatus.ALLOWED_TO_BE_DOWN));
// If the clustercontroller throws an error the nodes will be marked as allowed to be down
// and be set back up on next resume invocation.
diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ApplicationApiImpl.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ApplicationApiImpl.java
index eb5dcf790ba..def5edd0a97 100644
--- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ApplicationApiImpl.java
+++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ApplicationApiImpl.java
@@ -10,6 +10,7 @@ import com.yahoo.vespa.orchestrator.OrchestratorContext;
import com.yahoo.vespa.orchestrator.OrchestratorUtil;
import com.yahoo.vespa.orchestrator.controller.ClusterControllerClientFactory;
import com.yahoo.vespa.orchestrator.status.ApplicationInstanceStatus;
+import com.yahoo.vespa.orchestrator.status.HostInfos;
import com.yahoo.vespa.orchestrator.status.HostStatus;
import com.yahoo.vespa.orchestrator.status.MutableStatusRegistry;
@@ -17,10 +18,8 @@ import java.util.Collection;
import java.util.Comparator;
import java.util.HashSet;
import java.util.List;
-import java.util.Map;
import java.util.Optional;
import java.util.Set;
-import java.util.function.Function;
import java.util.stream.Collectors;
import static com.yahoo.vespa.orchestrator.OrchestratorUtil.getHostsUsedByApplicationInstance;
@@ -34,7 +33,7 @@ public class ApplicationApiImpl implements ApplicationApi {
private final NodeGroup nodeGroup;
private final MutableStatusRegistry hostStatusService;
private final List<ClusterApi> clusterInOrder;
- private final Map<HostName, HostStatus> hostStatusMap;
+ private final HostInfos hostInfos;
public ApplicationApiImpl(NodeGroup nodeGroup,
MutableStatusRegistry hostStatusService,
@@ -44,10 +43,8 @@ public class ApplicationApiImpl implements ApplicationApi {
this.nodeGroup = nodeGroup;
this.hostStatusService = hostStatusService;
Collection<HostName> hosts = getHostsUsedByApplicationInstance(applicationInstance);
- this.hostStatusMap = hosts.stream().collect(Collectors.toMap(Function.identity(),
- hostName -> hostStatusService.getSuspendedHosts().contains(hostName)
- ? HostStatus.ALLOWED_TO_BE_DOWN : HostStatus.NO_REMARKS));
- this.clusterInOrder = makeClustersInOrder(nodeGroup, hostStatusMap, clusterControllerClientFactory, numberOfConfigServers);
+ this.hostInfos = hostStatusService.getHostInfos();
+ this.clusterInOrder = makeClustersInOrder(nodeGroup, hostInfos, clusterControllerClientFactory, numberOfConfigServers);
}
@Override
@@ -56,7 +53,7 @@ public class ApplicationApiImpl implements ApplicationApi {
}
private HostStatus getHostStatus(HostName hostName) {
- return hostStatusMap.getOrDefault(hostName, HostStatus.NO_REMARKS);
+ return hostInfos.getOrNoRemarks(hostName).status();
}
@Override
@@ -67,6 +64,7 @@ public class ApplicationApiImpl implements ApplicationApi {
@Override
public List<StorageNode> getStorageNodesAllowedToBeDownInGroupInReverseClusterOrder() {
return getStorageNodesInGroupInClusterOrder().stream()
+ // PERMANENTLY_DOWN nodes are NOT included.
.filter(storageNode -> getHostStatus(storageNode.hostName()) == HostStatus.ALLOWED_TO_BE_DOWN)
.sorted(Comparator.reverseOrder())
.collect(Collectors.toList());
@@ -112,7 +110,8 @@ public class ApplicationApiImpl implements ApplicationApi {
.collect(Collectors.toList());
}
- private List<ClusterApi> makeClustersInOrder(NodeGroup nodeGroup, Map<HostName, HostStatus> hostStatusMap,
+ private List<ClusterApi> makeClustersInOrder(NodeGroup nodeGroup,
+ HostInfos hostInfos,
ClusterControllerClientFactory clusterControllerClientFactory,
int numberOfConfigServers) {
Set<ServiceCluster> clustersInGroup = getServiceClustersInGroup(nodeGroup);
@@ -121,7 +120,7 @@ public class ApplicationApiImpl implements ApplicationApi {
this,
serviceCluster,
nodeGroup,
- hostStatusMap,
+ hostInfos,
clusterControllerClientFactory,
numberOfConfigServers))
.sorted(ApplicationApiImpl::compareClusters)
diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ClusterApiImpl.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ClusterApiImpl.java
index 2aaebb18aa6..b747d8c2e22 100644
--- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ClusterApiImpl.java
+++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ClusterApiImpl.java
@@ -8,6 +8,7 @@ import com.yahoo.vespa.applicationmodel.ServiceInstance;
import com.yahoo.vespa.applicationmodel.ServiceStatus;
import com.yahoo.vespa.applicationmodel.ServiceType;
import com.yahoo.vespa.orchestrator.controller.ClusterControllerClientFactory;
+import com.yahoo.vespa.orchestrator.status.HostInfos;
import com.yahoo.vespa.orchestrator.status.HostStatus;
import java.util.Collections;
@@ -27,7 +28,7 @@ class ClusterApiImpl implements ClusterApi {
private final ApplicationApi applicationApi;
private final ServiceCluster serviceCluster;
private final NodeGroup nodeGroup;
- private final Map<HostName, HostStatus> hostStatusMap;
+ private final HostInfos hostInfos;
private final ClusterControllerClientFactory clusterControllerClientFactory;
private final Set<ServiceInstance> servicesInGroup;
private final Set<ServiceInstance> servicesDownInGroup;
@@ -50,13 +51,13 @@ class ClusterApiImpl implements ClusterApi {
public ClusterApiImpl(ApplicationApi applicationApi,
ServiceCluster serviceCluster,
NodeGroup nodeGroup,
- Map<HostName, HostStatus> hostStatusMap,
+ HostInfos hostInfos,
ClusterControllerClientFactory clusterControllerClientFactory,
int numberOfConfigServers) {
this.applicationApi = applicationApi;
this.serviceCluster = serviceCluster;
this.nodeGroup = nodeGroup;
- this.hostStatusMap = hostStatusMap;
+ this.hostInfos = hostInfos;
this.clusterControllerClientFactory = clusterControllerClientFactory;
Map<Boolean, Set<ServiceInstance>> serviceInstancesByLocality =
@@ -144,7 +145,7 @@ class ClusterApiImpl implements ClusterApi {
public String nodesAllowedToBeDownNotInGroupDescription() {
return servicesNotInGroup.stream()
.map(ServiceInstance::hostName)
- .filter(hostName -> hostStatus(hostName) == HostStatus.ALLOWED_TO_BE_DOWN)
+ .filter(hostName -> hostStatus(hostName).isSuspended())
.sorted()
.distinct()
.collect(Collectors.toList())
@@ -206,11 +207,11 @@ class ClusterApiImpl implements ClusterApi {
}
private HostStatus hostStatus(HostName hostName) {
- return hostStatusMap.getOrDefault(hostName, HostStatus.NO_REMARKS);
+ return hostInfos.getOrNoRemarks(hostName).status();
}
private boolean serviceEffectivelyDown(ServiceInstance service) {
- if (hostStatus(service.hostName()) == HostStatus.ALLOWED_TO_BE_DOWN) {
+ if (hostStatus(service.hostName()).isSuspended()) {
return true;
}
diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/InstanceResource.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/InstanceResource.java
index bd265ed39e4..7b8a74d7fe2 100644
--- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/InstanceResource.java
+++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/InstanceResource.java
@@ -14,7 +14,7 @@ import com.yahoo.vespa.applicationmodel.ServiceType;
import com.yahoo.vespa.orchestrator.InstanceLookupService;
import com.yahoo.vespa.orchestrator.OrchestratorUtil;
import com.yahoo.vespa.orchestrator.restapi.wire.SlobrokEntryResponse;
-import com.yahoo.vespa.orchestrator.status.HostStatus;
+import com.yahoo.vespa.orchestrator.status.HostInfos;
import com.yahoo.vespa.orchestrator.status.StatusService;
import com.yahoo.vespa.service.manager.MonitorManager;
import com.yahoo.vespa.service.manager.UnionMonitorManager;
@@ -81,12 +81,11 @@ public class InstanceResource {
= instanceLookupService.findInstanceById(instanceId)
.orElseThrow(() -> new WebApplicationException(Response.status(Response.Status.NOT_FOUND).build()));
- Set<HostName> suspendedHosts = statusService.getSuspendedHostsByApplication().apply(applicationInstance.reference());
+ HostInfos hostInfos = statusService.getHostInfosByApplicationResolver().apply(applicationInstance.reference());
Map<HostName, String> hostStatusMap = getHostsUsedByApplicationInstance(applicationInstance)
.stream()
.collect(Collectors.toMap(hostName -> hostName,
- hostName -> suspendedHosts.contains(hostName) ? HostStatus.ALLOWED_TO_BE_DOWN.name()
- : HostStatus.NO_REMARKS.name()));
+ hostName -> hostInfos.getOrNoRemarks(hostName).status().asString()));
return InstanceStatusResponse.create(applicationInstance, hostStatusMap);
}
diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/InstanceStatusResponse.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/InstanceStatusResponse.java
index c2ea0c9eddf..068423f7d24 100644
--- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/InstanceStatusResponse.java
+++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/InstanceStatusResponse.java
@@ -4,6 +4,7 @@ package com.yahoo.vespa.orchestrator.resources;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.yahoo.vespa.applicationmodel.ApplicationInstance;
import com.yahoo.vespa.applicationmodel.HostName;
+import com.yahoo.vespa.orchestrator.status.HostInfo;
import java.util.Map;
import java.util.Objects;
diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/HostInfos.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/HostInfos.java
index fe78a41672b..1acd82662a4 100644
--- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/HostInfos.java
+++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/HostInfos.java
@@ -8,10 +8,11 @@ import java.util.Set;
import java.util.stream.Collectors;
/**
- * Collection of suspended hosts.
+ * Collection of the suspended hosts of an application.
*
* @author hakonhall
*/
+// @Immutable
public class HostInfos {
private final Map<HostName, HostInfo> hostInfos;
@@ -23,16 +24,8 @@ public class HostInfos {
this.hostInfos = Map.of();
}
- /** Get all suspended hostnames. */
- public Set<HostName> suspendedHostsnames() {
- return hostInfos.entrySet().stream()
- .filter(entry -> entry.getValue().status().isSuspended())
- .map(entry -> entry.getKey())
- .collect(Collectors.toSet());
- }
-
/** Get host info for hostname, returning a NO_REMARKS HostInfo if unknown. */
- public HostInfo get(HostName hostname) {
+ public HostInfo getOrNoRemarks(HostName hostname) {
return hostInfos.getOrDefault(hostname, HostInfo.createNoRemarks());
}
}
diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/HostInfosCache.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/HostInfosCache.java
index 9998e7de918..680f3cbcc6d 100644
--- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/HostInfosCache.java
+++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/HostInfosCache.java
@@ -28,17 +28,24 @@ public class HostInfosCache implements HostInfosService {
this.cacheGeneration = new AtomicLong(counter.get());
}
- @Override
- public HostInfos getHostInfos(ApplicationInstanceReference application) {
+ public void refreshCache() {
long newCacheGeneration = counter.get();
if (cacheGeneration.getAndSet(newCacheGeneration) != newCacheGeneration) {
suspendedHosts.clear();
}
+ }
+ public HostInfos getCachedHostInfos(ApplicationInstanceReference application) {
return suspendedHosts.computeIfAbsent(application, wrappedService::getHostInfos);
}
@Override
+ public HostInfos getHostInfos(ApplicationInstanceReference application) {
+ refreshCache();
+ return getCachedHostInfos(application);
+ }
+
+ @Override
public boolean setHostStatus(ApplicationInstanceReference application, HostName hostName, HostStatus hostStatus) {
boolean isException = true;
boolean modified = false;
diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/HostStatus.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/HostStatus.java
index 6764ffb48ea..330e53f9586 100644
--- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/HostStatus.java
+++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/HostStatus.java
@@ -23,4 +23,5 @@ public enum HostStatus {
HostStatus(boolean suspended) { this.suspended = suspended; }
public boolean isSuspended() { return suspended; }
+ public String asString() { return name(); }
}
diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/MutableStatusRegistry.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/MutableStatusRegistry.java
index 92b0ec50011..d2042dc9fd2 100644
--- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/MutableStatusRegistry.java
+++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/MutableStatusRegistry.java
@@ -14,22 +14,16 @@ import java.util.Set;
*/
public interface MutableStatusRegistry extends AutoCloseable {
- /**
- * Returns the status of this application.
- */
+ /** Returns the status of this application. */
ApplicationInstanceStatus getStatus();
/** Returns the host info of the given host. */
HostInfo getHostInfo(HostName hostName);
- /**
- * Returns the set of all suspended hosts for this application.
- */
- Set<HostName> getSuspendedHosts();
+ /** Returns a snapshot of all host infos for this application. */
+ HostInfos getHostInfos();
- /**
- * Sets the state for the given host.
- */
+ /** Sets the state for the given host. */
void setHostState(HostName hostName, HostStatus status);
/**
diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/StatusService.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/StatusService.java
index 9e3ee84e1d9..e2be5ec7eb6 100644
--- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/StatusService.java
+++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/StatusService.java
@@ -57,12 +57,12 @@ public interface StatusService {
Set<ApplicationInstanceReference> getAllSuspendedApplications();
/**
- * Returns a fresh, but not necessarily consistent mapping from applications to their set of suspended hosts.
+ * Returns a lambda, which when invoked for an application, returns an up-to-date snapshot of {@link HostInfos host infos}.
*
- * If the lock for an application is held when this mapping is acquired, new sets returned for that application
- * are consistent and up to date for as long as the lock is held. (The sets themselves don't reflect changes.)
+ * <p>Unless the lock for the application is held, the returned snapshot may already be out of date.
+ * (The snapshot itself is immutable.)</p>
*/
- Function<ApplicationInstanceReference, Set<HostName>> getSuspendedHostsByApplication();
+ Function<ApplicationInstanceReference, HostInfos> getHostInfosByApplicationResolver();
/** Returns the status of the given application. This is consistent if its lock is held.*/
ApplicationInstanceStatus getApplicationInstanceStatus(ApplicationInstanceReference application);
diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/ZookeeperStatusService.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/ZookeeperStatusService.java
index c4a23baae2e..e192f25d68c 100644
--- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/ZookeeperStatusService.java
+++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/ZookeeperStatusService.java
@@ -99,12 +99,13 @@ public class ZookeeperStatusService implements StatusService {
/**
* Cache is checked for freshness when this mapping is created, and may be invalidated again later
- * by other users of the cache. Since this function is backed by the cache, any such invalidations
+ * by other users of the cache. Since this function is backed by the cache, any such invalidation
* will be reflected in the returned mapping; all users of the cache collaborate in repopulating it.
*/
@Override
- public Function<ApplicationInstanceReference, Set<HostName>> getSuspendedHostsByApplication() {
- return application -> hostInfosCache.getHostInfos(application).suspendedHostsnames();
+ public Function<ApplicationInstanceReference, HostInfos> getHostInfosByApplicationResolver() {
+ hostInfosCache.refreshCache();
+ return hostInfosCache::getCachedHostInfos;
}
@@ -255,7 +256,7 @@ public class ZookeeperStatusService implements StatusService {
@Override
public HostInfo getHostInfo(ApplicationInstanceReference applicationInstanceReference, HostName hostName) {
- return hostInfosCache.getHostInfos(applicationInstanceReference).get(hostName);
+ return hostInfosCache.getHostInfos(applicationInstanceReference).getOrNoRemarks(hostName);
}
/** Do not call this directly: should be called behind a cache. */
@@ -362,8 +363,8 @@ public class ZookeeperStatusService implements StatusService {
}
@Override
- public Set<HostName> getSuspendedHosts() {
- return hostInfosCache.getHostInfos(applicationInstanceReference).suspendedHostsnames();
+ public HostInfos getHostInfos() {
+ return hostInfosCache.getHostInfos(applicationInstanceReference);
}
@Override
diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/model/ClusterApiImplTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/model/ClusterApiImplTest.java
index 8e42ccdc79d..62925dc003e 100644
--- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/model/ClusterApiImplTest.java
+++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/model/ClusterApiImplTest.java
@@ -13,6 +13,7 @@ import com.yahoo.vespa.applicationmodel.TenantId;
import com.yahoo.vespa.orchestrator.OrchestratorUtil;
import com.yahoo.vespa.orchestrator.policy.HostStateChangeDeniedException;
import com.yahoo.vespa.orchestrator.policy.HostedVespaClusterPolicy;
+import com.yahoo.vespa.orchestrator.status.HostInfos;
import com.yahoo.vespa.orchestrator.status.HostStatus;
import org.junit.Test;
@@ -75,7 +76,7 @@ public class ClusterApiImplTest {
applicationApi,
serviceCluster,
new NodeGroup(modelUtils.createApplicationInstance(new ArrayList<>()), hostName5),
- modelUtils.getHostStatusMap(),
+ modelUtils.getHostInfos(),
modelUtils.getClusterControllerClientFactory(), ModelTestUtils.NUMBER_OF_CONFIG_SERVERS);
assertEquals("{ clusterId=cluster, serviceType=service-type }", clusterApi.clusterInfo());
@@ -184,7 +185,7 @@ public class ClusterApiImplTest {
applicationApi,
serviceCluster,
new NodeGroup(modelUtils.createApplicationInstance(new ArrayList<>()), groupNodes),
- modelUtils.getHostStatusMap(),
+ modelUtils.getHostInfos(),
modelUtils.getClusterControllerClientFactory(), ModelTestUtils.NUMBER_OF_CONFIG_SERVERS);
assertEquals(expectedNoServicesInGroupIsUp, clusterApi.noServicesInGroupIsUp());
@@ -214,7 +215,7 @@ public class ClusterApiImplTest {
applicationApi,
serviceCluster,
new NodeGroup(applicationInstance, hostName1, hostName3),
- new HashMap<>(),
+ new HostInfos(),
modelUtils.getClusterControllerClientFactory(), ModelTestUtils.NUMBER_OF_CONFIG_SERVERS);
assertTrue(clusterApi.isStorageCluster());
@@ -254,7 +255,7 @@ public class ClusterApiImplTest {
applicationApi,
serviceCluster,
new NodeGroup(application, hostnames.get(0)),
- modelUtils.getHostStatusMap(),
+ modelUtils.getHostInfos(),
modelUtils.getClusterControllerClientFactory(), clusterSize);
assertEquals(clusterSize - serviceStatusList.size(), clusterApi.missingServices());
diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/model/ModelTestUtils.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/model/ModelTestUtils.java
index 729b3ae79ff..87e5f226c42 100644
--- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/model/ModelTestUtils.java
+++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/model/ModelTestUtils.java
@@ -25,6 +25,8 @@ import com.yahoo.vespa.orchestrator.controller.ClusterControllerClientFactory;
import com.yahoo.vespa.orchestrator.controller.ClusterControllerClientFactoryMock;
import com.yahoo.vespa.orchestrator.policy.HostedVespaClusterPolicy;
import com.yahoo.vespa.orchestrator.policy.HostedVespaPolicy;
+import com.yahoo.vespa.orchestrator.status.HostInfo;
+import com.yahoo.vespa.orchestrator.status.HostInfos;
import com.yahoo.vespa.orchestrator.status.HostStatus;
import com.yahoo.vespa.orchestrator.status.MutableStatusRegistry;
import com.yahoo.vespa.orchestrator.status.StatusService;
@@ -33,11 +35,13 @@ import com.yahoo.vespa.service.monitor.ServiceModel;
import com.yahoo.yolean.Exceptions;
import java.time.Clock;
+import java.time.Instant;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
+import java.util.stream.Collectors;
import static org.mockito.Mockito.mock;
@@ -64,8 +68,23 @@ class ModelTestUtils {
return new ApplicationApiFactory(NUMBER_OF_CONFIG_SERVERS);
}
- Map<HostName, HostStatus> getHostStatusMap() {
- return hostStatusMap;
+ HostInfos getHostInfos() {
+ Instant now = Instant.now();
+
+ Map<HostName, HostInfo> hostInfosMap = hostStatusMap.entrySet().stream()
+ .collect(Collectors.toMap(
+ entry -> entry.getKey(),
+ entry -> {
+ HostStatus status = entry.getValue();
+ if (status == HostStatus.NO_REMARKS) {
+ return HostInfo.createNoRemarks();
+ } else {
+ return HostInfo.createSuspended(status, now);
+ }
+ }
+ ));
+
+ return new HostInfos(hostInfosMap);
}
HostName createNode(String name, HostStatus hostStatus) {
diff --git a/searchcore/src/tests/proton/common/selectpruner_test.cpp b/searchcore/src/tests/proton/common/selectpruner_test.cpp
index 5b1fa3ed4bf..744159962ad 100644
--- a/searchcore/src/tests/proton/common/selectpruner_test.cpp
+++ b/searchcore/src/tests/proton/common/selectpruner_test.cpp
@@ -75,7 +75,9 @@ makeDocTypeRepo()
addField("aaa", Array(DataType::T_INT)).
addField("aaw", Wset(DataType::T_INT)).
addField("ab", DataType::T_INT).
- addField("ae", DataType::T_INT));
+ addField("ae", DataType::T_INT)).
+ imported_field("my_imported_field").
+ imported_field("my_missing_imported_field");
builder.document(doc_type_id + 1, type_name_2,
Struct(header_name_2), Struct(body_name_2).
addField("ic", DataType::T_STRING).
@@ -150,6 +152,9 @@ TestFixture::TestFixture()
_amgr.addAttribute("aaa", AttributeFactory::createAttribute("aaa", { BasicType::INT32 , CollectionType::ARRAY}));
_amgr.addAttribute("aaw", AttributeFactory::createAttribute("aaw", { BasicType::INT32 , CollectionType::WSET}));
_amgr.addAttribute("ae", AttributeFactory::createAttribute("ae", { BasicType::INT32 }));
+ // We "fake" having an imported attribute to avoid having to set up reference attributes, mappings etc.
+ // This is fine since the attribute manager already abstracts away if an attribute is imported or not.
+ _amgr.addAttribute("my_imported_field", AttributeFactory::createAttribute("my_imported_field", { BasicType::INT32 }));
_repoUP = makeDocTypeRepo();
}
@@ -170,9 +175,9 @@ TestFixture::testParse(const string &selection)
select = parser.parse(selection);
} catch (document::select::ParsingFailedException &e) {
LOG(info, "Parse failed: %s", e.what());
- select.reset(0);
+ select.reset();
}
- ASSERT_TRUE(select.get() != NULL);
+ ASSERT_TRUE(select.get() != nullptr);
}
@@ -189,9 +194,9 @@ TestFixture::testParseFail(const string &selection)
select = parser.parse(selection);
} catch (document::select::ParsingFailedException &e) {
LOG(info, "Parse failed: %s", e.getMessage().c_str());
- select.reset(0);
+ select.reset();
}
- ASSERT_TRUE(select.get() == NULL);
+ ASSERT_TRUE(select.get() == nullptr);
}
@@ -208,15 +213,15 @@ TestFixture::testPrune(const string &selection, const string &exp, const string
select = parser.parse(selection);
} catch (document::select::ParsingFailedException &e) {
LOG(info, "Parse failed: %s", e.what());
- select.reset(0);
+ select.reset();
}
- ASSERT_TRUE(select.get() != NULL);
+ ASSERT_TRUE(select.get() != nullptr);
std::ostringstream os;
select->print(os, true, "");
LOG(info, "ParseTree: '%s'", os.str().c_str());
const DocumentType *docType = repo.getDocumentType(docTypeName);
- ASSERT_TRUE(docType != NULL);
- Document::UP emptyDoc(new Document(*docType, document::DocumentId("id:ns:" + docTypeName + "::1")));
+ ASSERT_TRUE(docType != nullptr);
+ auto emptyDoc = std::make_unique<Document>(*docType, document::DocumentId("id:ns:" + docTypeName + "::1"));
emptyDoc->setRepo(repo);
SelectPruner pruner(docTypeName, &_amgr, *emptyDoc, repo, _hasFields, _hasDocuments);
pruner.process(*select);
@@ -788,6 +793,29 @@ TEST_F("Test that field values are invalid when disabling document access", Test
"test.aa == 4 and test.ae == 5 and invalid");
}
+TEST_F("Imported fields with matching attribute names are supported", TestFixture)
+{
+ f.testPrune("test.my_imported_field > 0",
+ "test.my_imported_field > 0");
+}
+
+// Edge case: document type reconfigured but attribute not yet visible in Proton
+TEST_F("Imported fields without matching attribute are mapped to constant NullValue", TestFixture)
+{
+ f.testPrune("test.my_missing_imported_field != test.aa", "null != test.aa");
+ // Simplified to -> "null != null" -> "false"
+ f.testPrune("test.my_missing_imported_field != null", "false");
+ // Simplified to -> "null > 0" -> "invalid", as null is not well-defined
+ // for operators other than (in-)equality.
+ f.testPrune("test.my_missing_imported_field > 0", "invalid");
+}
+
+TEST_F("Complex imported field references return Invalid", TestFixture)
+{
+ f.testPrune("test.my_imported_field.foo", "invalid");
+ f.testPrune("test.my_imported_field[123]", "invalid");
+ f.testPrune("test.my_imported_field{foo}", "invalid");
+}
} // namespace
diff --git a/searchcore/src/vespa/searchcore/proton/common/attributefieldvaluenode.cpp b/searchcore/src/vespa/searchcore/proton/common/attributefieldvaluenode.cpp
index 3a15bb16409..c516bbfc06c 100644
--- a/searchcore/src/vespa/searchcore/proton/common/attributefieldvaluenode.cpp
+++ b/searchcore/src/vespa/searchcore/proton/common/attributefieldvaluenode.cpp
@@ -46,7 +46,7 @@ getValue(const Context &context) const
const auto &sc(static_cast<const SelectContext &>(context));
uint32_t docId(sc._docId);
assert(docId != 0u);
- const auto& v = *sc.read_guard_at_index(_attr_guard_index).attribute();
+ const auto& v = sc.guarded_attribute_at_index(_attr_guard_index);
if (v.isUndefined(docId)) {
return std::make_unique<NullValue>();
}
diff --git a/searchcore/src/vespa/searchcore/proton/common/selectcontext.cpp b/searchcore/src/vespa/searchcore/proton/common/selectcontext.cpp
index ae6a1f58bdc..ee713e19385 100644
--- a/searchcore/src/vespa/searchcore/proton/common/selectcontext.cpp
+++ b/searchcore/src/vespa/searchcore/proton/common/selectcontext.cpp
@@ -46,12 +46,12 @@ SelectContext::dropAttributeGuards()
_guards->clear();
}
-const search::attribute::AttributeReadGuard&
-SelectContext::read_guard_at_index(uint32_t index) const noexcept
+const search::attribute::IAttributeVector&
+SelectContext::guarded_attribute_at_index(uint32_t index) const noexcept
{
assert(index < _guards->size());
assert((*_guards)[index].get() != nullptr);
- return *((*_guards)[index]);
+ return *((*_guards)[index])->attribute();
}
}
diff --git a/searchcore/src/vespa/searchcore/proton/common/selectcontext.h b/searchcore/src/vespa/searchcore/proton/common/selectcontext.h
index 84e51bc611b..4dd9f873088 100644
--- a/searchcore/src/vespa/searchcore/proton/common/selectcontext.h
+++ b/searchcore/src/vespa/searchcore/proton/common/selectcontext.h
@@ -3,7 +3,7 @@
#include <vespa/document/select/context.h>
-namespace search::attribute { class AttributeReadGuard; }
+namespace search::attribute { class IAttributeVector; }
namespace proton {
@@ -22,7 +22,7 @@ public:
uint32_t _docId;
- const search::attribute::AttributeReadGuard& read_guard_at_index(uint32_t index) const noexcept;
+ const search::attribute::IAttributeVector& guarded_attribute_at_index(uint32_t index) const noexcept;
private:
std::unique_ptr<select::Guards> _guards;
const CachedSelect &_cachedSelect;
diff --git a/searchcore/src/vespa/searchcore/proton/common/selectpruner.cpp b/searchcore/src/vespa/searchcore/proton/common/selectpruner.cpp
index 4ed404333b5..2c237074907 100644
--- a/searchcore/src/vespa/searchcore/proton/common/selectpruner.cpp
+++ b/searchcore/src/vespa/searchcore/proton/common/selectpruner.cpp
@@ -98,10 +98,7 @@ SelectPruner::SelectPruner(const SelectPruner *rhs)
}
-SelectPruner::~SelectPruner()
-{
-}
-
+SelectPruner::~SelectPruner() = default;
void
SelectPruner::visitAndBranch(const And &expr)
@@ -406,32 +403,33 @@ SelectPruner::visitFieldValueNode(const FieldValueNode &expr)
const document::DocumentType *docType = _repo.getDocumentType(_docType);
bool complex = false; // Cannot handle attribute if complex expression
vespalib::string name = SelectUtils::extractFieldName(expr, complex);
- try {
- std::unique_ptr<Field> fp(new Field(docType->getField(name)));
- if (!fp) {
+ const bool is_imported = docType->has_imported_field_name(name);
+ if (complex || !is_imported) {
+ try {
+ std::unique_ptr<Field> fp(new Field(docType->getField(name)));
+ if (!fp) {
+ setInvalidVal();
+ return;
+ }
+ } catch (FieldNotFoundException &) {
+ setInvalidVal();
+ return;
+ }
+ try {
+ FieldPath path;
+ docType->buildFieldPath(path, expr.getFieldName());
+ } catch (vespalib::IllegalArgumentException &) {
+ setInvalidVal();
+ return;
+ } catch (FieldNotFoundException &) {
setInvalidVal();
return;
}
- } catch (FieldNotFoundException &) {
- setInvalidVal();
- return;
- }
- try {
- FieldPath path;
- docType->buildFieldPath(path, expr.getFieldName());
- } catch (vespalib::IllegalArgumentException &) {
- setInvalidVal();
- return;
- } catch (FieldNotFoundException &) {
- setInvalidVal();
- return;
}
_constVal = false;
if (!_hasFields) {
// If we're working on removed document sub db then we have no fields.
- _constVal = true;
- _valueNode.reset(new NullValueNode());
- _priority = NullValPriority;
+ set_null_value_node();
return;
}
@@ -447,6 +445,11 @@ SelectPruner::visitFieldValueNode(const FieldValueNode &expr)
if ((ag->attribute()->getCollectionType() == CollectionType::SINGLE) && !complex) {
svAttr = true;
}
+ } else if (is_imported) {
+ // Imported field present in document config but not yet in attribute config.
+ // Treat as missing (null) in document, as this matches behavior elsewhere in the pipeline.
+ set_null_value_node();
+ return;
}
}
if (!_hasDocuments && !svAttr) {
@@ -538,6 +541,13 @@ SelectPruner::setInvalidConst()
_node.reset(new InvalidConstant("invalid"));
}
+void
+SelectPruner::set_null_value_node()
+{
+ _constVal = true;
+ _valueNode = std::make_unique<NullValueNode>();
+ _priority = NullValPriority;
+}
void
SelectPruner::setTernaryConst(bool val)
diff --git a/searchcore/src/vespa/searchcore/proton/common/selectpruner.h b/searchcore/src/vespa/searchcore/proton/common/selectpruner.h
index 4350247d8b3..2c3729006c3 100644
--- a/searchcore/src/vespa/searchcore/proton/common/selectpruner.h
+++ b/searchcore/src/vespa/searchcore/proton/common/selectpruner.h
@@ -81,6 +81,7 @@ private:
void setInvalidVal();
void setInvalidConst();
void setTernaryConst(bool val);
+ void set_null_value_node();
void resolveTernaryConst(bool wantInverted);
bool isInvalidVal() const;
bool isNullVal() const;
diff --git a/searchlib/CMakeLists.txt b/searchlib/CMakeLists.txt
index 2c8eff4f4ad..d754fd78394 100644
--- a/searchlib/CMakeLists.txt
+++ b/searchlib/CMakeLists.txt
@@ -210,6 +210,7 @@ vespa_define_module(
src/tests/sortspec
src/tests/stringenum
src/tests/tensor/dense_tensor_store
+ src/tests/tensor/hnsw_index
src/tests/transactionlog
src/tests/transactionlogstress
src/tests/true
diff --git a/searchlib/src/tests/attribute/imported_attribute_vector/imported_attribute_vector_test.cpp b/searchlib/src/tests/attribute/imported_attribute_vector/imported_attribute_vector_test.cpp
index cfa05b7a765..48a06bec9e8 100644
--- a/searchlib/src/tests/attribute/imported_attribute_vector/imported_attribute_vector_test.cpp
+++ b/searchlib/src/tests/attribute/imported_attribute_vector/imported_attribute_vector_test.cpp
@@ -219,6 +219,15 @@ TEST_F("Weighted floating point attribute values can be retrieved via reference"
assert_multi_value_matches<WeightedFloat>(f, DocId(3), doc7_values);
}
+TEST_F("isUndefined() works for primitive attribute type", Fixture) {
+ reset_with_single_value_reference_mappings<IntegerAttribute, int32_t>(
+ f, BasicType::INT32,
+ {{DocId(3), dummy_gid(7), DocId(7), 5678}});
+
+ EXPECT_FALSE(f.get_imported_attr()->isUndefined(DocId(3))); // Mapped
+ EXPECT_TRUE(f.get_imported_attr()->isUndefined(DocId(2))); // Not mapped
+}
+
struct SingleStringAttrFixture : Fixture {
SingleStringAttrFixture() : Fixture() {
setup();
@@ -255,6 +264,11 @@ TEST_F("findEnum() returns target vector enum via reference", SingleStringAttrFi
EXPECT_EQUAL(expected_handle, actual_handle);
}
+TEST_F("isUndefined() works for enumerated attribute type", SingleStringAttrFixture) {
+ EXPECT_FALSE(f.get_imported_attr()->isUndefined(DocId(2))); // Mapped
+ EXPECT_TRUE(f.get_imported_attr()->isUndefined(DocId(3))); // Not mapped
+}
+
// Note: assumes that fixture has set up a string enum of value "foo" in target attribute
template <typename FixtureType>
void verify_get_string_from_enum_is_mapped(FixtureType& f) {
diff --git a/searchlib/src/tests/tensor/hnsw_index/CMakeLists.txt b/searchlib/src/tests/tensor/hnsw_index/CMakeLists.txt
new file mode 100644
index 00000000000..04c7312a63f
--- /dev/null
+++ b/searchlib/src/tests/tensor/hnsw_index/CMakeLists.txt
@@ -0,0 +1,9 @@
+# Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_executable(searchlib_hnsw_index_test_app TEST
+ SOURCES
+ hnsw_index_test.cpp
+ DEPENDS
+ searchlib
+ gtest
+)
+vespa_add_test(NAME searchlib_hnsw_index_test_app COMMAND searchlib_hnsw_index_test_app)
diff --git a/searchlib/src/tests/tensor/hnsw_index/hnsw_index_test.cpp b/searchlib/src/tests/tensor/hnsw_index/hnsw_index_test.cpp
new file mode 100644
index 00000000000..ffa49096ad4
--- /dev/null
+++ b/searchlib/src/tests/tensor/hnsw_index/hnsw_index_test.cpp
@@ -0,0 +1,94 @@
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/eval/tensor/dense/typed_cells.h>
+#include <vespa/searchlib/tensor/doc_vector_access.h>
+#include <vespa/searchlib/tensor/hnsw_index.h>
+#include <vespa/vespalib/gtest/gtest.h>
+#include <vector>
+
+#include <vespa/log/log.h>
+LOG_SETUP("hnsw_index_test");
+
+using namespace search::tensor;
+
+template <typename FloatType>
+class MyDocVectorAccess : public DocVectorAccess {
+private:
+ using Vector = std::vector<FloatType>;
+ using ArrayRef = vespalib::ConstArrayRef<FloatType>;
+ std::vector<Vector> _vectors;
+
+public:
+ MyDocVectorAccess() : _vectors() {}
+ MyDocVectorAccess& set(uint32_t docid, const Vector& vec) {
+ if (docid >= _vectors.size()) {
+ _vectors.resize(docid + 1);
+ }
+ _vectors[docid] = vec;
+ return *this;
+ }
+ vespalib::tensor::TypedCells get_vector(uint32_t docid) const override {
+ ArrayRef ref(_vectors[docid]);
+ return vespalib::tensor::TypedCells(ref);
+ }
+};
+
+class HnswIndexTest : public ::testing::Test {
+public:
+ MyDocVectorAccess<float> vectors;
+ HnswIndex<float> index;
+
+ HnswIndexTest()
+ : vectors(),
+ index(vectors, HnswIndexBase::Config(2, 0, 4))
+ {
+ }
+ void expect_level_0(uint32_t docid, const HnswNode::LinkArray& exp_links) {
+ auto node = index.get_node(docid);
+ ASSERT_EQ(1, node.size());
+ EXPECT_EQ(exp_links, node.level(0));
+ }
+};
+
+
+TEST_F(HnswIndexTest, 2d_vectors_inserted_in_level_0_graph_with_simple_select_neighbors)
+{
+ vectors.set(1, {2, 2}).set(2, {3, 2}).set(3, {2, 3})
+ .set(4, {1, 2}).set(5, {5, 3}).set(6, {6, 2});
+
+ index.add_document(1);
+ expect_level_0(1, {});
+
+ index.add_document(2);
+ expect_level_0(1, {2});
+ expect_level_0(2, {1});
+
+ index.add_document(3);
+ expect_level_0(1, {2, 3});
+ expect_level_0(2, {1, 3});
+ expect_level_0(3, {1, 2});
+
+ index.add_document(4);
+ expect_level_0(1, {2, 3, 4});
+ expect_level_0(2, {1, 3});
+ expect_level_0(3, {1, 2, 4});
+ expect_level_0(4, {1, 3});
+
+ index.add_document(5);
+ expect_level_0(1, {2, 3, 4});
+ expect_level_0(2, {1, 3, 5});
+ expect_level_0(3, {1, 2, 4, 5});
+ expect_level_0(4, {1, 3});
+ expect_level_0(5, {2, 3});
+
+ index.add_document(6);
+ expect_level_0(1, {2, 3, 4});
+ expect_level_0(2, {1, 3, 5, 6});
+ expect_level_0(3, {1, 2, 4, 5});
+ expect_level_0(4, {1, 3});
+ expect_level_0(5, {2, 3, 6});
+ expect_level_0(6, {2, 5});
+}
+
+GTEST_MAIN_RUN_ALL_TESTS()
+
diff --git a/searchlib/src/vespa/searchlib/attribute/imported_attribute_vector_read_guard.cpp b/searchlib/src/vespa/searchlib/attribute/imported_attribute_vector_read_guard.cpp
index b097b27ea53..f2949119ae2 100644
--- a/searchlib/src/vespa/searchlib/attribute/imported_attribute_vector_read_guard.cpp
+++ b/searchlib/src/vespa/searchlib/attribute/imported_attribute_vector_read_guard.cpp
@@ -152,6 +152,10 @@ bool ImportedAttributeVectorReadGuard::isImported() const
return true;
}
+bool ImportedAttributeVectorReadGuard::isUndefined(DocId doc) const {
+ return _target_attribute.isUndefined(getTargetLid(doc));
+}
+
long ImportedAttributeVectorReadGuard::onSerializeForAscendingSort(DocId doc,
void *serTo,
long available,
diff --git a/searchlib/src/vespa/searchlib/attribute/imported_attribute_vector_read_guard.h b/searchlib/src/vespa/searchlib/attribute/imported_attribute_vector_read_guard.h
index f130a095006..b31b11b8c74 100644
--- a/searchlib/src/vespa/searchlib/attribute/imported_attribute_vector_read_guard.h
+++ b/searchlib/src/vespa/searchlib/attribute/imported_attribute_vector_read_guard.h
@@ -81,6 +81,7 @@ public:
bool getIsFastSearch() const override;
uint32_t getCommittedDocIdLimit() const override;
bool isImported() const override;
+ bool isUndefined(DocId doc) const override;
protected:
long onSerializeForAscendingSort(DocId doc, void * serTo, long available,
diff --git a/searchlib/src/vespa/searchlib/docstore/chunk.cpp b/searchlib/src/vespa/searchlib/docstore/chunk.cpp
index c7976fd15ae..847cd3623c3 100644
--- a/searchlib/src/vespa/searchlib/docstore/chunk.cpp
+++ b/searchlib/src/vespa/searchlib/docstore/chunk.cpp
@@ -79,7 +79,7 @@ Chunk::Chunk(uint32_t id, const void * buffer, size_t len, bool skipcrc) :
os >> _lastSerial;
}
-Chunk::~Chunk() { }
+Chunk::~Chunk() = default;
vespalib::ConstBufferRef
Chunk::getLid(uint32_t lid) const
diff --git a/searchlib/src/vespa/searchlib/docstore/writeablefilechunk.cpp b/searchlib/src/vespa/searchlib/docstore/writeablefilechunk.cpp
index 70295d81a93..d12a3a54a93 100644
--- a/searchlib/src/vespa/searchlib/docstore/writeablefilechunk.cpp
+++ b/searchlib/src/vespa/searchlib/docstore/writeablefilechunk.cpp
@@ -203,6 +203,13 @@ seek_past(LidInfoWithLidV::const_iterator begin, LidInfoWithLidV::const_iterator
return begin;
}
+struct LidAndBuffer {
+ LidAndBuffer(uint32_t lid, uint32_t sz, vespalib::alloc::Alloc buf) : _lid(lid), _size(sz), _buf(std::move(buf)) {}
+ uint32_t _lid;
+ uint32_t _size;
+ vespalib::alloc::Alloc _buf;
+};
+
}
void
@@ -211,6 +218,7 @@ WriteableFileChunk::read(LidInfoWithLidV::const_iterator begin, size_t count, IB
if (count == 0) { return; }
if (!frozen()) {
vespalib::hash_map<uint32_t, ChunkInfo> chunksOnFile;
+ std::vector<LidAndBuffer> buffers;
{
LockGuard guard(_lock);
for (size_t i(0); i < count; i++) {
@@ -225,12 +233,18 @@ WriteableFileChunk::read(LidInfoWithLidV::const_iterator begin, size_t count, IB
assert(chunk == _active->getId());
buffer = _active->getLid(li.getLid());
}
- visitor.visit(li.getLid(), buffer);
+ auto copy = vespalib::alloc::Alloc::alloc(buffer.size());
+ memcpy(copy.get(), buffer.data(), buffer.size());
+ buffers.emplace_back(li.getLid(), buffer.size(), std::move(copy));
} else {
chunksOnFile[chunk] = _chunkInfo[chunk];
}
}
}
+ for (auto & entry : buffers) {
+ visitor.visit(entry._lid, vespalib::ConstBufferRef(entry._buf.get(), entry._size));
+ entry._buf = vespalib::alloc::Alloc();
+ }
for (auto & it : chunksOnFile) {
auto first = find_first(begin, it.first);
auto last = seek_past(first, begin + count, it.first);
diff --git a/searchlib/src/vespa/searchlib/tensor/CMakeLists.txt b/searchlib/src/vespa/searchlib/tensor/CMakeLists.txt
index ea5d30d9a47..7c05304d451 100644
--- a/searchlib/src/vespa/searchlib/tensor/CMakeLists.txt
+++ b/searchlib/src/vespa/searchlib/tensor/CMakeLists.txt
@@ -6,6 +6,8 @@ vespa_add_library(searchlib_tensor OBJECT
dense_tensor_store.cpp
generic_tensor_attribute.cpp
generic_tensor_store.cpp
+ hnsw_index_base.cpp
+ hnsw_index.cpp
imported_tensor_attribute_vector.cpp
imported_tensor_attribute_vector_read_guard.cpp
tensor_attribute.cpp
diff --git a/searchlib/src/vespa/searchlib/tensor/doc_vector_access.h b/searchlib/src/vespa/searchlib/tensor/doc_vector_access.h
new file mode 100644
index 00000000000..e5dfa35a529
--- /dev/null
+++ b/searchlib/src/vespa/searchlib/tensor/doc_vector_access.h
@@ -0,0 +1,21 @@
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <vespa/eval/tensor/dense/typed_cells.h>
+#include <cstdint>
+
+namespace search::tensor {
+
+/**
+ * Interface that provides access to the vector that is associated with the the given document id.
+ *
+ * All vectors should be the same size and either of type float or double.
+ */
+class DocVectorAccess {
+public:
+ virtual ~DocVectorAccess() {}
+ virtual vespalib::tensor::TypedCells get_vector(uint32_t docid) const = 0;
+};
+
+}
diff --git a/searchlib/src/vespa/searchlib/tensor/hnsw_index.cpp b/searchlib/src/vespa/searchlib/tensor/hnsw_index.cpp
new file mode 100644
index 00000000000..bd46854cf17
--- /dev/null
+++ b/searchlib/src/vespa/searchlib/tensor/hnsw_index.cpp
@@ -0,0 +1,102 @@
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include "hnsw_index.h"
+
+namespace search::tensor {
+
+template <typename FloatType>
+double
+HnswIndex<FloatType>::calc_distance(const Vector& lhs, uint32_t rhs_docid) const
+{
+ // TODO: Make it possible to specify the distance function from the outside and make it hardware optimized.
+ auto rhs = get_vector(rhs_docid);
+ double result = 0.0;
+ size_t sz = lhs.size();
+ assert(sz == rhs.size());
+ for (size_t i = 0; i < sz; ++i) {
+ double diff = lhs[i] - rhs[i];
+ result += diff * diff;
+ }
+ return result;
+}
+
+template <typename FloatType>
+void
+HnswIndex<FloatType>::search_layer(const Vector& input, uint32_t neighbors_to_find, FurthestPriQ& best_neighbors, uint32_t level)
+{
+ NearestPriQ candidates;
+ // TODO: Add proper handling of visited set.
+ auto visited = BitVector::create(_node_refs.size());
+ for (const auto &entry : best_neighbors.peek()) {
+ candidates.push(entry);
+ visited->setBit(entry.docid);
+ }
+ double limit_dist = std::numeric_limits<double>::max();
+
+ while (!candidates.empty()) {
+ auto cand = candidates.top();
+ if (cand.distance > limit_dist) {
+ break;
+ }
+ candidates.pop();
+ for (uint32_t neighbor_docid : get_link_array(cand.docid, level)) {
+ if (visited->testBit(neighbor_docid)) {
+ continue;
+ }
+ visited->setBit(neighbor_docid);
+ double dist_to_input = calc_distance(input, neighbor_docid);
+ if (dist_to_input < limit_dist) {
+ candidates.emplace(neighbor_docid, dist_to_input);
+ best_neighbors.emplace(neighbor_docid, dist_to_input);
+ if (best_neighbors.size() > neighbors_to_find) {
+ best_neighbors.pop();
+ limit_dist = best_neighbors.top().distance;
+ }
+ }
+ }
+ }
+}
+
+template <typename FloatType>
+HnswIndex<FloatType>::HnswIndex(const DocVectorAccess& vectors, const Config& cfg)
+ : HnswIndexBase(vectors, cfg)
+{
+}
+
+template <typename FloatType>
+HnswIndex<FloatType>::~HnswIndex() = default;
+
+template <typename FloatType>
+void
+HnswIndex<FloatType>::add_document(uint32_t docid)
+{
+ auto input = get_vector(docid);
+ _node_refs.ensure_size(docid + 1, EntryRef());
+ // A document cannot be added twice.
+ assert(!_node_refs[docid].valid());
+ make_node_for_document(docid);
+ if (_entry_docid == 0) {
+ _entry_docid = docid;
+ return;
+ }
+ double entry_dist = calc_distance(input, _entry_docid);
+ FurthestPriQ best_neighbors;
+ best_neighbors.emplace(_entry_docid, entry_dist);
+ // TODO: Add support for multiple levels.
+ // TODO: Rename to search_level?
+ search_layer(input, _cfg.neighbors_to_explore_at_construction(), best_neighbors, 0);
+ auto neighbors = select_neighbors_simple(best_neighbors.peek(), _cfg.max_links_at_level_0());
+ connect_new_node(docid, neighbors, 0);
+ // TODO: Shrink neighbors if needed
+}
+
+template <typename FloatType>
+void
+HnswIndex<FloatType>::remove_document(uint32_t docid)
+{
+ (void) docid;
+ // TODO: implement
+}
+
+}
+
diff --git a/searchlib/src/vespa/searchlib/tensor/hnsw_index.h b/searchlib/src/vespa/searchlib/tensor/hnsw_index.h
new file mode 100644
index 00000000000..b1f35af629c
--- /dev/null
+++ b/searchlib/src/vespa/searchlib/tensor/hnsw_index.h
@@ -0,0 +1,45 @@
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include "hnsw_index_base.h"
+#include <vespa/searchlib/common/bitvector.h>
+#include <vespa/vespalib/datastore/array_store.h>
+#include <vespa/vespalib/datastore/entryref.h>
+#include <vespa/vespalib/util/rcuvector.h>
+
+namespace search::tensor {
+
+/**
+ * Concrete implementation of a hierarchical navigable small world graph (HNSW)
+ * that is used for approximate K-nearest neighbor search.
+ *
+ * See HnswIndexBase for more details.
+ *
+ * The FloatType template argument specifies the data type used in the vectors (4 byte float or 8 byte double).
+ */
+template <typename FloatType = float>
+class HnswIndex : public HnswIndexBase {
+private:
+ using Vector = vespalib::ConstArrayRef<FloatType>;
+
+ inline Vector get_vector(uint32_t docid) const {
+ return _vectors.get_vector(docid).template typify<FloatType>();
+ }
+
+ double calc_distance(const Vector& lhs, uint32_t rhs_docid) const;
+ void search_layer(const Vector& input, uint32_t neighbors_to_find, FurthestPriQ& found_neighbors, uint32_t level);
+
+public:
+ HnswIndex(const DocVectorAccess& vectors, const Config& cfg);
+ ~HnswIndex() override;
+
+ void add_document(uint32_t docid) override;
+ void remove_document(uint32_t docid) override;
+};
+
+template class HnswIndex<float>;
+template class HnswIndex<double>;
+
+}
+
diff --git a/searchlib/src/vespa/searchlib/tensor/hnsw_index_base.cpp b/searchlib/src/vespa/searchlib/tensor/hnsw_index_base.cpp
new file mode 100644
index 00000000000..a1132bea4e7
--- /dev/null
+++ b/searchlib/src/vespa/searchlib/tensor/hnsw_index_base.cpp
@@ -0,0 +1,135 @@
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include "hnsw_index_base.h"
+#include <vespa/vespalib/datastore/array_store.hpp>
+
+namespace search::tensor {
+
+namespace {
+
+// TODO: Move this to MemoryAllocator, with name PAGE_SIZE.
+constexpr size_t small_page_size = 4 * 1024;
+constexpr size_t min_num_arrays_for_new_buffer = 8 * 1024;
+constexpr float alloc_grow_factor = 0.2;
+// TODO: Adjust these numbers to what we accept as max in config.
+constexpr size_t max_level_array_size = 16;
+constexpr size_t max_link_array_size = 64;
+
+}
+
+search::datastore::ArrayStoreConfig
+HnswIndexBase::make_default_node_store_config()
+{
+ return NodeStore::optimizedConfigForHugePage(max_level_array_size, vespalib::alloc::MemoryAllocator::HUGEPAGE_SIZE,
+ small_page_size, min_num_arrays_for_new_buffer, alloc_grow_factor).enable_free_lists(true);
+}
+
+search::datastore::ArrayStoreConfig
+HnswIndexBase::make_default_link_store_config()
+{
+ return LinkStore::optimizedConfigForHugePage(max_link_array_size, vespalib::alloc::MemoryAllocator::HUGEPAGE_SIZE,
+ small_page_size, min_num_arrays_for_new_buffer, alloc_grow_factor).enable_free_lists(true);
+}
+
+void
+HnswIndexBase::make_node_for_document(uint32_t docid)
+{
+ // TODO: Draw this number from a random generator that is provided from the outside.
+ uint32_t num_levels = 1;
+ // Note: The level array instance lives as long as the document is present in the index.
+ LevelArray levels(num_levels, EntryRef());
+ auto node_ref = _nodes.add(levels);
+ // TODO: Add memory barrier?
+ _node_refs[docid] = node_ref;
+}
+
+HnswIndexBase::LevelArrayRef
+HnswIndexBase::get_level_array(uint32_t docid) const
+{
+ // TODO: Add memory barrier?
+ auto node_ref = _node_refs[docid];
+ return _nodes.get(node_ref);
+}
+
+HnswIndexBase::LinkArrayRef
+HnswIndexBase::get_link_array(uint32_t docid, uint32_t level) const
+{
+ auto levels = get_level_array(docid);
+ assert(level < levels.size());
+ return _links.get(levels[level]);
+}
+
+void
+HnswIndexBase::set_link_array(uint32_t docid, uint32_t level, const LinkArrayRef& links)
+{
+ auto links_ref = _links.add(links);
+ auto levels = get_level_array(docid);
+ // TODO: Add function to ArrayStore that returns mutable array ref, eg. get_writable()
+ auto mutable_levels = vespalib::unconstify(levels);
+ // TODO: Make this change atomic.
+ mutable_levels[level] = links_ref;
+}
+
+HnswIndexBase::LinkArray
+HnswIndexBase::select_neighbors_simple(const HnswCandidateVector& neighbors, uint32_t max_links) const
+{
+ LinkArray result;
+ NearestPriQ nearest;
+ for (const auto& entry : neighbors) {
+ nearest.push(entry);
+ }
+ while (!nearest.empty()) {
+ HnswCandidate candidate = nearest.top();
+ nearest.pop();
+ result.push_back(candidate.docid);
+ if (result.size() == max_links) {
+ return result;
+ }
+ }
+ return result;
+}
+
+void
+HnswIndexBase::connect_new_node(uint32_t docid, const LinkArray& neighbors, uint32_t level)
+{
+ set_link_array(docid, level, neighbors);
+ for (uint32_t neighbor_docid : neighbors) {
+ auto old_links = get_link_array(neighbor_docid, level);
+ LinkArray new_links(old_links.begin(), old_links.end());
+ new_links.push_back(docid);
+ set_link_array(neighbor_docid, level, new_links);
+ }
+}
+
+HnswIndexBase::HnswIndexBase(const DocVectorAccess& vectors, const Config& cfg)
+ : _vectors(vectors),
+ _cfg(cfg),
+ _node_refs(),
+ _nodes(make_default_node_store_config()),
+ _links(make_default_link_store_config()),
+ _entry_docid(0)
+{
+}
+
+HnswIndexBase::~HnswIndexBase() = default;
+
+HnswNode
+HnswIndexBase::get_node(uint32_t docid) const
+{
+ auto node_ref = _node_refs[docid];
+ if (!node_ref.valid()) {
+ return HnswNode();
+ }
+ auto levels = _nodes.get(node_ref);
+ HnswNode::LevelArray result;
+ for (const auto& links_ref : levels) {
+ auto links = _links.get(links_ref);
+ HnswNode::LinkArray result_links(links.begin(), links.end());
+ std::sort(result_links.begin(), result_links.end());
+ result.push_back(result_links);
+ }
+ return HnswNode(result);
+}
+
+}
+
diff --git a/searchlib/src/vespa/searchlib/tensor/hnsw_index_base.h b/searchlib/src/vespa/searchlib/tensor/hnsw_index_base.h
new file mode 100644
index 00000000000..b40eef5016a
--- /dev/null
+++ b/searchlib/src/vespa/searchlib/tensor/hnsw_index_base.h
@@ -0,0 +1,105 @@
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include "doc_vector_access.h"
+#include "hnsw_index_utils.h"
+#include "hnsw_node.h"
+#include "nearest_neighbor_index.h"
+#include <vespa/searchlib/common/bitvector.h>
+#include <vespa/vespalib/datastore/array_store.h>
+#include <vespa/vespalib/datastore/entryref.h>
+#include <vespa/vespalib/util/rcuvector.h>
+
+namespace search::tensor {
+
+/**
+ * Base class for an implementation of a hierarchical navigable small world graph (HNSW)
+ * that is used for approximate K-nearest neighbor search.
+ *
+ * The implementation supports 1 write thread and multiple search threads without the use of mutexes.
+ * This is achieved by using data stores that use generation tracking and associated memory management.
+ *
+ * The implementation is mainly based on the algorithms described in
+ * "Efficient and robust approximate nearest neighbor search using Hierarchical Navigable Small World graphs" (Yu. A. Malkov, D. A. Yashunin),
+ * but some adjustments are made to support proper removes.
+ *
+ * TODO: Add details on how to handle removes.
+ */
+class HnswIndexBase : public NearestNeighborIndex {
+public:
+ class Config {
+ private:
+ uint32_t _max_links_at_level_0;
+ uint32_t _max_links_at_hierarchic_levels;
+ uint32_t _neighbors_to_explore_at_construction;
+
+ public:
+ Config(uint32_t max_links_at_level_0_in,
+ uint32_t max_links_at_hierarchic_levels_in,
+ uint32_t neighbors_to_explore_at_construction_in)
+ : _max_links_at_level_0(max_links_at_level_0_in),
+ _max_links_at_hierarchic_levels(max_links_at_hierarchic_levels_in),
+ _neighbors_to_explore_at_construction(neighbors_to_explore_at_construction_in)
+ {}
+ uint32_t max_links_at_level_0() const { return _max_links_at_level_0; }
+ uint32_t max_links_at_hierarchic_levels() const { return _max_links_at_hierarchic_levels; }
+ uint32_t neighbors_to_explore_at_construction() const { return _neighbors_to_explore_at_construction; }
+ };
+
+protected:
+ using EntryRef = search::datastore::EntryRef;
+
+ // This uses 10 bits for buffer id -> 1024 buffers.
+ // As we have very short arrays we get less fragmentation with fewer and larger buffers.
+ using EntryRefType = search::datastore::EntryRefT<22>;
+
+ // Provides mapping from document id -> node reference.
+ // The reference is used to lookup the node data in NodeStore.
+ using NodeRefVector = vespalib::RcuVector<EntryRef>;
+
+ // This stores the level arrays for all nodes.
+ // Each node consists of an array of levels (from level 0 to n) where each entry is a reference to the link array at that level.
+ // TODO: Make replacing all links on a level atomically, e.g. AtomicEntryRef
+ using NodeStore = search::datastore::ArrayStore<EntryRef, EntryRefType>;
+ using LevelArrayRef = NodeStore::ConstArrayRef;
+ using LevelArray = vespalib::Array<EntryRef>;
+
+ // This stores all link arrays.
+ // A link array consists of the document ids of the nodes a particular node is linked to.
+ using LinkStore = search::datastore::ArrayStore<uint32_t, EntryRefType>;
+ using LinkArrayRef = LinkStore::ConstArrayRef;
+ using LinkArray = vespalib::Array<uint32_t>;
+
+ const DocVectorAccess& _vectors;
+ Config _cfg;
+ NodeRefVector _node_refs;
+ NodeStore _nodes;
+ LinkStore _links;
+ uint32_t _entry_docid;
+
+ static search::datastore::ArrayStoreConfig make_default_node_store_config();
+ static search::datastore::ArrayStoreConfig make_default_link_store_config();
+
+ void make_node_for_document(uint32_t docid);
+ LevelArrayRef get_level_array(uint32_t docid) const;
+ LinkArrayRef get_link_array(uint32_t docid, uint32_t level) const;
+ void set_link_array(uint32_t docid, uint32_t level, const LinkArrayRef& links);
+
+ LinkArray select_neighbors_simple(const HnswCandidateVector& neighbors, uint32_t max_links) const;
+ void connect_new_node(uint32_t docid, const LinkArray& neighbors, uint32_t level);
+
+public:
+ HnswIndexBase(const DocVectorAccess& vectors, const Config& cfg);
+ ~HnswIndexBase() override;
+
+ // TODO: Add support for generation handling and cleanup (transfer_hold_lists, trim_hold_lists)
+
+ // Should only be used by unit tests.
+ HnswNode get_node(uint32_t docid) const;
+
+ // TODO: Implement set_node() as well for use in unit tests.
+};
+
+}
+
diff --git a/searchlib/src/vespa/searchlib/tensor/hnsw_index_utils.h b/searchlib/src/vespa/searchlib/tensor/hnsw_index_utils.h
new file mode 100644
index 00000000000..f7f56d3adb6
--- /dev/null
+++ b/searchlib/src/vespa/searchlib/tensor/hnsw_index_utils.h
@@ -0,0 +1,47 @@
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <queue>
+#include <vector>
+
+namespace search::tensor {
+
+/**
+ * Represents a candidate node with its distance to another point in space.
+ */
+struct HnswCandidate {
+ uint32_t docid;
+ double distance;
+ HnswCandidate(uint32_t docid_in, double distance_in) : docid(docid_in), distance(distance_in) {}
+};
+
+struct GreaterDistance {
+ bool operator() (const HnswCandidate& lhs, const HnswCandidate& rhs) const {
+ return (rhs.distance < lhs.distance);
+ }
+};
+
+struct LesserDistance {
+ bool operator() (const HnswCandidate& lhs, const HnswCandidate& rhs) const {
+ return (lhs.distance < rhs.distance);
+ }
+};
+
+using HnswCandidateVector = std::vector<HnswCandidate>;
+
+/**
+ * Priority queue that keeps the candidate node that is nearest a point in space on top.
+ */
+using NearestPriQ = std::priority_queue<HnswCandidate, HnswCandidateVector, GreaterDistance>;
+
+/**
+ * Priority queue that keeps the candidate node that is furthest away a point in space on top.
+ */
+class FurthestPriQ : public std::priority_queue<HnswCandidate, HnswCandidateVector, LesserDistance> {
+public:
+ const HnswCandidateVector& peek() const { return c; }
+};
+
+}
+
diff --git a/searchlib/src/vespa/searchlib/tensor/hnsw_node.h b/searchlib/src/vespa/searchlib/tensor/hnsw_node.h
new file mode 100644
index 00000000000..c7e719e2985
--- /dev/null
+++ b/searchlib/src/vespa/searchlib/tensor/hnsw_node.h
@@ -0,0 +1,34 @@
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <vector>
+
+namespace search::tensor {
+
+/**
+ * Represents a snapshot of a graph node with all its levels and links.
+ * Should only be used by unit tests.
+ */
+class HnswNode {
+public:
+ using LinkArray = std::vector<uint32_t>;
+ using LevelArray = std::vector<LinkArray>;
+
+private:
+ LevelArray _levels;
+
+public:
+ HnswNode() : _levels() {}
+ HnswNode(const LinkArray& level_0) : _levels() { _levels.push_back(level_0); }
+ HnswNode(const LevelArray& levels_in) : _levels(levels_in) {}
+ bool empty() const { return _levels.empty(); }
+ size_t size() const { return _levels.size(); }
+ const LinkArray& level(size_t idx) const { return _levels[idx]; }
+ bool operator==(const HnswNode& rhs) {
+ return _levels == rhs._levels;
+ }
+};
+
+}
+
diff --git a/searchlib/src/vespa/searchlib/tensor/nearest_neighbor_index.h b/searchlib/src/vespa/searchlib/tensor/nearest_neighbor_index.h
new file mode 100644
index 00000000000..2167157f6cb
--- /dev/null
+++ b/searchlib/src/vespa/searchlib/tensor/nearest_neighbor_index.h
@@ -0,0 +1,19 @@
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <cstdint>
+
+namespace search::tensor {
+
+/**
+ * Interface for an index that is used for (approximate) nearest neighbor search.
+ */
+class NearestNeighborIndex {
+public:
+ virtual ~NearestNeighborIndex() {}
+ virtual void add_document(uint32_t docid) = 0;
+ virtual void remove_document(uint32_t docid) = 0;
+};
+
+}
diff --git a/security-utils/src/main/java/com/yahoo/security/tls/TlsContext.java b/security-utils/src/main/java/com/yahoo/security/tls/TlsContext.java
index e12ea3cf47d..886cf3e886b 100644
--- a/security-utils/src/main/java/com/yahoo/security/tls/TlsContext.java
+++ b/security-utils/src/main/java/com/yahoo/security/tls/TlsContext.java
@@ -35,7 +35,7 @@ public interface TlsContext extends AutoCloseable {
"TLS_CHACHA20_POLY1305_SHA256"); // TLSv1.3, Java 12
Set<String> ALLOWED_PROTOCOLS = com.yahoo.vespa.jdk8compat.Set.of("TLSv1.2"); // TODO Enable TLSv1.3
- String SSL_CONTEXT_VERSION = "TLSv1.2"; // TODO Enable TLSv1.3
+ String SSL_CONTEXT_VERSION = "TLS"; // Use SSLContext implementations that supports all TLS versions
/**
* @return the allowed cipher suites supported by the provided context instance
diff --git a/storage/src/vespa/storage/storageserver/CMakeLists.txt b/storage/src/vespa/storage/storageserver/CMakeLists.txt
index 73873e78032..1aafe58af6c 100644
--- a/storage/src/vespa/storage/storageserver/CMakeLists.txt
+++ b/storage/src/vespa/storage/storageserver/CMakeLists.txt
@@ -7,11 +7,12 @@ vespa_add_library(storage_storageserver
changedbucketownershiphandler.cpp
communicationmanager.cpp
communicationmanagermetrics.cpp
- configurable_bucket_resolver.cpp
config_logging.cpp
+ configurable_bucket_resolver.cpp
distributornode.cpp
distributornodecontext.cpp
documentapiconverter.cpp
+ fnet_metrics_wrapper.cpp
fnetlistener.cpp
mergethrottler.cpp
messagesink.cpp
diff --git a/storage/src/vespa/storage/storageserver/fnet_metrics_wrapper.cpp b/storage/src/vespa/storage/storageserver/fnet_metrics_wrapper.cpp
new file mode 100644
index 00000000000..476ececd078
--- /dev/null
+++ b/storage/src/vespa/storage/storageserver/fnet_metrics_wrapper.cpp
@@ -0,0 +1,21 @@
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include "fnet_metrics_wrapper.h"
+
+namespace storage {
+
+FnetMetricsWrapper::FnetMetricsWrapper(metrics::MetricSet* owner)
+ : metrics::MetricSet("fnet", {}, "transport layer metrics", owner),
+ _num_connections("num-connections", {}, "total number of connection objects", this)
+{
+}
+
+FnetMetricsWrapper::~FnetMetricsWrapper() = default;
+
+void
+FnetMetricsWrapper::update_metrics()
+{
+ _num_connections.set(FNET_Connection::get_num_connections());
+}
+
+}
diff --git a/storage/src/vespa/storage/storageserver/fnet_metrics_wrapper.h b/storage/src/vespa/storage/storageserver/fnet_metrics_wrapper.h
new file mode 100644
index 00000000000..dacb9e9f52c
--- /dev/null
+++ b/storage/src/vespa/storage/storageserver/fnet_metrics_wrapper.h
@@ -0,0 +1,22 @@
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <vespa/metrics/metrics.h>
+#include <vespa/fnet/connection.h>
+
+namespace storage {
+
+// Simple wrapper around low-level fnet network metrics
+class FnetMetricsWrapper : public metrics::MetricSet
+{
+private:
+ metrics::LongValueMetric _num_connections;
+
+public:
+ explicit FnetMetricsWrapper(metrics::MetricSet* owner);
+ ~FnetMetricsWrapper() override;
+ void update_metrics();
+};
+
+}
diff --git a/storage/src/vespa/storage/storageserver/storagemetricsset.cpp b/storage/src/vespa/storage/storageserver/storagemetricsset.cpp
index f3240f0663b..6ed5c1ab650 100644
--- a/storage/src/vespa/storage/storageserver/storagemetricsset.cpp
+++ b/storage/src/vespa/storage/storageserver/storagemetricsset.cpp
@@ -23,7 +23,8 @@ StorageMetricSet::StorageMetricSet()
memoryUse_messages(this),
memoryUse_visiting("memoryusage_visiting", {{"memory"}},
"Message use from visiting", this),
- tls_metrics(this)
+ tls_metrics(this),
+ fnet_metrics(this)
{}
StorageMetricSet::~StorageMetricSet() = default;
@@ -35,6 +36,7 @@ void StorageMetricSet::updateMetrics() {
// be erased from history. This will no longer be a problem once we move to a
// metrics system built around absolute (rather than derived) values.
tls_metrics.update_metrics_with_snapshot_delta();
+ fnet_metrics.update_metrics();
}
} // storage
diff --git a/storage/src/vespa/storage/storageserver/storagemetricsset.h b/storage/src/vespa/storage/storageserver/storagemetricsset.h
index 49795c63324..a315e974c01 100644
--- a/storage/src/vespa/storage/storageserver/storagemetricsset.h
+++ b/storage/src/vespa/storage/storageserver/storagemetricsset.h
@@ -3,6 +3,7 @@
#pragma once
#include "tls_statistics_metrics_wrapper.h"
+#include "fnet_metrics_wrapper.h"
#include <vespa/metrics/metrics.h>
namespace storage {
@@ -27,6 +28,7 @@ struct StorageMetricSet : public metrics::MetricSet
metrics::LongValueMetric memoryUse_visiting;
TlsStatisticsMetricsWrapper tls_metrics;
+ FnetMetricsWrapper fnet_metrics;
StorageMetricSet();
~StorageMetricSet() override;
diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/api/AthenzAccessToken.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/api/AthenzAccessToken.java
index 86deb0b59b3..ec8c1f3f9f3 100644
--- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/api/AthenzAccessToken.java
+++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/api/AthenzAccessToken.java
@@ -22,9 +22,13 @@ public class AthenzAccessToken {
private static String stripBearerTokenPrefix(String rawValue) {
String stripped = rawValue.strip();
- return stripped.startsWith(BEARER_TOKEN_PREFIX)
- ? stripped.substring(BEARER_TOKEN_PREFIX.length())
+ String prefixRemoved = stripped.startsWith(BEARER_TOKEN_PREFIX)
+ ? stripped.substring(BEARER_TOKEN_PREFIX.length()).strip()
: stripped;
+ if (prefixRemoved.isBlank()) {
+ throw new IllegalArgumentException(String.format("Access token is blank: '%s'", prefixRemoved));
+ }
+ return prefixRemoved;
}
public String value() { return value; }
diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/api/AthenzRole.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/api/AthenzRole.java
index 3a81e4a5e17..4e432768298 100644
--- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/api/AthenzRole.java
+++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/api/AthenzRole.java
@@ -7,6 +7,8 @@ import java.util.Objects;
* @author tokle
*/
public class AthenzRole {
+ private static final String ROLE_RESOURCE_PREFIX = "role.";
+
private final AthenzDomain domain;
private final String roleName;
@@ -20,6 +22,19 @@ public class AthenzRole {
this.roleName = roleName;
}
+ public static AthenzRole fromResourceNameString(String string) {
+ return fromResourceName(AthenzResourceName.fromString(string));
+ }
+
+ public static AthenzRole fromResourceName(AthenzResourceName resourceName) {
+ String entityName = resourceName.getEntityName();
+ if (!entityName.startsWith(ROLE_RESOURCE_PREFIX)) {
+ throw new IllegalArgumentException("Not a valid role: " + resourceName.toResourceNameString());
+ }
+ String roleName = entityName.substring(ROLE_RESOURCE_PREFIX.length());
+ return new AthenzRole(resourceName.getDomain(), roleName);
+ }
+
public AthenzDomain domain() {
return domain;
}
@@ -28,6 +43,10 @@ public class AthenzRole {
return roleName;
}
+ public String toResourceNameString() { return toResourceName().toResourceNameString(); }
+
+ public AthenzResourceName toResourceName() { return new AthenzResourceName(domain, ROLE_RESOURCE_PREFIX + roleName); }
+
@Override
public boolean equals(Object o) {
if (this == o) return true;
diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/DefaultZtsClient.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/DefaultZtsClient.java
index eec578ddc26..c05213c8008 100644
--- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/DefaultZtsClient.java
+++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/DefaultZtsClient.java
@@ -2,14 +2,17 @@
package com.yahoo.vespa.athenz.client.zts;
import com.yahoo.security.Pkcs10Csr;
+import com.yahoo.vespa.athenz.api.AthenzAccessToken;
import com.yahoo.vespa.athenz.api.AthenzDomain;
import com.yahoo.vespa.athenz.api.AthenzIdentity;
+import com.yahoo.vespa.athenz.api.AthenzResourceName;
import com.yahoo.vespa.athenz.api.AthenzRole;
import com.yahoo.vespa.athenz.api.AwsRole;
import com.yahoo.vespa.athenz.api.AwsTemporaryCredentials;
import com.yahoo.vespa.athenz.api.NToken;
import com.yahoo.vespa.athenz.api.ZToken;
import com.yahoo.vespa.athenz.client.common.ClientBase;
+import com.yahoo.vespa.athenz.client.zts.bindings.AccessTokenResponseEntity;
import com.yahoo.vespa.athenz.client.zts.bindings.AwsTemporaryCredentialsResponseEntity;
import com.yahoo.vespa.athenz.client.zts.bindings.IdentityRefreshRequestEntity;
import com.yahoo.vespa.athenz.client.zts.bindings.IdentityResponseEntity;
@@ -36,6 +39,7 @@ import java.time.Duration;
import java.util.List;
import java.util.Optional;
import java.util.function.Supplier;
+import java.util.stream.Collectors;
import static java.util.stream.Collectors.toList;
@@ -147,6 +151,33 @@ public class DefaultZtsClient extends ClientBase implements ZtsClient {
}
@Override
+ public AthenzAccessToken getAccessToken(AthenzDomain domain) {
+ return this.getAccessTokenImpl(List.of(new AthenzResourceName(domain, "domain")));
+ }
+
+ @Override
+ public AthenzAccessToken getAccessToken(List<AthenzRole> athenzRole) {
+ List<AthenzResourceName> athenzResourceNames = athenzRole.stream()
+ .map(AthenzRole::toResourceName)
+ .collect(toList());
+ return this.getAccessTokenImpl(athenzResourceNames);
+ }
+
+ private AthenzAccessToken getAccessTokenImpl(List<AthenzResourceName> resources) {
+ URI uri = ztsUrl.resolve("oauth2/token");
+ RequestBuilder requestBuilder = RequestBuilder.post(uri)
+ .addHeader("Content-Type", "application/x-www-form-urlencoded")
+ .addParameter("grant_type", "client_credentials")
+ .addParameter("scope", resources.stream().map(AthenzResourceName::toResourceNameString).collect(Collectors.joining(" ")));
+
+ HttpUriRequest request = requestBuilder.build();
+ return execute(request, response -> {
+ AccessTokenResponseEntity accessTokenResponseEntity = readEntity(response, AccessTokenResponseEntity.class);
+ return accessTokenResponseEntity.accessToken();
+ });
+ }
+
+ @Override
public X509Certificate getRoleCertificate(AthenzRole role, Pkcs10Csr csr, Duration expiry) {
RoleCertificateRequestEntity requestEntity = new RoleCertificateRequestEntity(csr, expiry);
URI uri = ztsUrl.resolve(String.format("domain/%s/role/%s/token", role.domain().getName(), role.roleName()));
diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/ZtsClient.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/ZtsClient.java
index c09ad8f48a0..baeb4271905 100644
--- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/ZtsClient.java
+++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/ZtsClient.java
@@ -2,6 +2,7 @@
package com.yahoo.vespa.athenz.client.zts;
import com.yahoo.security.Pkcs10Csr;
+import com.yahoo.vespa.athenz.api.AthenzAccessToken;
import com.yahoo.vespa.athenz.api.AthenzDomain;
import com.yahoo.vespa.athenz.api.AthenzIdentity;
import com.yahoo.vespa.athenz.api.AthenzRole;
@@ -78,6 +79,22 @@ public interface ZtsClient extends AutoCloseable {
ZToken getRoleToken(AthenzRole athenzRole);
/**
+ * Fetch an access token for the target domain
+ *
+ * @param domain Target domain
+ * @return An Athenz access token
+ */
+ AthenzAccessToken getAccessToken(AthenzDomain domain);
+
+ /**
+ * Fetch an access token for the target roles
+ *
+ * @param athenzRole List of athenz roles to get access token for
+ * @return An Athenz access token
+ */
+ AthenzAccessToken getAccessToken(List<AthenzRole> athenzRole);
+
+ /**
* Fetch role certificate for the target domain and role
*
* @param role Target role
diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/bindings/AccessTokenResponseEntity.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/bindings/AccessTokenResponseEntity.java
new file mode 100644
index 00000000000..edd423b52bd
--- /dev/null
+++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/bindings/AccessTokenResponseEntity.java
@@ -0,0 +1,49 @@
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.athenz.client.zts.bindings;
+
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.yahoo.vespa.athenz.api.AthenzAccessToken;
+import com.yahoo.vespa.athenz.api.AthenzResourceName;
+import com.yahoo.vespa.athenz.api.AthenzRole;
+
+import java.time.Instant;
+import java.util.List;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+/**
+ * @author mortent
+ */
+@JsonIgnoreProperties(ignoreUnknown = true)
+public class AccessTokenResponseEntity {
+ private final AthenzAccessToken accessToken;
+ private final Instant expiryTime;
+ private final List<AthenzRole> roles;
+
+ public AccessTokenResponseEntity(
+ @JsonProperty("access_token") String accessToken,
+ @JsonProperty("expires_in") int expiresIn,
+ @JsonProperty("scope") String roles) {
+
+ this.accessToken = new AthenzAccessToken(accessToken);
+ // We do not know from when, so best we can do is assume now ...
+ this.expiryTime = Instant.now().plusSeconds(expiresIn);
+ this.roles = Stream.of(roles.split(" "))
+ .map(AthenzResourceName::fromString)
+ .map(AthenzRole::fromResourceName)
+ .collect(Collectors.toList());
+ }
+
+ public AthenzAccessToken accessToken() {
+ return accessToken;
+ }
+
+ public Instant expiryTime() {
+ return expiryTime;
+ }
+
+ public List<AthenzRole> roles() {
+ return roles;
+ }
+}
diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzIdentityProviderImpl.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzIdentityProviderImpl.java
index bea9af458b4..5d6f0e3ce16 100644
--- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzIdentityProviderImpl.java
+++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzIdentityProviderImpl.java
@@ -16,6 +16,7 @@ import com.yahoo.security.KeyStoreType;
import com.yahoo.security.Pkcs10Csr;
import com.yahoo.security.SslContextBuilder;
import com.yahoo.security.tls.MutableX509KeyManager;
+import com.yahoo.vespa.athenz.api.AthenzAccessToken;
import com.yahoo.vespa.athenz.api.AthenzDomain;
import com.yahoo.vespa.athenz.api.AthenzRole;
import com.yahoo.vespa.athenz.api.AthenzService;
@@ -84,6 +85,8 @@ public final class AthenzIdentityProviderImpl extends AbstractComponent implemen
private final LoadingCache<AthenzRole, SSLContext> roleSslContextCache;
private final LoadingCache<AthenzRole, ZToken> roleSpecificRoleTokenCache;
private final LoadingCache<AthenzDomain, ZToken> domainSpecificRoleTokenCache;
+ private final LoadingCache<AthenzDomain, AthenzAccessToken> domainSpecificAccessTokenCache;
+ private final LoadingCache<List<AthenzRole>, AthenzAccessToken> roleSpecificAccessTokenCache;
private final CsrGenerator csrGenerator;
@Inject
@@ -116,6 +119,8 @@ public final class AthenzIdentityProviderImpl extends AbstractComponent implemen
roleSslContextCache = createCache(ROLE_SSL_CONTEXT_EXPIRY, this::createRoleSslContext);
roleSpecificRoleTokenCache = createCache(ROLE_TOKEN_EXPIRY, this::createRoleToken);
domainSpecificRoleTokenCache = createCache(ROLE_TOKEN_EXPIRY, this::createRoleToken);
+ domainSpecificAccessTokenCache = createCache(ROLE_TOKEN_EXPIRY, this::createAccessToken);
+ roleSpecificAccessTokenCache = createCache(ROLE_TOKEN_EXPIRY, this::createAccessToken);
this.csrGenerator = new CsrGenerator(config.athenzDnsSuffix(), config.configserverIdentityName());
this.identitySslContext = createIdentitySslContext(identityKeyManager, trustStore);
registerInstance();
@@ -199,6 +204,16 @@ public final class AthenzIdentityProviderImpl extends AbstractComponent implemen
}
@Override
+ public String getAccessToken(String domain) {
+ return null;
+ }
+
+ @Override
+ public String getAccessToken(String domain, List<String> roles) {
+ return null;
+ }
+
+ @Override
public PrivateKey getPrivateKey() {
return credentials.getKeyPair().getPrivate();
}
@@ -240,6 +255,18 @@ public final class AthenzIdentityProviderImpl extends AbstractComponent implemen
}
}
+ private AthenzAccessToken createAccessToken(AthenzDomain domain) {
+ try (ZtsClient client = createZtsClient()) {
+ return client.getAccessToken(domain);
+ }
+ }
+
+ private AthenzAccessToken createAccessToken(List<AthenzRole> roles) {
+ try (ZtsClient client = createZtsClient()) {
+ return client.getAccessToken(roles);
+ }
+ }
+
private DefaultZtsClient createZtsClient() {
return new DefaultZtsClient(ztsEndpoint, getIdentitySslContext());
}
diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/tls/AthenzX509CertificateUtils.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/tls/AthenzX509CertificateUtils.java
index 33e5552eaf6..81525918f03 100644
--- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/tls/AthenzX509CertificateUtils.java
+++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/tls/AthenzX509CertificateUtils.java
@@ -43,10 +43,7 @@ public class AthenzX509CertificateUtils {
public static AthenzRole getRolesFromRoleCertificate(X509Certificate certificate) {
String commonName = com.yahoo.security.X509CertificateUtils.getSubjectCommonNames(certificate).get(0);
- int delimiterIndex = commonName.indexOf(COMMON_NAME_ROLE_DELIMITER);
- String domain = commonName.substring(0, delimiterIndex);
- String roleName = commonName.substring(delimiterIndex + COMMON_NAME_ROLE_DELIMITER.length());
- return new AthenzRole(domain, roleName);
+ return AthenzRole.fromResourceNameString(commonName);
}
private static AthenzIdentity getIdentityFromSanEmail(String email) {
diff --git a/zookeeper-server/zookeeper-server-3.5/src/test/java/com/yahoo/vespa/zookeeper/VespaZooKeeperServerImplTest.java b/zookeeper-server/zookeeper-server-3.5/src/test/java/com/yahoo/vespa/zookeeper/VespaZooKeeperServerImplTest.java
index 0981ae615df..863d2dba708 100644
--- a/zookeeper-server/zookeeper-server-3.5/src/test/java/com/yahoo/vespa/zookeeper/VespaZooKeeperServerImplTest.java
+++ b/zookeeper-server/zookeeper-server-3.5/src/test/java/com/yahoo/vespa/zookeeper/VespaZooKeeperServerImplTest.java
@@ -221,7 +221,7 @@ public class VespaZooKeeperServerImplTest {
"ssl.quorum.clientAuth=NEED\n" +
"ssl.quorum.ciphersuites=TLS_AES_128_GCM_SHA256,TLS_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384\n" +
"ssl.quorum.enabledProtocols=TLSv1.2\n" +
- "ssl.quorum.protocol=TLSv1.2\n";
+ "ssl.quorum.protocol=TLS\n";
}
private String commonTlsClientServerConfig() {
@@ -229,7 +229,7 @@ public class VespaZooKeeperServerImplTest {
"ssl.clientAuth=NEED\n" +
"ssl.ciphersuites=TLS_AES_128_GCM_SHA256,TLS_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384\n" +
"ssl.enabledProtocols=TLSv1.2\n" +
- "ssl.protocol=TLSv1.2\n";
+ "ssl.protocol=TLS\n";
}
private void validateConfigFileMultipleHosts(File cfgFile) throws IOException {