summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorHarald Musum <musum@verizonmedia.com>2021-07-09 13:52:30 +0200
committerHarald Musum <musum@verizonmedia.com>2021-07-09 13:52:30 +0200
commit273e9572331ae4360d3f0e4e7089dfc86392983d (patch)
tree3b0b1b46a4e793ac10c3ca0613f21c33ac175df7
parent95a7a658c876c62de0f04226f80f5ce5003e77a4 (diff)
parent6422dedc3926d604e071c087e0e8263f4c8f9390 (diff)
Merge branch 'master' into musum/cleanup-ConfigCurator-1
-rw-r--r--application/src/test/java/com/yahoo/application/container/ContainerTest.java8
-rw-r--r--athenz-identity-provider-service/pom.xml16
-rw-r--r--bundle-plugin/src/main/resources/META-INF/plexus/components.xml3
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/derived/SummaryClass.java5
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/documentmodel/DocumentSummary.java9
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java2
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/container/ContainerModelEvaluation.java12
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/container/xml/ConfigServerContainerModelBuilder.java6
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java9
-rw-r--r--config-model/src/main/javacc/SDParser.jj2
-rw-r--r--config-model/src/test/derived/advanced/summary.cfg2
-rw-r--r--config-model/src/test/derived/array_of_struct_attribute/summary.cfg1
-rw-r--r--config-model/src/test/derived/attributeprefetch/summary.cfg54
-rw-r--r--config-model/src/test/derived/complex/summary.cfg82
-rw-r--r--config-model/src/test/derived/emptychild/summary.cfg2
-rw-r--r--config-model/src/test/derived/emptydefault/summary.cfg1
-rw-r--r--config-model/src/test/derived/id/summary.cfg1
-rw-r--r--config-model/src/test/derived/imported_position_field/summary.cfg34
-rw-r--r--config-model/src/test/derived/imported_position_field_summary/summary.cfg3
-rw-r--r--config-model/src/test/derived/imported_struct_fields/summary.cfg88
-rw-r--r--config-model/src/test/derived/importedfields/summary.cfg3
-rw-r--r--config-model/src/test/derived/indexswitches/summary.cfg1
-rw-r--r--config-model/src/test/derived/inheritance/summary.cfg2
-rw-r--r--config-model/src/test/derived/integerattributetostringindex/summary.cfg2
-rw-r--r--config-model/src/test/derived/map_attribute/summary.cfg1
-rw-r--r--config-model/src/test/derived/map_of_struct_attribute/summary.cfg1
-rw-r--r--config-model/src/test/derived/mlr/summary.cfg2
-rw-r--r--config-model/src/test/derived/music/summary.cfg2
-rw-r--r--config-model/src/test/derived/newrank/summary.cfg2
-rw-r--r--config-model/src/test/derived/position_nosummary/summary.cfg2
-rw-r--r--config-model/src/test/derived/position_summary/summary.cfg2
-rw-r--r--config-model/src/test/derived/predicate_attribute/summary.cfg2
-rw-r--r--config-model/src/test/derived/rankexpression/summary.cfg2
-rw-r--r--config-model/src/test/derived/ranktypes/summary.cfg1
-rw-r--r--config-model/src/test/derived/reference_fields/summary.cfg3
-rw-r--r--config-model/src/test/derived/streamingstruct/summary.cfg2
-rw-r--r--config-model/src/test/derived/streamingstructdefault/summary.cfg1
-rw-r--r--config-model/src/test/derived/tensor/summary.cfg2
-rw-r--r--config-model/src/test/derived/tokenization/ilscripts.cfg9
-rw-r--r--config-model/src/test/derived/tokenization/tokenization.sd23
-rw-r--r--config-model/src/test/derived/types/summary.cfg2
-rw-r--r--config-model/src/test/java/com/yahoo/searchdefinition/derived/SummaryTestCase.java28
-rwxr-xr-xconfig-model/src/test/java/com/yahoo/searchdefinition/derived/TokenizationTestCase.java19
-rw-r--r--config-model/src/test/java/com/yahoo/vespa/model/container/configserver/ConfigserverClusterTest.java12
-rw-r--r--config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java15
-rw-r--r--config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTestBase.java6
-rw-r--r--config-proxy/src/test/java/com/yahoo/vespa/config/proxy/ConfigProxyRpcServerTest.java33
-rwxr-xr-xconfig/pom.xml6
-rw-r--r--configdefinitions/src/vespa/summary.def1
-rw-r--r--configserver-client/pom.xml6
-rw-r--r--configserver/pom.xml6
-rw-r--r--configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java24
-rw-r--r--configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpConfigRequest.java4
-rw-r--r--configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationApiHandler.java2
-rw-r--r--configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java533
-rw-r--r--configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/HostHandler.java1
-rw-r--r--configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/HttpGetConfigHandler.java1
-rw-r--r--configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/HttpListConfigsHandler.java2
-rw-r--r--configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/HttpListNamedConfigsHandler.java2
-rw-r--r--configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ListApplicationsHandler.java1
-rw-r--r--configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionActiveHandler.java1
-rw-r--r--configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionContentHandler.java1
-rw-r--r--configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionCreateHandler.java4
-rw-r--r--configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionPrepareHandler.java1
-rw-r--r--configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/TenantHandler.java4
-rw-r--r--configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/request/ApplicationContentRequest.java (renamed from configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationContentRequest.java)14
-rw-r--r--configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/request/HttpConfigRequests.java (renamed from configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/HttpConfigRequests.java)4
-rw-r--r--configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/request/HttpListConfigsRequest.java (renamed from configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/HttpListConfigsRequest.java)2
-rw-r--r--configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/request/SessionContentRequestV2.java (renamed from configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionContentRequestV2.java)6
-rw-r--r--configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/request/TenantRequest.java (renamed from configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/TenantRequest.java)2
-rw-r--r--configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/response/ApplicationSuspendedResponse.java12
-rw-r--r--configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/response/DeleteApplicationResponse.java11
-rw-r--r--configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/response/DeploymentMetricsResponse.java (renamed from configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/DeploymentMetricsResponse.java)2
-rw-r--r--configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/response/GetApplicationResponse.java19
-rw-r--r--configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/response/ListApplicationsResponse.java (renamed from configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ListApplicationsResponse.java)2
-rw-r--r--configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/response/ListTenantsResponse.java (renamed from configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ListTenantsResponse.java)7
-rw-r--r--configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/response/ProtonMetricsResponse.java (renamed from configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ProtonMetricsResponse.java)2
-rw-r--r--configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/response/QuotaUsageResponse.java12
-rw-r--r--configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/response/ReindexingResponse.java52
-rw-r--r--configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/response/SessionActiveResponse.java (renamed from configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionActiveResponse.java)2
-rw-r--r--configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/response/SessionCreateResponse.java (renamed from configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionCreateResponse.java)2
-rw-r--r--configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/response/SessionPrepareAndActivateResponse.java (renamed from configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionPrepareAndActivateResponse.java)7
-rw-r--r--configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/response/SessionPrepareResponse.java (renamed from configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionPrepareResponse.java)9
-rw-r--r--configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/response/TenantCreateResponse.java (renamed from configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/TenantCreateResponse.java)2
-rw-r--r--configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/response/TenantDeleteResponse.java (renamed from configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/TenantDeleteResponse.java)2
-rw-r--r--configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/response/TenantGetResponse.java (renamed from configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/TenantGetResponse.java)2
-rw-r--r--configserver/src/main/java/com/yahoo/vespa/config/server/metrics/DeploymentMetricsRetriever.java2
-rw-r--r--configserver/src/main/java/com/yahoo/vespa/config/server/metrics/ProtonMetricsRetriever.java2
-rw-r--r--configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClient.java2
-rwxr-xr-xconfigserver/src/main/sh/start-configserver1
-rw-r--r--configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java25
-rw-r--r--container-apache-http-client-bundle/src/main/java/org/apache/http/client/entity/package-info.java8
-rw-r--r--container-core/pom.xml26
-rw-r--r--container-core/src/main/java/com/yahoo/container/core/config/HandlersConfigurerDi.java7
-rw-r--r--container-core/src/test/java/com/yahoo/jdisc/http/server/jetty/EchoRequestHandler.java24
-rw-r--r--container-core/src/test/java/com/yahoo/jdisc/http/server/jetty/HttpServerTest.java390
-rw-r--r--container-core/src/test/java/com/yahoo/jdisc/http/server/jetty/ProxyProtocolTest.java198
-rw-r--r--container-core/src/test/java/com/yahoo/jdisc/http/server/jetty/SslHandshakeMetricsTest.java189
-rw-r--r--container-core/src/test/java/com/yahoo/jdisc/http/server/jetty/Utils.java68
-rw-r--r--container-disc/pom.xml3
-rw-r--r--container-disc/src/main/java/com/yahoo/container/jdisc/ConfiguredApplication.java4
-rwxr-xr-xcontainer-disc/src/main/sh/vespa-start-container-daemon.sh1
-rw-r--r--container-messagebus/src/main/java/com/yahoo/container/jdisc/messagebus/SessionCache.java114
-rw-r--r--container-messagebus/src/main/java/com/yahoo/messagebus/jdisc/test/ClientTestDriver.java5
-rw-r--r--container-messagebus/src/main/java/com/yahoo/messagebus/jdisc/test/RemoteClient.java34
-rw-r--r--container-messagebus/src/main/java/com/yahoo/messagebus/jdisc/test/RemoteServer.java39
-rw-r--r--container-messagebus/src/main/java/com/yahoo/messagebus/jdisc/test/ServerTestDriver.java9
-rw-r--r--container-messagebus/src/main/java/com/yahoo/messagebus/jdisc/test/TestUtils.java21
-rw-r--r--container-messagebus/src/main/java/com/yahoo/messagebus/shared/SharedMessageBus.java5
-rw-r--r--container-messagebus/src/test/java/com/yahoo/container/jdisc/messagebus/MbusClientProviderTest.java17
-rw-r--r--container-messagebus/src/test/java/com/yahoo/messagebus/jdisc/test/ClientTestDriverTestCase.java6
-rw-r--r--container-messagebus/src/test/java/com/yahoo/messagebus/jdisc/test/ServerTestDriverTestCase.java5
-rw-r--r--container-messagebus/src/test/java/com/yahoo/messagebus/shared/SharedDestinationSessionTestCase.java10
-rw-r--r--container-messagebus/src/test/java/com/yahoo/messagebus/shared/SharedIntermediateSessionTestCase.java14
-rw-r--r--container-messagebus/src/test/java/com/yahoo/messagebus/shared/SharedSourceSessionTestCase.java10
-rw-r--r--container-messagebus/src/test/resources/config/clientprovider/container-mbus.cfg0
-rw-r--r--container-messagebus/src/test/resources/config/clientprovider/documentmanager.cfg0
-rw-r--r--container-messagebus/src/test/resources/config/clientprovider/load-type.cfg0
-rw-r--r--container-messagebus/src/test/resources/config/clientprovider/messagebus.cfg0
-rw-r--r--container-messagebus/src/test/resources/config/clientprovider/slobroks.cfg0
-rw-r--r--controller-api/pom.xml7
-rw-r--r--controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServer.java2
-rw-r--r--controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/ApplicationSummary.java7
-rw-r--r--controller-server/pom.xml25
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java36
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java25
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Run.java31
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Step.java24
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ApplicationOwnershipConfirmer.java46
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java1
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentExpirer.java6
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentUpgrader.java92
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainer.java4
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgradeScheduler.java7
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java5
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java110
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java22
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/Badges.java2
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java65
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java31
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentExpirerTest.java2
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentUpgraderTest.java101
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainerTest.java4
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java4
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgradeSchedulerTest.java10
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializerTest.java2
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java6
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelperTest.java8
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/system-test-job.json178
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json3
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/BadgeApiTest.java5
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/responses/single-done.svg81
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/responses/single-running.svg81
-rw-r--r--default_build_settings.cmake6
-rw-r--r--dist/vespa.spec14
-rw-r--r--docproc/src/test/java/com/yahoo/docproc/jdisc/DocumentProcessingHandlerAllMessageTypesTestCase.java2
-rw-r--r--docproc/src/test/java/com/yahoo/docproc/jdisc/DocumentProcessingHandlerTestBase.java15
-rw-r--r--filedistribution/pom.xml10
-rw-r--r--flags/src/main/java/com/yahoo/vespa/flags/Flags.java12
-rw-r--r--http-utils/pom.xml9
-rw-r--r--indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/NormalizeExpression.java13
-rw-r--r--indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/parser/ExpressionTestCase.java1
-rw-r--r--jdisc-cloud-aws/pom.xml12
-rw-r--r--messagebus/src/tests/targetpool/targetpool.cpp2
-rw-r--r--messagebus/src/vespa/messagebus/network/rpcnetwork.cpp2
-rw-r--r--messagebus/src/vespa/messagebus/network/rpctargetpool.cpp78
-rw-r--r--messagebus/src/vespa/messagebus/network/rpctargetpool.h22
-rw-r--r--metrics-proxy/pom.xml6
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/ContainerEngine.java12
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/ContainerOperations.java5
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/PartialContainer.java5
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/image/ContainerImagePruner.java9
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumCommand.java27
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumPackageName.java87
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumTester.java27
-rw-r--r--node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumVersion.java25
-rw-r--r--node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumPackageNameTest.java54
-rw-r--r--node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumTest.java68
-rw-r--r--node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumTesterTest.java2
-rw-r--r--node-repository/pom.xml10
-rw-r--r--node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/AllocatableClusterResources.java9
-rw-r--r--node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeResourceLimits.java6
-rw-r--r--node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicAllocationTest.java33
-rw-r--r--node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicProvisioningTest.java33
-rw-r--r--orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/ZkStatusService.java15
-rw-r--r--parent/pom.xml1
-rw-r--r--searchcore/CMakeLists.txt1
-rw-r--r--searchcore/src/tests/proton/server/disk_mem_usage_metrics/CMakeLists.txt9
-rw-r--r--searchcore/src/tests/proton/server/disk_mem_usage_metrics/disk_mem_usage_metrics_test.cpp39
-rw-r--r--searchcore/src/vespa/searchcore/proton/docsummary/docsumcontext.cpp2
-rw-r--r--searchcore/src/vespa/searchcore/proton/server/CMakeLists.txt1
-rw-r--r--searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_filter.cpp13
-rw-r--r--searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_filter.h8
-rw-r--r--searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_metrics.cpp31
-rw-r--r--searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_metrics.h30
-rw-r--r--searchcore/src/vespa/searchcore/proton/server/proton.cpp10
-rw-r--r--searchcore/src/vespa/searchcore/proton/server/resource_usage_state.h2
-rw-r--r--searchsummary/src/vespa/searchsummary/docsummary/docsumstate.cpp1
-rw-r--r--searchsummary/src/vespa/searchsummary/docsummary/docsumstate.h1
-rw-r--r--searchsummary/src/vespa/searchsummary/docsummary/resultclass.cpp3
-rw-r--r--searchsummary/src/vespa/searchsummary/docsummary/resultclass.h11
-rw-r--r--searchsummary/src/vespa/searchsummary/docsummary/resultconfig.cpp22
-rw-r--r--searchsummary/src/vespa/searchsummary/docsummary/summaryfeaturesdfw.cpp3
-rw-r--r--service-monitor/pom.xml14
-rw-r--r--standalone-container/pom.xml2
-rwxr-xr-xstandalone-container/src/main/sh/standalone-container.sh1
-rw-r--r--vespa-athenz/pom.xml20
-rw-r--r--vespa-hadoop/src/main/java/com/yahoo/vespa/hadoop/mapreduce/VespaOutputFormat.java2
-rw-r--r--vespa-hadoop/src/main/java/com/yahoo/vespa/hadoop/mapreduce/util/VespaConfiguration.java8
-rw-r--r--vespa-maven-plugin/src/main/java/ai/vespa/hosted/plugin/GenerateTestDescriptorMojo.java2
-rw-r--r--vespaclient-container-plugin/pom.xml5
-rw-r--r--vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/DocumentV1ApiHandler.java2
-rw-r--r--zkfacade/pom.xml40
-rw-r--r--zookeeper-server/zookeeper-server-3.6.2/pom.xml33
-rw-r--r--zookeeper-server/zookeeper-server-3.6.3/pom.xml33
215 files changed, 2736 insertions, 1810 deletions
diff --git a/application/src/test/java/com/yahoo/application/container/ContainerTest.java b/application/src/test/java/com/yahoo/application/container/ContainerTest.java
index 63347c2475a..29f895ce176 100644
--- a/application/src/test/java/com/yahoo/application/container/ContainerTest.java
+++ b/application/src/test/java/com/yahoo/application/container/ContainerTest.java
@@ -35,7 +35,7 @@ import static org.junit.Assert.fail;
public class ContainerTest {
@Test
- public void jdisc_can_be_used_as_top_level_element() {
+ public void container_can_be_used_as_top_level_element() {
try (JDisc container = fromServicesXml("<container version=\"1.0\">" + //
"<search />" + //
"</container>", Networking.disable)) {
@@ -44,7 +44,7 @@ public class ContainerTest {
}
@Test
- public void jdisc_id_can_be_set() {
+ public void container_id_can_be_set() {
try (JDisc container = fromServicesXml("<container version=\"1.0\" id=\"my-service-id\">" + //
"<search />" + //
"</container>", Networking.disable)) {
@@ -53,7 +53,7 @@ public class ContainerTest {
}
@Test
- public void jdisc_can_be_embedded_in_services_tag() {
+ public void container_can_be_embedded_in_services_tag() {
try (JDisc container = fromServicesXml("<services>" + //
"<container version=\"1.0\" id=\"my-service-id\">" + //
"<search />" + //
@@ -65,7 +65,7 @@ public class ContainerTest {
@Test
@SuppressWarnings("try") // container is unused inside the try block
- public void multiple_jdisc_elements_gives_exception() {
+ public void multiple_container_elements_gives_exception() {
try (JDisc container = fromServicesXml("<services>" + //
"<container version=\"1.0\" id=\"id1\" />" + //
"<container version=\"1.0\" />" + //
diff --git a/athenz-identity-provider-service/pom.xml b/athenz-identity-provider-service/pom.xml
index 855b3afafaf..3c5f96a1fec 100644
--- a/athenz-identity-provider-service/pom.xml
+++ b/athenz-identity-provider-service/pom.xml
@@ -28,6 +28,12 @@
</dependency>
<dependency>
<groupId>com.yahoo.vespa</groupId>
+ <artifactId>container-apache-http-client-bundle</artifactId>
+ <version>${project.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>com.yahoo.vespa</groupId>
<artifactId>container-dev</artifactId>
<version>${project.version}</version>
<scope>provided</scope>
@@ -79,16 +85,6 @@
<scope>provided</scope>
</dependency>
- <!-- COMPILE -->
- <dependency>
- <groupId>org.apache.httpcomponents</groupId>
- <artifactId>httpclient</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.httpcomponents</groupId>
- <artifactId>httpcore</artifactId>
- </dependency>
-
<!-- TEST -->
<dependency>
<groupId>com.yahoo.vespa</groupId>
diff --git a/bundle-plugin/src/main/resources/META-INF/plexus/components.xml b/bundle-plugin/src/main/resources/META-INF/plexus/components.xml
index 31e21eb18fd..f305d02c4ae 100644
--- a/bundle-plugin/src/main/resources/META-INF/plexus/components.xml
+++ b/bundle-plugin/src/main/resources/META-INF/plexus/components.xml
@@ -16,8 +16,7 @@
<process-resources>org.apache.maven.plugins:maven-resources-plugin:resources</process-resources>
<compile>org.apache.maven.plugins:maven-compiler-plugin:compile</compile>
<process-test-resources>
- org.apache.maven.plugins:maven-resources-plugin:testResources,
- com.yahoo.vespa:bundle-plugin:generate-bundle-classpath-mappings
+ org.apache.maven.plugins:maven-resources-plugin:testResources
</process-test-resources>
<test-compile>org.apache.maven.plugins:maven-compiler-plugin:testCompile</test-compile>
<test>org.apache.maven.plugins:maven-surefire-plugin:test</test>
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/SummaryClass.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/SummaryClass.java
index af2168545dc..c173b2c11d9 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/SummaryClass.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/SummaryClass.java
@@ -28,6 +28,7 @@ public class SummaryClass extends Derived {
/** True if this summary class needs to access summary information on disk */
private boolean accessingDiskSummary = false;
private final boolean rawAsBase64;
+ private final boolean omitSummaryFeatures;
/** The summary fields of this indexed by name */
private Map<String,SummaryClassField> fields = new java.util.LinkedHashMap<>();
@@ -44,6 +45,7 @@ public class SummaryClass extends Derived {
public SummaryClass(Search search, DocumentSummary summary, DeployLogger deployLogger) {
this.deployLogger = deployLogger;
this.rawAsBase64 = search.isRawAsBase64();
+ this.omitSummaryFeatures = summary.omitSummaryFeatures();
deriveName(summary);
deriveFields(search,summary);
deriveImplicitFields(summary);
@@ -128,7 +130,8 @@ public class SummaryClass extends Derived {
}
classBuilder.
id(id).
- name(getName());
+ name(getName()).
+ omitsummaryfeatures(omitSummaryFeatures);
for (SummaryClassField field : fields.values() ) {
classBuilder.fields(new SummaryConfig.Classes.Fields.Builder().
name(field.getName()).
diff --git a/config-model/src/main/java/com/yahoo/vespa/documentmodel/DocumentSummary.java b/config-model/src/main/java/com/yahoo/vespa/documentmodel/DocumentSummary.java
index 3c6aa881af4..454d0f08ff0 100644
--- a/config-model/src/main/java/com/yahoo/vespa/documentmodel/DocumentSummary.java
+++ b/config-model/src/main/java/com/yahoo/vespa/documentmodel/DocumentSummary.java
@@ -14,6 +14,7 @@ import java.util.List;
public class DocumentSummary extends FieldView {
private boolean fromDisk = false;
+ private boolean omitSummaryFeatures = false;
private DocumentSummary inherited;
/**
@@ -30,6 +31,14 @@ public class DocumentSummary extends FieldView {
/** Returns whether the user has noted explicitly that this summary accesses disk */
public boolean isFromDisk() { return fromDisk; }
+ public void setOmitSummaryFeatures(boolean value) {
+ omitSummaryFeatures = value;
+ }
+
+ public boolean omitSummaryFeatures() {
+ return omitSummaryFeatures;
+ }
+
/**
* The model is constrained to ensure that summary fields of the same name
* in different classes have the same summary transform, because this is
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java
index fde3e3f012e..317ed0f66c7 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java
@@ -111,8 +111,6 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat
addSimpleComponent("com.yahoo.container.jdisc.AthenzIdentityProviderProvider");
addSimpleComponent("com.yahoo.container.jdisc.SystemInfoProvider");
addSimpleComponent(com.yahoo.container.core.documentapi.DocumentAccessProvider.class.getName());
- addSimpleComponent("com.yahoo.container.jdisc.messagebus.SessionCache");
-
addMetricsHandlers();
addTestrunnerComponentsIfTester(deployState);
}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerModelEvaluation.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerModelEvaluation.java
index 37bfb8821c3..65247f29281 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerModelEvaluation.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerModelEvaluation.java
@@ -10,7 +10,9 @@ import com.yahoo.vespa.config.search.core.RankingConstantsConfig;
import com.yahoo.vespa.config.search.core.RankingExpressionsConfig;
import com.yahoo.vespa.model.container.component.Handler;
import com.yahoo.vespa.model.container.component.SystemBindingPattern;
+import com.yahoo.vespa.model.container.xml.PlatformBundles;
+import java.nio.file.Path;
import java.util.List;
import java.util.Objects;
@@ -26,17 +28,21 @@ public class ContainerModelEvaluation implements
OnnxModelsConfig.Producer,
RankingExpressionsConfig.Producer {
- private final static String BUNDLE_NAME = "model-evaluation";
+ private final static String EVALUATION_BUNDLE_NAME = "model-evaluation";
+ private final static String INTEGRATION_BUNDLE_NAME = "model-integration";
private final static String EVALUATOR_NAME = ModelsEvaluator.class.getName();
private final static String REST_HANDLER_NAME = "ai.vespa.models.handler.ModelsEvaluationHandler";
private final static String REST_BINDING_PATH = "/model-evaluation/v1";
+ public static final Path MODEL_EVALUATION_BUNDLE_FILE = PlatformBundles.absoluteBundlePath(EVALUATION_BUNDLE_NAME);
+ public static final Path MODEL_INTEGRATION_BUNDLE_FILE = PlatformBundles.absoluteBundlePath(INTEGRATION_BUNDLE_NAME);
+
/** Global rank profiles, aka models */
private final RankProfileList rankProfileList;
public ContainerModelEvaluation(ApplicationContainerCluster cluster, RankProfileList rankProfileList) {
this.rankProfileList = Objects.requireNonNull(rankProfileList, "rankProfileList cannot be null");
- cluster.addSimpleComponent(EVALUATOR_NAME, null, BUNDLE_NAME);
+ cluster.addSimpleComponent(EVALUATOR_NAME, null, EVALUATION_BUNDLE_NAME);
cluster.addComponent(ContainerModelEvaluation.getHandler());
}
@@ -64,7 +70,7 @@ public class ContainerModelEvaluation implements
}
public static Handler<?> getHandler() {
- Handler<?> handler = new Handler<>(new ComponentModel(REST_HANDLER_NAME, null, BUNDLE_NAME));
+ Handler<?> handler = new Handler<>(new ComponentModel(REST_HANDLER_NAME, null, EVALUATION_BUNDLE_NAME));
handler.addServerBindings(
SystemBindingPattern.fromHttpPath(REST_BINDING_PATH),
SystemBindingPattern.fromHttpPath(REST_BINDING_PATH + "/*"));
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ConfigServerContainerModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ConfigServerContainerModelBuilder.java
index a37d0ef416f..a1f52cca9fd 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ConfigServerContainerModelBuilder.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ConfigServerContainerModelBuilder.java
@@ -6,6 +6,7 @@ import com.yahoo.config.model.deploy.DeployState;
import com.yahoo.container.logging.FileConnectionLog;
import com.yahoo.vespa.model.container.ApplicationContainerCluster;
import com.yahoo.vespa.model.container.ContainerModel;
+import com.yahoo.vespa.model.container.ContainerModelEvaluation;
import com.yahoo.vespa.model.container.component.AccessLogComponent;
import com.yahoo.vespa.model.container.component.ConnectionLogComponent;
import com.yahoo.vespa.model.container.configserver.ConfigserverCluster;
@@ -61,6 +62,11 @@ public class ConfigServerContainerModelBuilder extends ContainerModelBuilder {
cluster.getHttp().getHttpServer().get().setHostedVespa(isHosted());
}
+ @Override
+ protected void addModelEvaluationBundles(ApplicationContainerCluster cluster) {
+ // Model evaluation bundles are pre-installed in the standalone container.
+ }
+
/** Note: using {@link CloudConfigOptions} as {@link DeployState#isHosted()} returns <em>false</em> for hosted configserver/controller */
private boolean isHosted() { return options.hostedVespa().orElse(Boolean.FALSE); }
}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java
index 3b04d536300..5bf8aa5228e 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java
@@ -187,6 +187,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> {
throwUponRestApi(spec); // TODO: remove
addServlets(deployState, spec, cluster);
addModelEvaluation(spec, cluster, context);
+ addModelEvaluationBundles(cluster);
addProcessing(deployState, spec, cluster);
addSearch(deployState, spec, cluster);
@@ -565,6 +566,14 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> {
cluster.setModelEvaluation(new ContainerModelEvaluation(cluster, profiles));
}
+ protected void addModelEvaluationBundles(ApplicationContainerCluster cluster) {
+ /* These bundles are added to all application container clusters, even if they haven't
+ * declared 'model-evaluation' in services.xml, because there are many public API packages
+ * in the model-evaluation bundle that could be used by customer code. */
+ cluster.addPlatformBundle(ContainerModelEvaluation.MODEL_EVALUATION_BUNDLE_FILE);
+ cluster.addPlatformBundle(ContainerModelEvaluation.MODEL_INTEGRATION_BUNDLE_FILE);
+ }
+
private void addProcessing(DeployState deployState, Element spec, ApplicationContainerCluster cluster) {
Element processingElement = XML.getChild(spec, "processing");
if (processingElement == null) return;
diff --git a/config-model/src/main/javacc/SDParser.jj b/config-model/src/main/javacc/SDParser.jj
index c247100db76..fb24b50aa99 100644
--- a/config-model/src/main/javacc/SDParser.jj
+++ b/config-model/src/main/javacc/SDParser.jj
@@ -291,6 +291,7 @@ TOKEN :
| < TO: "to" >
| < DIRECT: "direct" >
| < FROMDISK: "from-disk" >
+| < OMITSUMMARYFEATURES: "omit-summary-features" >
| < ALWAYS: "always" >
| < ONDEMAND: "on-demand" >
| < NEVER: "never" >
@@ -1767,6 +1768,7 @@ Object documentSummary(Search search) :
lbrace()
(
<FROMDISK> { summary.setFromDisk(true); } |
+ <OMITSUMMARYFEATURES> { summary.setOmitSummaryFeatures(true); } |
documentSummaryItem(summary) |
<NL>
)*
diff --git a/config-model/src/test/derived/advanced/summary.cfg b/config-model/src/test/derived/advanced/summary.cfg
index a56d0e6aa4f..f497461b460 100644
--- a/config-model/src/test/derived/advanced/summary.cfg
+++ b/config-model/src/test/derived/advanced/summary.cfg
@@ -1,6 +1,7 @@
defaultsummaryid 1271952241
classes[].id 1271952241
classes[].name "default"
+classes[].omitsummaryfeatures false
classes[].fields[].name "debug"
classes[].fields[].type "longstring"
classes[].fields[].name "attributes"
@@ -25,6 +26,7 @@ classes[].fields[].name "documentid"
classes[].fields[].type "longstring"
classes[].id 472092010
classes[].name "attributeprefetch"
+classes[].omitsummaryfeatures false
classes[].fields[].name "location_zcurve"
classes[].fields[].type "int64"
classes[].fields[].name "rankfeatures"
diff --git a/config-model/src/test/derived/array_of_struct_attribute/summary.cfg b/config-model/src/test/derived/array_of_struct_attribute/summary.cfg
index c1679c57d1a..965c875d5ce 100644
--- a/config-model/src/test/derived/array_of_struct_attribute/summary.cfg
+++ b/config-model/src/test/derived/array_of_struct_attribute/summary.cfg
@@ -1,6 +1,7 @@
defaultsummaryid 252850086
classes[].id 252850086
classes[].name "default"
+classes[].omitsummaryfeatures false
classes[].fields[].name "elem_array"
classes[].fields[].type "jsonstring"
classes[].fields[].name "rankfeatures"
diff --git a/config-model/src/test/derived/attributeprefetch/summary.cfg b/config-model/src/test/derived/attributeprefetch/summary.cfg
index 13b17464946..f0189f9a3c7 100644
--- a/config-model/src/test/derived/attributeprefetch/summary.cfg
+++ b/config-model/src/test/derived/attributeprefetch/summary.cfg
@@ -1,27 +1,29 @@
defaultsummaryid 1151071433
-classes[0].id 1151071433
-classes[0].name "default"
-classes[0].fields[0].name "rankfeatures"
-classes[0].fields[0].type "featuredata"
-classes[0].fields[1].name "summaryfeatures"
-classes[0].fields[1].type "featuredata"
-classes[0].fields[2].name "documentid"
-classes[0].fields[2].type "longstring"
-classes[1].id 1980470965
-classes[1].name "attributeprefetch"
-classes[1].fields[0].name "singlebyte"
-classes[1].fields[0].type "byte"
-classes[1].fields[1].name "singleint"
-classes[1].fields[1].type "integer"
-classes[1].fields[2].name "singlelong"
-classes[1].fields[2].type "int64"
-classes[1].fields[3].name "singlefloat"
-classes[1].fields[3].type "float"
-classes[1].fields[4].name "singledouble"
-classes[1].fields[4].type "double"
-classes[1].fields[5].name "singlestring"
-classes[1].fields[5].type "longstring"
-classes[1].fields[6].name "rankfeatures"
-classes[1].fields[6].type "featuredata"
-classes[1].fields[7].name "summaryfeatures"
-classes[1].fields[7].type "featuredata" \ No newline at end of file
+classes[].id 1151071433
+classes[].name "default"
+classes[].omitsummaryfeatures false
+classes[].fields[].name "rankfeatures"
+classes[].fields[].type "featuredata"
+classes[].fields[].name "summaryfeatures"
+classes[].fields[].type "featuredata"
+classes[].fields[].name "documentid"
+classes[].fields[].type "longstring"
+classes[].id 1980470965
+classes[].name "attributeprefetch"
+classes[].omitsummaryfeatures false
+classes[].fields[].name "singlebyte"
+classes[].fields[].type "byte"
+classes[].fields[].name "singleint"
+classes[].fields[].type "integer"
+classes[].fields[].name "singlelong"
+classes[].fields[].type "int64"
+classes[].fields[].name "singlefloat"
+classes[].fields[].type "float"
+classes[].fields[].name "singledouble"
+classes[].fields[].type "double"
+classes[].fields[].name "singlestring"
+classes[].fields[].type "longstring"
+classes[].fields[].name "rankfeatures"
+classes[].fields[].type "featuredata"
+classes[].fields[].name "summaryfeatures"
+classes[].fields[].type "featuredata"
diff --git a/config-model/src/test/derived/complex/summary.cfg b/config-model/src/test/derived/complex/summary.cfg
index 5bb472b5f41..2dac4736d23 100644
--- a/config-model/src/test/derived/complex/summary.cfg
+++ b/config-model/src/test/derived/complex/summary.cfg
@@ -1,41 +1,43 @@
defaultsummaryid 1506848752
-classes[0].id 1506848752
-classes[0].name "default"
-classes[0].fields[0].name "woe"
-classes[0].fields[0].type "longstring"
-classes[0].fields[1].name "exact"
-classes[0].fields[1].type "longstring"
-classes[0].fields[2].name "title"
-classes[0].fields[2].type "longstring"
-classes[0].fields[3].name "dyntitle"
-classes[0].fields[3].type "longstring"
-classes[0].fields[4].name "source"
-classes[0].fields[4].type "longstring"
-classes[0].fields[5].name "stringfield"
-classes[0].fields[5].type "longstring"
-classes[0].fields[6].name "rankfeatures"
-classes[0].fields[6].type "featuredata"
-classes[0].fields[7].name "summaryfeatures"
-classes[0].fields[7].type "featuredata"
-classes[0].fields[8].name "documentid"
-classes[0].fields[8].type "longstring"
-classes[1].id 128090024
-classes[1].name "attributeprefetch"
-classes[1].fields[0].name "year_sub"
-classes[1].fields[0].type "integer"
-classes[1].fields[1].name "prefixenabled"
-classes[1].fields[1].type "longstring"
-classes[1].fields[2].name "fleeting2"
-classes[1].fields[2].type "float"
-classes[1].fields[3].name "foundat"
-classes[1].fields[3].type "int64"
-classes[1].fields[4].name "collapseby"
-classes[1].fields[4].type "integer"
-classes[1].fields[5].name "ts"
-classes[1].fields[5].type "int64"
-classes[1].fields[6].name "combineda"
-classes[1].fields[6].type "integer"
-classes[1].fields[7].name "rankfeatures"
-classes[1].fields[7].type "featuredata"
-classes[1].fields[8].name "summaryfeatures"
-classes[1].fields[8].type "featuredata" \ No newline at end of file
+classes[].id 1506848752
+classes[].name "default"
+classes[].omitsummaryfeatures false
+classes[].fields[].name "woe"
+classes[].fields[].type "longstring"
+classes[].fields[].name "exact"
+classes[].fields[].type "longstring"
+classes[].fields[].name "title"
+classes[].fields[].type "longstring"
+classes[].fields[].name "dyntitle"
+classes[].fields[].type "longstring"
+classes[].fields[].name "source"
+classes[].fields[].type "longstring"
+classes[].fields[].name "stringfield"
+classes[].fields[].type "longstring"
+classes[].fields[].name "rankfeatures"
+classes[].fields[].type "featuredata"
+classes[].fields[].name "summaryfeatures"
+classes[].fields[].type "featuredata"
+classes[].fields[].name "documentid"
+classes[].fields[].type "longstring"
+classes[].id 128090024
+classes[].name "attributeprefetch"
+classes[].omitsummaryfeatures false
+classes[].fields[].name "year_sub"
+classes[].fields[].type "integer"
+classes[].fields[].name "prefixenabled"
+classes[].fields[].type "longstring"
+classes[].fields[].name "fleeting2"
+classes[].fields[].type "float"
+classes[].fields[].name "foundat"
+classes[].fields[].type "int64"
+classes[].fields[].name "collapseby"
+classes[].fields[].type "integer"
+classes[].fields[].name "ts"
+classes[].fields[].type "int64"
+classes[].fields[].name "combineda"
+classes[].fields[].type "integer"
+classes[].fields[].name "rankfeatures"
+classes[].fields[].type "featuredata"
+classes[].fields[].name "summaryfeatures"
+classes[].fields[].type "featuredata"
diff --git a/config-model/src/test/derived/emptychild/summary.cfg b/config-model/src/test/derived/emptychild/summary.cfg
index ed3a61a5de5..82bed7fd55e 100644
--- a/config-model/src/test/derived/emptychild/summary.cfg
+++ b/config-model/src/test/derived/emptychild/summary.cfg
@@ -1,6 +1,7 @@
defaultsummaryid 1814603381
classes[].id 1814603381
classes[].name "default"
+classes[].omitsummaryfeatures false
classes[].fields[].name "a1"
classes[].fields[].type "longstring"
classes[].fields[].name "rankfeatures"
@@ -11,6 +12,7 @@ classes[].fields[].name "documentid"
classes[].fields[].type "longstring"
classes[].id 1490368133
classes[].name "attributeprefetch"
+classes[].omitsummaryfeatures false
classes[].fields[].name "a1"
classes[].fields[].type "longstring"
classes[].fields[].name "rankfeatures"
diff --git a/config-model/src/test/derived/emptydefault/summary.cfg b/config-model/src/test/derived/emptydefault/summary.cfg
index e47f24b21c3..61294d97b4c 100644
--- a/config-model/src/test/derived/emptydefault/summary.cfg
+++ b/config-model/src/test/derived/emptydefault/summary.cfg
@@ -1,6 +1,7 @@
defaultsummaryid 1151071433
classes[].id 1151071433
classes[].name "default"
+classes[].omitsummaryfeatures false
classes[].fields[].name "rankfeatures"
classes[].fields[].type "featuredata"
classes[].fields[].name "summaryfeatures"
diff --git a/config-model/src/test/derived/id/summary.cfg b/config-model/src/test/derived/id/summary.cfg
index dbc9a90ebce..b50b970afe2 100644
--- a/config-model/src/test/derived/id/summary.cfg
+++ b/config-model/src/test/derived/id/summary.cfg
@@ -1,6 +1,7 @@
defaultsummaryid 1814716401
classes[].id 1814716401
classes[].name "default"
+classes[].omitsummaryfeatures false
classes[].fields[].name "uri"
classes[].fields[].type "longstring"
classes[].fields[].name "rankfeatures"
diff --git a/config-model/src/test/derived/imported_position_field/summary.cfg b/config-model/src/test/derived/imported_position_field/summary.cfg
index 8da2598a5fc..3ab8e7e29e5 100644
--- a/config-model/src/test/derived/imported_position_field/summary.cfg
+++ b/config-model/src/test/derived/imported_position_field/summary.cfg
@@ -1,17 +1,19 @@
defaultsummaryid 1570252291
-classes[0].id 1570252291
-classes[0].name "default"
-classes[0].fields[0].name "parent_ref"
-classes[0].fields[0].type "longstring"
-classes[0].fields[1].name "rankfeatures"
-classes[0].fields[1].type "featuredata"
-classes[0].fields[2].name "summaryfeatures"
-classes[0].fields[2].type "featuredata"
-classes[0].fields[3].name "documentid"
-classes[0].fields[3].type "longstring"
-classes[1].id 1274088866
-classes[1].name "attributeprefetch"
-classes[1].fields[0].name "rankfeatures"
-classes[1].fields[0].type "featuredata"
-classes[1].fields[1].name "summaryfeatures"
-classes[1].fields[1].type "featuredata" \ No newline at end of file
+classes[].id 1570252291
+classes[].name "default"
+classes[].omitsummaryfeatures false
+classes[].fields[].name "parent_ref"
+classes[].fields[].type "longstring"
+classes[].fields[].name "rankfeatures"
+classes[].fields[].type "featuredata"
+classes[].fields[].name "summaryfeatures"
+classes[].fields[].type "featuredata"
+classes[].fields[].name "documentid"
+classes[].fields[].type "longstring"
+classes[].id 1274088866
+classes[].name "attributeprefetch"
+classes[].omitsummaryfeatures false
+classes[].fields[].name "rankfeatures"
+classes[].fields[].type "featuredata"
+classes[].fields[].name "summaryfeatures"
+classes[].fields[].type "featuredata"
diff --git a/config-model/src/test/derived/imported_position_field_summary/summary.cfg b/config-model/src/test/derived/imported_position_field_summary/summary.cfg
index b607786ca36..76faac23170 100644
--- a/config-model/src/test/derived/imported_position_field_summary/summary.cfg
+++ b/config-model/src/test/derived/imported_position_field_summary/summary.cfg
@@ -1,6 +1,7 @@
defaultsummaryid 1194448774
classes[].id 1194448774
classes[].name "default"
+classes[].omitsummaryfeatures false
classes[].fields[].name "parent_ref"
classes[].fields[].type "longstring"
classes[].fields[].name "rankfeatures"
@@ -17,6 +18,7 @@ classes[].fields[].name "documentid"
classes[].fields[].type "longstring"
classes[].id 890647799
classes[].name "mysummary"
+classes[].omitsummaryfeatures false
classes[].fields[].name "my_pos"
classes[].fields[].type "jsonstring"
classes[].fields[].name "rankfeatures"
@@ -29,6 +31,7 @@ classes[].fields[].name "my_pos.distance"
classes[].fields[].type "integer"
classes[].id 1274088866
classes[].name "attributeprefetch"
+classes[].omitsummaryfeatures false
classes[].fields[].name "rankfeatures"
classes[].fields[].type "featuredata"
classes[].fields[].name "summaryfeatures"
diff --git a/config-model/src/test/derived/imported_struct_fields/summary.cfg b/config-model/src/test/derived/imported_struct_fields/summary.cfg
index 3a9bf4f5e0a..ab6c6853925 100644
--- a/config-model/src/test/derived/imported_struct_fields/summary.cfg
+++ b/config-model/src/test/derived/imported_struct_fields/summary.cfg
@@ -1,43 +1,47 @@
defaultsummaryid 1570252291
-classes[0].id 1570252291
-classes[0].name "default"
-classes[0].fields[0].name "parent_ref"
-classes[0].fields[0].type "longstring"
-classes[0].fields[1].name "rankfeatures"
-classes[0].fields[1].type "featuredata"
-classes[0].fields[2].name "summaryfeatures"
-classes[0].fields[2].type "featuredata"
-classes[0].fields[3].name "documentid"
-classes[0].fields[3].type "longstring"
-classes[1].id 2126652894
-classes[1].name "mysummary"
-classes[1].fields[0].name "documentid"
-classes[1].fields[0].type "longstring"
-classes[1].fields[1].name "my_elem_array"
-classes[1].fields[1].type "jsonstring"
-classes[1].fields[2].name "my_elem_map"
-classes[1].fields[2].type "jsonstring"
-classes[1].fields[3].name "my_str_int_map"
-classes[1].fields[3].type "jsonstring"
-classes[1].fields[4].name "rankfeatures"
-classes[1].fields[4].type "featuredata"
-classes[1].fields[5].name "summaryfeatures"
-classes[1].fields[5].type "featuredata"
-classes[2].id 1629947863
-classes[2].name "filtered"
-classes[2].fields[0].name "elem_array_filtered"
-classes[2].fields[0].type "jsonstring"
-classes[2].fields[1].name "elem_map_filtered"
-classes[2].fields[1].type "jsonstring"
-classes[2].fields[2].name "str_int_map_filtered"
-classes[2].fields[2].type "jsonstring"
-classes[2].fields[3].name "rankfeatures"
-classes[2].fields[3].type "featuredata"
-classes[2].fields[4].name "summaryfeatures"
-classes[2].fields[4].type "featuredata"
-classes[3].id 1274088866
-classes[3].name "attributeprefetch"
-classes[3].fields[0].name "rankfeatures"
-classes[3].fields[0].type "featuredata"
-classes[3].fields[1].name "summaryfeatures"
-classes[3].fields[1].type "featuredata" \ No newline at end of file
+classes[].id 1570252291
+classes[].name "default"
+classes[].omitsummaryfeatures false
+classes[].fields[].name "parent_ref"
+classes[].fields[].type "longstring"
+classes[].fields[].name "rankfeatures"
+classes[].fields[].type "featuredata"
+classes[].fields[].name "summaryfeatures"
+classes[].fields[].type "featuredata"
+classes[].fields[].name "documentid"
+classes[].fields[].type "longstring"
+classes[].id 2126652894
+classes[].name "mysummary"
+classes[].omitsummaryfeatures false
+classes[].fields[].name "documentid"
+classes[].fields[].type "longstring"
+classes[].fields[].name "my_elem_array"
+classes[].fields[].type "jsonstring"
+classes[].fields[].name "my_elem_map"
+classes[].fields[].type "jsonstring"
+classes[].fields[].name "my_str_int_map"
+classes[].fields[].type "jsonstring"
+classes[].fields[].name "rankfeatures"
+classes[].fields[].type "featuredata"
+classes[].fields[].name "summaryfeatures"
+classes[].fields[].type "featuredata"
+classes[].id 1629947863
+classes[].name "filtered"
+classes[].omitsummaryfeatures false
+classes[].fields[].name "elem_array_filtered"
+classes[].fields[].type "jsonstring"
+classes[].fields[].name "elem_map_filtered"
+classes[].fields[].type "jsonstring"
+classes[].fields[].name "str_int_map_filtered"
+classes[].fields[].type "jsonstring"
+classes[].fields[].name "rankfeatures"
+classes[].fields[].type "featuredata"
+classes[].fields[].name "summaryfeatures"
+classes[].fields[].type "featuredata"
+classes[].id 1274088866
+classes[].name "attributeprefetch"
+classes[].omitsummaryfeatures false
+classes[].fields[].name "rankfeatures"
+classes[].fields[].type "featuredata"
+classes[].fields[].name "summaryfeatures"
+classes[].fields[].type "featuredata"
diff --git a/config-model/src/test/derived/importedfields/summary.cfg b/config-model/src/test/derived/importedfields/summary.cfg
index f95949cfa62..74b5b44214e 100644
--- a/config-model/src/test/derived/importedfields/summary.cfg
+++ b/config-model/src/test/derived/importedfields/summary.cfg
@@ -1,6 +1,7 @@
defaultsummaryid 1294344677
classes[].id 1294344677
classes[].name "default"
+classes[].omitsummaryfeatures false
classes[].fields[].name "b_ref_with_summary"
classes[].fields[].type "longstring"
classes[].fields[].name "rankfeatures"
@@ -11,6 +12,7 @@ classes[].fields[].name "documentid"
classes[].fields[].type "longstring"
classes[].id 159551552
classes[].name "mysummary"
+classes[].omitsummaryfeatures false
classes[].fields[].name "a_ref"
classes[].fields[].type "longstring"
classes[].fields[].name "b_ref_with_summary"
@@ -33,6 +35,7 @@ classes[].fields[].name "summaryfeatures"
classes[].fields[].type "featuredata"
classes[].id 1274088866
classes[].name "attributeprefetch"
+classes[].omitsummaryfeatures false
classes[].fields[].name "rankfeatures"
classes[].fields[].type "featuredata"
classes[].fields[].name "summaryfeatures"
diff --git a/config-model/src/test/derived/indexswitches/summary.cfg b/config-model/src/test/derived/indexswitches/summary.cfg
index 5bdc8fcdef4..d04bc4eb167 100644
--- a/config-model/src/test/derived/indexswitches/summary.cfg
+++ b/config-model/src/test/derived/indexswitches/summary.cfg
@@ -1,6 +1,7 @@
defaultsummaryid 1698765342
classes[].id 1698765342
classes[].name "default"
+classes[].omitsummaryfeatures false
classes[].fields[].name "source"
classes[].fields[].type "longstring"
classes[].fields[].name "title"
diff --git a/config-model/src/test/derived/inheritance/summary.cfg b/config-model/src/test/derived/inheritance/summary.cfg
index dde71a1378c..dde9f95ecbe 100644
--- a/config-model/src/test/derived/inheritance/summary.cfg
+++ b/config-model/src/test/derived/inheritance/summary.cfg
@@ -1,6 +1,7 @@
defaultsummaryid 1797992819
classes[].id 1797992819
classes[].name "default"
+classes[].omitsummaryfeatures false
classes[].fields[].name "onlyfather"
classes[].fields[].type "longstring"
classes[].fields[].name "rankfeatures"
@@ -11,6 +12,7 @@ classes[].fields[].name "documentid"
classes[].fields[].type "longstring"
classes[].id 1608562186
classes[].name "attributeprefetch"
+classes[].omitsummaryfeatures false
classes[].fields[].name "onlygrandparent"
classes[].fields[].type "integer"
classes[].fields[].name "overridden"
diff --git a/config-model/src/test/derived/integerattributetostringindex/summary.cfg b/config-model/src/test/derived/integerattributetostringindex/summary.cfg
index 267585e3fda..d5eb316ff01 100644
--- a/config-model/src/test/derived/integerattributetostringindex/summary.cfg
+++ b/config-model/src/test/derived/integerattributetostringindex/summary.cfg
@@ -1,6 +1,7 @@
defaultsummaryid 1195656216
classes[].id 1195656216
classes[].name "default"
+classes[].omitsummaryfeatures false
classes[].fields[].name "attinx"
classes[].fields[].type "integer"
classes[].fields[].name "artist"
@@ -17,6 +18,7 @@ classes[].fields[].name "documentid"
classes[].fields[].type "longstring"
classes[].id 1706878063
classes[].name "attributeprefetch"
+classes[].omitsummaryfeatures false
classes[].fields[].name "attinx"
classes[].fields[].type "integer"
classes[].fields[].name "artist"
diff --git a/config-model/src/test/derived/map_attribute/summary.cfg b/config-model/src/test/derived/map_attribute/summary.cfg
index 24d6cab7697..b465bdfa541 100644
--- a/config-model/src/test/derived/map_attribute/summary.cfg
+++ b/config-model/src/test/derived/map_attribute/summary.cfg
@@ -1,6 +1,7 @@
defaultsummaryid 1376056200
classes[].id 1376056200
classes[].name "default"
+classes[].omitsummaryfeatures false
classes[].fields[].name "str_map"
classes[].fields[].type "jsonstring"
classes[].fields[].name "int_map"
diff --git a/config-model/src/test/derived/map_of_struct_attribute/summary.cfg b/config-model/src/test/derived/map_of_struct_attribute/summary.cfg
index f70025c8f02..67988dbf30e 100644
--- a/config-model/src/test/derived/map_of_struct_attribute/summary.cfg
+++ b/config-model/src/test/derived/map_of_struct_attribute/summary.cfg
@@ -1,6 +1,7 @@
defaultsummaryid 1131098132
classes[].id 1131098132
classes[].name "default"
+classes[].omitsummaryfeatures false
classes[].fields[].name "str_elem_map"
classes[].fields[].type "jsonstring"
classes[].fields[].name "int_elem_map"
diff --git a/config-model/src/test/derived/mlr/summary.cfg b/config-model/src/test/derived/mlr/summary.cfg
index cb5bf17df84..b6a53a9a1d9 100644
--- a/config-model/src/test/derived/mlr/summary.cfg
+++ b/config-model/src/test/derived/mlr/summary.cfg
@@ -1,6 +1,7 @@
defaultsummaryid 1868876861
classes[].id 1868876861
classes[].name "default"
+classes[].omitsummaryfeatures false
classes[].fields[].name "a"
classes[].fields[].type "longstring"
classes[].fields[].name "b"
@@ -13,6 +14,7 @@ classes[].fields[].name "documentid"
classes[].fields[].type "longstring"
classes[].id 1944325986
classes[].name "attributeprefetch"
+classes[].omitsummaryfeatures false
classes[].fields[].name "a"
classes[].fields[].type "longstring"
classes[].fields[].name "ranklog"
diff --git a/config-model/src/test/derived/music/summary.cfg b/config-model/src/test/derived/music/summary.cfg
index 3eca077dbc8..bc55727b407 100644
--- a/config-model/src/test/derived/music/summary.cfg
+++ b/config-model/src/test/derived/music/summary.cfg
@@ -1,6 +1,7 @@
defaultsummaryid 2086497905
classes[].id 2086497905
classes[].name "default"
+classes[].omitsummaryfeatures false
classes[].fields[].name "bgndata"
classes[].fields[].type "longstring"
classes[].fields[].name "sales"
@@ -79,6 +80,7 @@ classes[].fields[].name "documentid"
classes[].fields[].type "longstring"
classes[].id 2060710706
classes[].name "attributeprefetch"
+classes[].omitsummaryfeatures false
classes[].fields[].name "sales"
classes[].fields[].type "integer"
classes[].fields[].name "pto"
diff --git a/config-model/src/test/derived/newrank/summary.cfg b/config-model/src/test/derived/newrank/summary.cfg
index 7cd92c26e02..0b98b20c342 100644
--- a/config-model/src/test/derived/newrank/summary.cfg
+++ b/config-model/src/test/derived/newrank/summary.cfg
@@ -1,6 +1,7 @@
defaultsummaryid 912980235
classes[].id 912980235
classes[].name "default"
+classes[].omitsummaryfeatures false
classes[].fields[].name "bgndata"
classes[].fields[].type "longstring"
classes[].fields[].name "sales"
@@ -71,6 +72,7 @@ classes[].fields[].name "documentid"
classes[].fields[].type "longstring"
classes[].id 1606815285
classes[].name "attributeprefetch"
+classes[].omitsummaryfeatures false
classes[].fields[].name "sales"
classes[].fields[].type "integer"
classes[].fields[].name "pto"
diff --git a/config-model/src/test/derived/position_nosummary/summary.cfg b/config-model/src/test/derived/position_nosummary/summary.cfg
index fad012393ef..4222e88cc2f 100644
--- a/config-model/src/test/derived/position_nosummary/summary.cfg
+++ b/config-model/src/test/derived/position_nosummary/summary.cfg
@@ -1,6 +1,7 @@
defaultsummaryid 1727020212
classes[].id 1727020212
classes[].name "default"
+classes[].omitsummaryfeatures false
classes[].fields[].name "pos.position"
classes[].fields[].type "xmlstring"
classes[].fields[].name "pos.distance"
@@ -13,6 +14,7 @@ classes[].fields[].name "documentid"
classes[].fields[].type "longstring"
classes[].id 1530141163
classes[].name "attributeprefetch"
+classes[].omitsummaryfeatures false
classes[].fields[].name "pos_zcurve"
classes[].fields[].type "int64"
classes[].fields[].name "rankfeatures"
diff --git a/config-model/src/test/derived/position_summary/summary.cfg b/config-model/src/test/derived/position_summary/summary.cfg
index af801f43cc0..f54066d865e 100644
--- a/config-model/src/test/derived/position_summary/summary.cfg
+++ b/config-model/src/test/derived/position_summary/summary.cfg
@@ -1,6 +1,7 @@
defaultsummaryid 230670304
classes[].id 230670304
classes[].name "default"
+classes[].omitsummaryfeatures false
classes[].fields[].name "pos"
classes[].fields[].type "jsonstring"
classes[].fields[].name "pos.position"
@@ -15,6 +16,7 @@ classes[].fields[].name "documentid"
classes[].fields[].type "longstring"
classes[].id 1530141163
classes[].name "attributeprefetch"
+classes[].omitsummaryfeatures false
classes[].fields[].name "pos_zcurve"
classes[].fields[].type "int64"
classes[].fields[].name "rankfeatures"
diff --git a/config-model/src/test/derived/predicate_attribute/summary.cfg b/config-model/src/test/derived/predicate_attribute/summary.cfg
index 6e33bd4e567..9cc613107e0 100644
--- a/config-model/src/test/derived/predicate_attribute/summary.cfg
+++ b/config-model/src/test/derived/predicate_attribute/summary.cfg
@@ -1,6 +1,7 @@
defaultsummaryid 1391971216
classes[].id 1391971216
classes[].name "default"
+classes[].omitsummaryfeatures false
classes[].fields[].name "some_predicate_field"
classes[].fields[].type "string"
classes[].fields[].name "rankfeatures"
@@ -11,6 +12,7 @@ classes[].fields[].name "documentid"
classes[].fields[].type "longstring"
classes[].id 1274088866
classes[].name "attributeprefetch"
+classes[].omitsummaryfeatures false
classes[].fields[].name "rankfeatures"
classes[].fields[].type "featuredata"
classes[].fields[].name "summaryfeatures"
diff --git a/config-model/src/test/derived/rankexpression/summary.cfg b/config-model/src/test/derived/rankexpression/summary.cfg
index f8b56baf8f2..4f417a848a7 100644
--- a/config-model/src/test/derived/rankexpression/summary.cfg
+++ b/config-model/src/test/derived/rankexpression/summary.cfg
@@ -1,6 +1,7 @@
defaultsummaryid 1753207254
classes[].id 1753207254
classes[].name "default"
+classes[].omitsummaryfeatures false
classes[].fields[].name "artist"
classes[].fields[].type "longstring"
classes[].fields[].name "title"
@@ -17,6 +18,7 @@ classes[].fields[].name "documentid"
classes[].fields[].type "longstring"
classes[].id 1736696699
classes[].name "attributeprefetch"
+classes[].omitsummaryfeatures false
classes[].fields[].name "year"
classes[].fields[].type "integer"
classes[].fields[].name "foo1"
diff --git a/config-model/src/test/derived/ranktypes/summary.cfg b/config-model/src/test/derived/ranktypes/summary.cfg
index 9644eb878ea..49b668e9edf 100644
--- a/config-model/src/test/derived/ranktypes/summary.cfg
+++ b/config-model/src/test/derived/ranktypes/summary.cfg
@@ -1,6 +1,7 @@
defaultsummaryid 1567556360
classes[].id 1567556360
classes[].name "default"
+classes[].omitsummaryfeatures false
classes[].fields[].name "title"
classes[].fields[].type "longstring"
classes[].fields[].name "descr"
diff --git a/config-model/src/test/derived/reference_fields/summary.cfg b/config-model/src/test/derived/reference_fields/summary.cfg
index 49037473d88..410bccff7b3 100644
--- a/config-model/src/test/derived/reference_fields/summary.cfg
+++ b/config-model/src/test/derived/reference_fields/summary.cfg
@@ -1,6 +1,7 @@
defaultsummaryid 1987541865
classes[].id 1987541865
classes[].name "default"
+classes[].omitsummaryfeatures false
classes[].fields[].name "campaign_ref"
classes[].fields[].type "longstring"
classes[].fields[].name "rankfeatures"
@@ -11,6 +12,7 @@ classes[].fields[].name "documentid"
classes[].fields[].type "longstring"
classes[].id 428144659
classes[].name "explicit_summary"
+classes[].omitsummaryfeatures false
classes[].fields[].name "yet_another_ref"
classes[].fields[].type "longstring"
classes[].fields[].name "rankfeatures"
@@ -19,6 +21,7 @@ classes[].fields[].name "summaryfeatures"
classes[].fields[].type "featuredata"
classes[].id 1274088866
classes[].name "attributeprefetch"
+classes[].omitsummaryfeatures false
classes[].fields[].name "rankfeatures"
classes[].fields[].type "featuredata"
classes[].fields[].name "summaryfeatures"
diff --git a/config-model/src/test/derived/streamingstruct/summary.cfg b/config-model/src/test/derived/streamingstruct/summary.cfg
index 28f19e6fe25..655499a88be 100644
--- a/config-model/src/test/derived/streamingstruct/summary.cfg
+++ b/config-model/src/test/derived/streamingstruct/summary.cfg
@@ -1,6 +1,7 @@
defaultsummaryid 569269436
classes[].id 569269436
classes[].name "default"
+classes[].omitsummaryfeatures false
classes[].fields[].name "coupleof"
classes[].fields[].type "longstring"
classes[].fields[].name "anothersummaryfield"
@@ -41,6 +42,7 @@ classes[].fields[].name "documentid"
classes[].fields[].type "longstring"
classes[].id 109252281
classes[].name "summ"
+classes[].omitsummaryfeatures false
classes[].fields[].name "snippet"
classes[].fields[].type "longstring"
classes[].fields[].name "snippet2"
diff --git a/config-model/src/test/derived/streamingstructdefault/summary.cfg b/config-model/src/test/derived/streamingstructdefault/summary.cfg
index caa44931c6a..a52b34925dc 100644
--- a/config-model/src/test/derived/streamingstructdefault/summary.cfg
+++ b/config-model/src/test/derived/streamingstructdefault/summary.cfg
@@ -1,6 +1,7 @@
defaultsummaryid 718801936
classes[].id 718801936
classes[].name "default"
+classes[].omitsummaryfeatures false
classes[].fields[].name "sum1"
classes[].fields[].type "longstring"
classes[].fields[].name "f1"
diff --git a/config-model/src/test/derived/tensor/summary.cfg b/config-model/src/test/derived/tensor/summary.cfg
index fb32eacbb4c..355cba0e561 100644
--- a/config-model/src/test/derived/tensor/summary.cfg
+++ b/config-model/src/test/derived/tensor/summary.cfg
@@ -1,6 +1,7 @@
defaultsummaryid 898020074
classes[].id 898020074
classes[].name "default"
+classes[].omitsummaryfeatures false
classes[].fields[].name "f1"
classes[].fields[].type "tensor"
classes[].fields[].name "f3"
@@ -17,6 +18,7 @@ classes[].fields[].name "documentid"
classes[].fields[].type "longstring"
classes[].id 1476352352
classes[].name "attributeprefetch"
+classes[].omitsummaryfeatures false
classes[].fields[].name "f2"
classes[].fields[].type "tensor"
classes[].fields[].name "f3"
diff --git a/config-model/src/test/derived/tokenization/ilscripts.cfg b/config-model/src/test/derived/tokenization/ilscripts.cfg
new file mode 100644
index 00000000000..ead74110db3
--- /dev/null
+++ b/config-model/src/test/derived/tokenization/ilscripts.cfg
@@ -0,0 +1,9 @@
+maxtermoccurrences 100
+fieldmatchmaxlength 1000000
+ilscript[].doctype "tokenization"
+ilscript[].docfield[] "text"
+ilscript[].docfield[] "text_array"
+ilscript[].content[] "clear_state | guard { input text_array | for_each { lowercase } | for_each { normalize } | for_each { tokenize normalize stem:\"BEST\" } | index text_array_derived | summary text_array_derived; }"
+ilscript[].content[] "clear_state | guard { input text | normalize | tokenize normalize stem:\"BEST\" | index text_derived | summary text_derived; }"
+ilscript[].content[] "clear_state | guard { input text | tokenize normalize stem:\"BEST\" | index text | summary text; }"
+ilscript[].content[] "clear_state | guard { input text_array | for_each { tokenize normalize stem:\"BEST\" } | index text_array | summary text_array; }" \ No newline at end of file
diff --git a/config-model/src/test/derived/tokenization/tokenization.sd b/config-model/src/test/derived/tokenization/tokenization.sd
new file mode 100644
index 00000000000..4510a574d60
--- /dev/null
+++ b/config-model/src/test/derived/tokenization/tokenization.sd
@@ -0,0 +1,23 @@
+schema tokenization {
+
+ document tokenization {
+
+ field text type string {
+ indexing: index | summary
+ }
+
+ field text_array type array<string> {
+ indexing: index | summary
+ }
+
+ }
+
+ field text_derived type string {
+ indexing: input text | normalize | index | summary
+ }
+
+ field text_array_derived type array<string> {
+ indexing: input text_array | for_each { lowercase } | for_each { normalize } | index | summary
+ }
+
+} \ No newline at end of file
diff --git a/config-model/src/test/derived/types/summary.cfg b/config-model/src/test/derived/types/summary.cfg
index e5485a24c8c..e0e67a5669d 100644
--- a/config-model/src/test/derived/types/summary.cfg
+++ b/config-model/src/test/derived/types/summary.cfg
@@ -1,6 +1,7 @@
defaultsummaryid 1131946680
classes[].id 1131946680
classes[].name "default"
+classes[].omitsummaryfeatures false
classes[].fields[].name "abyte"
classes[].fields[].type "byte"
classes[].fields[].name "along"
@@ -25,6 +26,7 @@ classes[].fields[].name "documentid"
classes[].fields[].type "longstring"
classes[].id 1027812395
classes[].name "attributeprefetch"
+classes[].omitsummaryfeatures false
classes[].fields[].name "other"
classes[].fields[].type "int64"
classes[].fields[].name "abyte"
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/SummaryTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/SummaryTestCase.java
index bfc738a4f87..ef7da4f23d0 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/SummaryTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/SummaryTestCase.java
@@ -6,6 +6,7 @@ import com.yahoo.searchdefinition.Search;
import com.yahoo.searchdefinition.SearchBuilder;
import com.yahoo.searchdefinition.SchemaTestCase;
import com.yahoo.searchdefinition.parser.ParseException;
+import com.yahoo.vespa.config.search.SummaryConfig;
import org.junit.Test;
import java.io.IOException;
@@ -14,6 +15,7 @@ import java.util.Iterator;
import static com.yahoo.config.model.test.TestUtil.joinLines;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
/**
* Tests summary extraction
@@ -151,4 +153,30 @@ public class SummaryTestCase extends SchemaTestCase {
return builder.getSearch("ad");
}
+ @Test
+ public void omit_summary_features_specified_for_document_summary() throws ParseException {
+ String sd = joinLines(
+ "schema test {",
+ " document test {",
+ " field foo type string { indexing: summary }",
+ " }",
+ " document-summary bar {",
+ " summary foo type string {}",
+ " omit-summary-features",
+ " }",
+ " document-summary baz {",
+ " summary foo type string {}",
+ " }",
+ "}");
+ var search = SearchBuilder.createFromString(sd).getSearch();
+ assertOmitSummaryFeatures(true, search, "bar");
+ assertOmitSummaryFeatures(false, search, "baz");
+ }
+
+ private void assertOmitSummaryFeatures(boolean expected, Search search, String summaryName) {
+ var summary = new SummaryClass(search, search.getSummary(summaryName), new BaseDeployLogger());
+ var config = new SummaryConfig.Classes(summary.getSummaryClassConfig());
+ assertEquals(expected, config.omitsummaryfeatures());
+ }
+
}
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/TokenizationTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/TokenizationTestCase.java
new file mode 100755
index 00000000000..6fe367ef6d1
--- /dev/null
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/TokenizationTestCase.java
@@ -0,0 +1,19 @@
+// Copyright Verizon media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.derived;
+
+import com.yahoo.searchdefinition.parser.ParseException;
+import org.junit.Test;
+
+import java.io.IOException;
+
+/**
+ * @author bratseh
+ */
+public class TokenizationTestCase extends AbstractExportingTestCase {
+
+ @Test
+ public void testTokenizationScripts() throws IOException, ParseException {
+ assertCorrectDeriving("tokenization");
+ }
+
+}
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/configserver/ConfigserverClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/configserver/ConfigserverClusterTest.java
index 2b55d7a3948..2144c3c9a66 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/container/configserver/ConfigserverClusterTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/container/configserver/ConfigserverClusterTest.java
@@ -9,6 +9,7 @@ import com.yahoo.config.model.deploy.DeployState;
import com.yahoo.config.model.producer.AbstractConfigProducerRoot;
import com.yahoo.config.model.test.MockRoot;
import com.yahoo.container.StatisticsConfig;
+import com.yahoo.container.di.config.PlatformBundlesConfig;
import com.yahoo.container.jdisc.config.HealthMonitorConfig;
import com.yahoo.net.HostName;
import com.yahoo.text.XML;
@@ -16,6 +17,7 @@ import com.yahoo.vespa.defaults.Defaults;
import com.yahoo.vespa.model.HostResource;
import com.yahoo.vespa.model.container.Container;
import com.yahoo.vespa.model.container.ContainerModel;
+import com.yahoo.vespa.model.container.ContainerModelEvaluation;
import com.yahoo.vespa.model.container.configserver.option.CloudConfigOptions;
import com.yahoo.vespa.model.container.xml.ConfigServerContainerModelBuilder;
import org.junit.Test;
@@ -27,6 +29,8 @@ import java.util.Optional;
import java.util.function.Function;
import java.util.stream.Collectors;
+import static org.hamcrest.CoreMatchers.not;
+import static org.hamcrest.Matchers.hasItem;
import static org.hamcrest.Matchers.is;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
@@ -124,6 +128,14 @@ public class ConfigserverClusterTest {
assertTrue(config.zookeeperLocalhostAffinity());
}
+ @Test
+ public void model_evaluation_bundles_are_not_installed_via_config() {
+ // These bundles must be pre-installed because they are used by config-model.
+ PlatformBundlesConfig config = getConfig(PlatformBundlesConfig.class);
+ assertThat(config.bundlePaths(), not(hasItem(ContainerModelEvaluation.MODEL_INTEGRATION_BUNDLE_FILE.toString())));
+ assertThat(config.bundlePaths(), not(hasItem(ContainerModelEvaluation.MODEL_EVALUATION_BUNDLE_FILE.toString())));
+ }
+
@SuppressWarnings("varargs")
private static <T> void assertZookeeperServerProperty(
List<ZookeeperServerConfig.Server> zkServers, Function<ZookeeperServerConfig.Server, T> propertyMapper, T... expectedProperties) {
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java
index 543318f9224..ccee21c87dc 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java
@@ -27,6 +27,7 @@ import com.yahoo.container.ComponentsConfig;
import com.yahoo.container.QrConfig;
import com.yahoo.container.core.ChainsConfig;
import com.yahoo.container.core.VipStatusConfig;
+import com.yahoo.container.di.config.PlatformBundlesConfig;
import com.yahoo.container.handler.VipStatusHandler;
import com.yahoo.container.handler.metrics.MetricsV2Handler;
import com.yahoo.container.handler.observability.ApplicationStatusHandler;
@@ -52,6 +53,7 @@ import com.yahoo.vespa.model.VespaModel;
import com.yahoo.vespa.model.container.ApplicationContainer;
import com.yahoo.vespa.model.container.ApplicationContainerCluster;
import com.yahoo.vespa.model.container.ContainerCluster;
+import com.yahoo.vespa.model.container.ContainerModelEvaluation;
import com.yahoo.vespa.model.container.SecretStore;
import com.yahoo.vespa.model.container.component.Component;
import com.yahoo.vespa.model.container.http.ConnectorFactory;
@@ -124,6 +126,14 @@ public class ContainerModelBuilderTest extends ContainerModelBuilderTestBase {
public TemporaryFolder applicationFolder = new TemporaryFolder();
@Test
+ public void model_evaluation_bundles_are_deployed() {
+ createBasicContainerModel();
+ PlatformBundlesConfig config = root.getConfig(PlatformBundlesConfig.class, "default");
+ assertThat(config.bundlePaths(), hasItem(ContainerModelEvaluation.MODEL_EVALUATION_BUNDLE_FILE.toString()));
+ assertThat(config.bundlePaths(), hasItem(ContainerModelEvaluation.MODEL_INTEGRATION_BUNDLE_FILE.toString()));
+ }
+
+ @Test
public void deprecated_jdisc_tag_is_allowed() {
Element clusterElem = DomBuilderTest.parse(
"<jdisc version='1.0'>",
@@ -244,10 +254,7 @@ public class ContainerModelBuilderTest extends ContainerModelBuilderTestBase {
@Test
public void verify_bindings_for_builtin_handlers() {
- Element clusterElem = DomBuilderTest.parse(
- "<container id='default' version='1.0' />"
- );
- createModel(root, clusterElem);
+ createBasicContainerModel();
JdiscBindingsConfig config = root.getConfig(JdiscBindingsConfig.class, "default/container.0");
JdiscBindingsConfig.Handlers defaultRootHandler = config.handlers(BindingsOverviewHandler.class.getName());
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTestBase.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTestBase.java
index 9e02572737e..7034176da14 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTestBase.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTestBase.java
@@ -4,6 +4,7 @@ package com.yahoo.vespa.model.container.xml;
import com.yahoo.collections.Pair;
import com.yahoo.component.ComponentId;
import com.yahoo.config.application.api.DeployLogger;
+import com.yahoo.config.model.builder.xml.test.DomBuilderTest;
import com.yahoo.config.model.deploy.DeployState;
import com.yahoo.config.model.test.MockRoot;
import com.yahoo.container.ComponentsConfig;
@@ -51,6 +52,11 @@ public abstract class ContainerModelBuilderTestBase {
protected MockRoot root;
+ protected void createBasicContainerModel() {
+ Element clusterElem = DomBuilderTest.parse("<container id='default' version='1.0' />");
+ createModel(root, clusterElem);
+ }
+
public static void createModel(MockRoot root, DeployState deployState, VespaModel vespaModel, Element... containerElems) {
for (Element containerElem : containerElems) {
ContainerModel model = new ContainerModelBuilder(false, ContainerModelBuilder.Networking.enable)
diff --git a/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/ConfigProxyRpcServerTest.java b/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/ConfigProxyRpcServerTest.java
index 29bd38ea891..628e1c013e6 100644
--- a/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/ConfigProxyRpcServerTest.java
+++ b/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/ConfigProxyRpcServerTest.java
@@ -11,8 +11,8 @@ import com.yahoo.jrt.Supervisor;
import com.yahoo.jrt.Target;
import com.yahoo.jrt.Transport;
import com.yahoo.vespa.config.RawConfig;
-import org.junit.After;
-import org.junit.Before;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
@@ -32,24 +32,29 @@ public class ConfigProxyRpcServerTest {
private static final String hostname = "localhost";
private static final int port = 12345;
private static final String configSourceAddress = "tcp/" + hostname + ":" + port;
- private TestServer server;
- private TestClient client;
+ private static TestServer server;
+ private static TestClient client;
@Rule
public TemporaryFolder temporaryFolder = new TemporaryFolder();
- @Before
- public void setup() throws ListenFailedException {
+ @BeforeClass
+ public static void setup() throws ListenFailedException {
server = new TestServer();
client = new TestClient(server.listenPort());
}
- @After
- public void teardown() {
+ @AfterClass
+ public static void teardown() {
client.close();
server.close();
}
+ private static void reset() throws ListenFailedException {
+ teardown();
+ setup();
+ }
+
@Test
public void basic() {
ProxyServer proxy = createTestServer(new MockConfigSource());
@@ -75,7 +80,9 @@ public class ConfigProxyRpcServerTest {
* Tests listCachedConfig RPC command
*/
@Test
- public void testRpcMethodListCachedConfig() {
+ public void testRpcMethodListCachedConfig() throws ListenFailedException {
+ reset();
+
Request req = new Request("listCachedConfig");
client.invoke(req);
@@ -129,7 +136,9 @@ public class ConfigProxyRpcServerTest {
* Tests printStatistics RPC command
*/
@Test
- public void testRpcMethodListSourceConnections() {
+ public void testRpcMethodListSourceConnections() throws ListenFailedException {
+ reset();
+
Request req = new Request("listSourceConnections");
client.invoke(req);
@@ -218,7 +227,9 @@ public class ConfigProxyRpcServerTest {
* Tests updateSources RPC command
*/
@Test
- public void testRpcMethodUpdateSources() {
+ public void testRpcMethodUpdateSources() throws ListenFailedException {
+ reset();
+
Request req = new Request("updateSources");
String spec1 = "tcp/a:19070";
String spec2 = "tcp/b:19070";
diff --git a/config/pom.xml b/config/pom.xml
index 6e4e26ed0f1..8355587c10b 100755
--- a/config/pom.xml
+++ b/config/pom.xml
@@ -15,6 +15,12 @@
<dependencies>
<!-- provided scope -->
<dependency>
+ <groupId>com.yahoo.vespa</groupId>
+ <artifactId>container-apache-http-client-bundle</artifactId>
+ <version>${project.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
<scope>provided</scope>
diff --git a/configdefinitions/src/vespa/summary.def b/configdefinitions/src/vespa/summary.def
index 20ca6b10450..26b0e4a4a37 100644
--- a/configdefinitions/src/vespa/summary.def
+++ b/configdefinitions/src/vespa/summary.def
@@ -4,5 +4,6 @@ namespace=vespa.config.search
defaultsummaryid int default=-1
classes[].id int
classes[].name string
+classes[].omitsummaryfeatures bool default=false
classes[].fields[].name string
classes[].fields[].type string
diff --git a/configserver-client/pom.xml b/configserver-client/pom.xml
index 0a29ba003f4..39005c9ccab 100644
--- a/configserver-client/pom.xml
+++ b/configserver-client/pom.xml
@@ -30,6 +30,12 @@
</dependency>
<dependency>
<groupId>com.yahoo.vespa</groupId>
+ <artifactId>container-apache-http-client-bundle</artifactId>
+ <version>${project.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>com.yahoo.vespa</groupId>
<artifactId>security-utils</artifactId>
<version>${project.version}</version>
<scope>provided</scope>
diff --git a/configserver/pom.xml b/configserver/pom.xml
index 3b7fef085b1..a237d7e00ad 100644
--- a/configserver/pom.xml
+++ b/configserver/pom.xml
@@ -142,6 +142,12 @@
<scope>provided</scope>
</dependency>
<dependency>
+ <groupId>com.yahoo.vespa</groupId>
+ <artifactId>container-apache-http-client-bundle</artifactId>
+ <version>${project.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
<!-- To get all necessary test deps. -->
<groupId>com.yahoo.vespa</groupId>
<artifactId>container-test</artifactId>
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java
index 2d5940ebba2..56fd6a64305 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java
@@ -56,9 +56,9 @@ import com.yahoo.vespa.config.server.http.LogRetriever;
import com.yahoo.vespa.config.server.http.SecretStoreValidator;
import com.yahoo.vespa.config.server.http.SimpleHttpFetcher;
import com.yahoo.vespa.config.server.http.TesterClient;
-import com.yahoo.vespa.config.server.http.v2.DeploymentMetricsResponse;
+import com.yahoo.vespa.config.server.http.v2.response.DeploymentMetricsResponse;
import com.yahoo.vespa.config.server.http.v2.PrepareResult;
-import com.yahoo.vespa.config.server.http.v2.ProtonMetricsResponse;
+import com.yahoo.vespa.config.server.http.v2.response.ProtonMetricsResponse;
import com.yahoo.vespa.config.server.metrics.DeploymentMetricsRetriever;
import com.yahoo.vespa.config.server.metrics.ProtonMetricsRetriever;
import com.yahoo.vespa.config.server.provision.HostProvisionerProvider;
@@ -547,15 +547,27 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye
}
}
- public HttpResponse clusterControllerStatusPage(ApplicationId applicationId, String hostName, String pathSuffix) {
+ public HttpResponse serviceStatusPage(ApplicationId applicationId, String hostName, String serviceName, String pathSuffix) {
// WARNING: pathSuffix may be given by the external user. Make sure no security issues arise...
// We should be OK here, because at most, pathSuffix may change the parent path, but cannot otherwise
// change the hostname and port. Exposing other paths on the cluster controller should be fine.
// TODO: It would be nice to have a simple check to verify pathSuffix doesn't contain /../ components.
- String relativePath = "clustercontroller-status/" + pathSuffix;
+ String pathPrefix;
+ switch (serviceName) {
+ case "container-clustercontroller": {
+ pathPrefix = "clustercontroller-status/v1/";
+ break;
+ }
+ case "distributor":
+ case "storagenode": {
+ pathPrefix = "";
+ break;
+ }
+ default:
+ throw new NotFoundException("No status page for service: " + serviceName);
+ }
- return httpProxy.get(getApplication(applicationId), hostName,
- CLUSTERCONTROLLER_CONTAINER.serviceName, relativePath);
+ return httpProxy.get(getApplication(applicationId), hostName, serviceName, pathPrefix + pathSuffix);
}
public Map<String, ClusterReindexing> getClusterReindexingStatus(ApplicationId applicationId) {
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpConfigRequest.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpConfigRequest.java
index ff70ce1e1b2..cdbd7378151 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpConfigRequest.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpConfigRequest.java
@@ -18,8 +18,8 @@ import com.yahoo.vespa.config.protocol.DefContent;
import com.yahoo.vespa.config.protocol.VespaVersion;
import com.yahoo.vespa.config.server.RequestHandler;
import com.yahoo.config.provision.ApplicationId;
-import com.yahoo.vespa.config.server.http.v2.HttpConfigRequests;
-import com.yahoo.vespa.config.server.http.v2.TenantRequest;
+import com.yahoo.vespa.config.server.http.v2.request.HttpConfigRequests;
+import com.yahoo.vespa.config.server.http.v2.request.TenantRequest;
import com.yahoo.vespa.config.util.ConfigUtils;
/**
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationApiHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationApiHandler.java
index dfbce72d4ba..7e8940d28b3 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationApiHandler.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationApiHandler.java
@@ -14,9 +14,9 @@ import com.yahoo.vespa.config.server.application.CompressedApplicationInputStrea
import com.yahoo.vespa.config.server.http.BadRequestException;
import com.yahoo.vespa.config.server.http.SessionHandler;
import com.yahoo.vespa.config.server.http.Utils;
+import com.yahoo.vespa.config.server.http.v2.response.SessionPrepareAndActivateResponse;
import com.yahoo.vespa.config.server.session.PrepareParams;
import com.yahoo.vespa.config.server.tenant.TenantRepository;
-import com.yahoo.vespa.model.content.Content;
import org.apache.hc.core5.http.ContentType;
import org.eclipse.jetty.http.MultiPartFormInputStream;
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java
index 3634a6825a3..ec7b6616bb6 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java
@@ -6,47 +6,42 @@ import com.yahoo.component.Version;
import com.yahoo.config.application.api.ApplicationFile;
import com.yahoo.config.model.api.Model;
import com.yahoo.config.provision.ApplicationId;
-import com.yahoo.config.provision.ApplicationName;
import com.yahoo.config.provision.HostFilter;
-import com.yahoo.config.provision.TenantName;
+import com.yahoo.config.provision.InstanceName;
import com.yahoo.config.provision.Zone;
import com.yahoo.container.jdisc.HttpRequest;
import com.yahoo.container.jdisc.HttpResponse;
import com.yahoo.io.IOUtils;
import com.yahoo.jdisc.Response;
-import com.yahoo.jdisc.application.BindingMatch;
-import com.yahoo.jdisc.application.UriPattern;
-import com.yahoo.slime.Cursor;
+import com.yahoo.restapi.ErrorResponse;
+import com.yahoo.restapi.MessageResponse;
+import com.yahoo.restapi.Path;
import com.yahoo.slime.SlimeUtils;
import com.yahoo.text.StringUtilities;
import com.yahoo.vespa.config.server.ApplicationRepository;
-import com.yahoo.vespa.config.server.application.ApplicationReindexing;
-import com.yahoo.vespa.config.server.application.ClusterReindexing;
import com.yahoo.vespa.config.server.http.ContentHandler;
import com.yahoo.vespa.config.server.http.ContentRequest;
-import com.yahoo.vespa.config.server.http.HttpErrorResponse;
import com.yahoo.vespa.config.server.http.HttpHandler;
-import com.yahoo.vespa.config.server.http.JSONResponse;
import com.yahoo.vespa.config.server.http.NotFoundException;
+import com.yahoo.vespa.config.server.http.v2.request.ApplicationContentRequest;
+import com.yahoo.vespa.config.server.http.v2.response.ApplicationSuspendedResponse;
+import com.yahoo.vespa.config.server.http.v2.response.DeleteApplicationResponse;
+import com.yahoo.vespa.config.server.http.v2.response.GetApplicationResponse;
+import com.yahoo.vespa.config.server.http.v2.response.QuotaUsageResponse;
+import com.yahoo.vespa.config.server.http.v2.response.ReindexingResponse;
import com.yahoo.vespa.config.server.tenant.Tenant;
import java.io.IOException;
-import java.net.URLDecoder;
import java.time.Duration;
import java.time.Instant;
-import java.util.List;
import java.util.Map;
-import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import java.util.StringJoiner;
import java.util.TreeMap;
import java.util.TreeSet;
-import java.util.stream.Stream;
import static com.yahoo.yolean.Exceptions.uncheck;
-import static java.nio.charset.StandardCharsets.UTF_8;
-import static java.util.stream.Collectors.toList;
/**
* Operations on applications (delete, wait for config convergence, restart, application content etc.)
@@ -55,28 +50,6 @@ import static java.util.stream.Collectors.toList;
*/
public class ApplicationHandler extends HttpHandler {
- private static final List<UriPattern> URI_PATTERNS = Stream.of(
- "http://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*/content/*",
- "http://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*/filedistributionstatus",
- "http://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*/restart",
- "http://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*/reindex",
- "http://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*/reindexing",
- "http://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*/suspended",
- "http://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*/serviceconverge",
- "http://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*/serviceconverge/*",
- "http://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*/clustercontroller/*/status/*",
- "http://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*/metrics/*",
- "http://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*/metrics/*",
- "http://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*/logs",
- "http://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*/validate-secret-store",
- "http://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*/tester/*/*",
- "http://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*/tester/*",
- "http://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*/quota",
- "http://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*",
- "http://*/application/v2/tenant/*/application/*")
- .map(UriPattern::new)
- .collect(toList());
-
private final Zone zone;
private final ApplicationRepository applicationRepository;
@@ -90,152 +63,135 @@ public class ApplicationHandler extends HttpHandler {
}
@Override
- public HttpResponse handleDELETE(HttpRequest request) {
- ApplicationId applicationId = getApplicationIdFromRequest(request);
-
- if (isReindexingRequest(request)) {
- applicationRepository.modifyReindexing(applicationId, reindexing -> reindexing.enabled(false));
- return createMessageResponse("Reindexing disabled");
- }
+ public HttpResponse handleGET(HttpRequest request) {
+ Path path = new Path(request.getUri());
+
+ if (path.matches("/application/v2/tenant/{tenant}/application/{application}")) return getApplicationResponse(ApplicationId.from(path.get("tenant"), path.get("application"), InstanceName.defaultName().value()));
+ if (path.matches("/application/v2/tenant/{tenant}/application/{application}/environment/{ignore}/region/{ignore}/instance/{instance}")) return getApplicationResponse(applicationId(path));
+ if (path.matches("/application/v2/tenant/{tenant}/application/{application}/environment/{ignore}/region/{ignore}/instance/{instance}/content/{*}")) return content(applicationId(path), path.getRest(), request);
+ if (path.matches("/application/v2/tenant/{tenant}/application/{application}/environment/{ignore}/region/{ignore}/instance/{instance}/filedistributionstatus")) return filedistributionStatus(applicationId(path), request);
+ if (path.matches("/application/v2/tenant/{tenant}/application/{application}/environment/{ignore}/region/{ignore}/instance/{instance}/logs")) return logs(applicationId(path), request);
+ if (path.matches("/application/v2/tenant/{tenant}/application/{application}/environment/{ignore}/region/{ignore}/instance/{instance}/metrics/deployment")) return deploymentMetrics(applicationId(path));
+ if (path.matches("/application/v2/tenant/{tenant}/application/{application}/environment/{ignore}/region/{ignore}/instance/{instance}/metrics/proton")) return protonMetrics(applicationId(path));
+ if (path.matches("/application/v2/tenant/{tenant}/application/{application}/environment/{ignore}/region/{ignore}/instance/{instance}/reindexing")) return getReindexingStatus(applicationId(path));
+ if (path.matches("/application/v2/tenant/{tenant}/application/{application}/environment/{ignore}/region/{ignore}/instance/{instance}/service/{service}/{hostname}/status/{*}")) return serviceStatusPage(applicationId(path), path.get("service"), path.get("hostname"), path.getRest());
+ if (path.matches("/application/v2/tenant/{tenant}/application/{application}/environment/{ignore}/region/{ignore}/instance/{instance}/serviceconverge")) return listServiceConverge(applicationId(path), request);
+ if (path.matches("/application/v2/tenant/{tenant}/application/{application}/environment/{ignore}/region/{ignore}/instance/{instance}/serviceconverge/{hostAndPort}")) return checkServiceConverge(applicationId(path), path.get("hostAndPort"), request);
+ if (path.matches("/application/v2/tenant/{tenant}/application/{application}/environment/{ignore}/region/{ignore}/instance/{instance}/suspended")) return isSuspended(applicationId(path));
+ if (path.matches("/application/v2/tenant/{tenant}/application/{application}/environment/{ignore}/region/{ignore}/instance/{instance}/tester/{command}")) return testerRequest(applicationId(path), path.get("command"), request);
+ if (path.matches("/application/v2/tenant/{tenant}/application/{application}/environment/{ignore}/region/{ignore}/instance/{instance}/quota")) return quotaUsage(applicationId(path));
+ return ErrorResponse.notFoundError("Nothing at " + path);
+ }
- if (applicationRepository.delete(applicationId))
- return new DeleteApplicationResponse(Response.Status.OK, applicationId);
+ @Override
+ public HttpResponse handlePOST(HttpRequest request) {
+ Path path = new Path(request.getUri());
- return HttpErrorResponse.notFoundError("Unable to delete " + applicationId.toFullString() + ": Not found");
+ if (path.matches("/application/v2/tenant/{tenant}/application/{application}/environment/{ignore}/region/{ignore}/instance/{instance}/reindex")) return triggerReindexing(applicationId(path), request);
+ if (path.matches("/application/v2/tenant/{tenant}/application/{application}/environment/{ignore}/region/{ignore}/instance/{instance}/reindexing")) return enableReindexing(applicationId(path));
+ if (path.matches("/application/v2/tenant/{tenant}/application/{application}/environment/{ignore}/region/{ignore}/instance/{instance}/restart")) return restart(applicationId(path), request);
+ if (path.matches("/application/v2/tenant/{tenant}/application/{application}/environment/{ignore}/region/{ignore}/instance/{instance}/tester/run/{suite}")) return testerStartTests(applicationId(path), path.get("suite"), request);
+ if (path.matches("/application/v2/tenant/{tenant}/application/{application}/environment/{ignore}/region/{ignore}/instance/{instance}/validate-secret-store")) return validateSecretStore(applicationId(path), request);
+ return ErrorResponse.notFoundError("Nothing at " + path);
}
@Override
- public HttpResponse handleGET(HttpRequest request) {
- ApplicationId applicationId = getApplicationIdFromRequest(request);
- Duration timeout = HttpHandler.getRequestTimeout(request, Duration.ofSeconds(5));
-
- if (isServiceConvergeRequest(request)) {
- // Expects both hostname and port in the request (hostname:port)
- String hostAndPort = getHostNameFromRequest(request);
- return applicationRepository.checkServiceForConfigConvergence(applicationId, hostAndPort, request.getUri(),
- timeout, getVespaVersionFromRequest(request));
- }
+ public HttpResponse handleDELETE(HttpRequest request) {
+ Path path = new Path(request.getUri());
- if (isClusterControllerStatusRequest(request)) {
- String hostName = getHostNameFromRequest(request);
- String pathSuffix = URLDecoder.decode(getPathSuffix(request), UTF_8);
- return applicationRepository.clusterControllerStatusPage(applicationId, hostName, pathSuffix);
- }
+ if (path.matches("/application/v2/tenant/{tenant}/application/{application}")) return deleteApplication(ApplicationId.from(path.get("tenant"), path.get("application"), InstanceName.defaultName().value()));
+ if (path.matches("/application/v2/tenant/{tenant}/application/{application}/environment/{ignore}/region/{ignore}/instance/{instance}")) return deleteApplication(applicationId(path));
+ if (path.matches("/application/v2/tenant/{tenant}/application/{application}/environment/{ignore}/region/{ignore}/instance/{instance}/reindexing")) return disableReindexing(applicationId(path));
+ return ErrorResponse.notFoundError("Nothing at " + path);
+ }
- if (isReindexingRequest(request)) {
- return getReindexingStatus(applicationId);
- }
+ private HttpResponse listServiceConverge(ApplicationId applicationId, HttpRequest request) {
+ return applicationRepository.servicesToCheckForConfigConvergence(applicationId, request.getUri(),
+ getTimeoutFromRequest(request), getVespaVersionFromRequest(request));
+ }
- if (isContentRequest(request)) {
- long sessionId = applicationRepository.getSessionIdForApplication(applicationId);
- String contentPath = getBindingMatch(request).group(7);
- ApplicationFile applicationFile =
- applicationRepository.getApplicationFileFromSession(applicationId.tenant(),
- sessionId,
- contentPath,
- ContentRequest.getApplicationFileMode(request.getMethod()));
- ApplicationContentRequest contentRequest = new ApplicationContentRequest(request,
- sessionId,
- applicationId,
- zone,
- contentPath,
- applicationFile);
- return new ContentHandler().get(contentRequest);
- }
+ private HttpResponse checkServiceConverge(ApplicationId applicationId, String hostAndPort, HttpRequest request) {
+ return applicationRepository.checkServiceForConfigConvergence(applicationId, hostAndPort, request.getUri(),
+ getTimeoutFromRequest(request), getVespaVersionFromRequest(request));
+ }
- if (isServiceConvergeListRequest(request)) {
- return applicationRepository.servicesToCheckForConfigConvergence(applicationId, request.getUri(), timeout,
- getVespaVersionFromRequest(request));
- }
+ private HttpResponse serviceStatusPage(ApplicationId applicationId, String service, String hostname, String pathSuffix) {
+ return applicationRepository.serviceStatusPage(applicationId, hostname, service, pathSuffix);
+ }
- if (isFiledistributionStatusRequest(request)) {
- return applicationRepository.filedistributionStatus(applicationId, timeout);
- }
+ private HttpResponse content(ApplicationId applicationId, String contentPath, HttpRequest request) {
+ long sessionId = applicationRepository.getSessionIdForApplication(applicationId);
+ ApplicationFile applicationFile =
+ applicationRepository.getApplicationFileFromSession(applicationId.tenant(),
+ sessionId,
+ contentPath,
+ ContentRequest.getApplicationFileMode(request.getMethod()));
+ ApplicationContentRequest contentRequest = new ApplicationContentRequest(request,
+ sessionId,
+ applicationId,
+ zone,
+ contentPath,
+ applicationFile);
+ return new ContentHandler().get(contentRequest);
+ }
- if (isLogRequest(request)) {
- Optional<String> hostname = Optional.ofNullable(request.getProperty("hostname"));
- String apiParams = Optional.ofNullable(request.getUri().getQuery()).map(q -> "?" + q).orElse("");
- return applicationRepository.getLogs(applicationId, hostname, apiParams);
- }
+ private HttpResponse filedistributionStatus(ApplicationId applicationId, HttpRequest request) {
+ return applicationRepository.filedistributionStatus(applicationId, getTimeoutFromRequest(request));
+ }
- if (isProtonMetricsRequest(request)) {
- return applicationRepository.getProtonMetrics(applicationId);
- }
+ private HttpResponse logs(ApplicationId applicationId, HttpRequest request) {
+ Optional<String> hostname = Optional.ofNullable(request.getProperty("hostname"));
+ String apiParams = Optional.ofNullable(request.getUri().getQuery()).map(q -> "?" + q).orElse("");
+ return applicationRepository.getLogs(applicationId, hostname, apiParams);
+ }
- if (isDeploymentMetricsRequest(request)) {
- return applicationRepository.getDeploymentMetrics(applicationId);
- }
+ private HttpResponse protonMetrics(ApplicationId applicationId) {
+ return applicationRepository.getProtonMetrics(applicationId);
+ }
- if (isIsSuspendedRequest(request)) {
- return new ApplicationSuspendedResponse(applicationRepository.isSuspended(applicationId));
- }
+ private HttpResponse deploymentMetrics(ApplicationId applicationId) {
+ return applicationRepository.getDeploymentMetrics(applicationId);
+ }
- if (isTesterRequest(request)) {
- String testerCommand = getTesterCommandFromRequest(request);
- switch (testerCommand) {
- case "status":
- return applicationRepository.getTesterStatus(applicationId);
- case "log":
- Long after = Long.valueOf(request.getProperty("after"));
- return applicationRepository.getTesterLog(applicationId, after);
- case "ready":
- return applicationRepository.isTesterReady(applicationId);
- case "report":
- return applicationRepository.getTestReport(applicationId);
- default:
- throw new IllegalArgumentException("Unknown tester command in request " + request.getUri().toString());
- }
- }
+ private HttpResponse isSuspended(ApplicationId applicationId) {
+ return new ApplicationSuspendedResponse(applicationRepository.isSuspended(applicationId));
+ }
- if (isQuotaUsageRequest(request)) {
- var quotaUsageRate = applicationRepository.getQuotaUsageRate(applicationId);
- return new QuotaUsageResponse(quotaUsageRate);
+ private HttpResponse testerRequest(ApplicationId applicationId, String command, HttpRequest request) {
+ switch (command) {
+ case "status":
+ return applicationRepository.getTesterStatus(applicationId);
+ case "log":
+ Long after = Long.valueOf(request.getProperty("after"));
+ return applicationRepository.getTesterLog(applicationId, after);
+ case "ready":
+ return applicationRepository.isTesterReady(applicationId);
+ case "report":
+ return applicationRepository.getTestReport(applicationId);
+ default:
+ throw new IllegalArgumentException("Unknown tester command in request " + request.getUri().toString());
}
+ }
- return getApplicationResponse(applicationId);
+ private HttpResponse quotaUsage(ApplicationId applicationId) {
+ double quotaUsageRate = applicationRepository.getQuotaUsageRate(applicationId);
+ return new QuotaUsageResponse(quotaUsageRate);
}
- GetApplicationResponse getApplicationResponse(ApplicationId applicationId) {
+ private HttpResponse getApplicationResponse(ApplicationId applicationId) {
return new GetApplicationResponse(Response.Status.OK,
- applicationRepository.getApplicationGeneration(applicationId),
- applicationRepository.getAllVersions(applicationId),
- applicationRepository.getApplicationPackageReference(applicationId));
+ applicationRepository.getApplicationGeneration(applicationId),
+ applicationRepository.getAllVersions(applicationId),
+ applicationRepository.getApplicationPackageReference(applicationId));
}
- @Override
- public HttpResponse handlePOST(HttpRequest request) {
- ApplicationId applicationId = getApplicationIdFromRequest(request);
-
- if (isRestartRequest(request))
- return restart(request, applicationId);
-
- if (isTesterStartTestsRequest(request)) {
- byte[] data;
- try {
- data = IOUtils.readBytes(request.getData(), 1024 * 1000);
- } catch (IOException e) {
- throw new IllegalArgumentException("Could not read data in request " + request);
- }
- return applicationRepository.startTests(applicationId, getSuiteFromRequest(request), data);
- }
-
- if (isReindexRequest(request)) {
- return triggerReindexing(request, applicationId);
- }
-
- if (isReindexingRequest(request)) {
- applicationRepository.modifyReindexing(applicationId, reindexing -> reindexing.enabled(true));
- return createMessageResponse("Reindexing enabled");
- }
-
- if (isValidateSecretStoreRequest(request)) {
- var slime = uncheck(() -> SlimeUtils.jsonToSlime(request.getData().readAllBytes()));
- return applicationRepository.validateSecretStore(applicationId, zone.system(), slime);
- }
-
- throw new NotFoundException("Illegal POST request '" + request.getUri() + "'");
+ public HttpResponse deleteApplication(ApplicationId applicationId) {
+ if (applicationRepository.delete(applicationId))
+ return new DeleteApplicationResponse(applicationId);
+ return ErrorResponse.notFoundError("Unable to delete " + applicationId.toFullString() + ": Not found");
}
+
private Model getActiveModelOrThrow(ApplicationId id) {
return applicationRepository.getActiveApplicationSet(id)
.orElseThrow(() -> new NotFoundException("Application '" + id + "' not found"))
@@ -243,7 +199,7 @@ public class ApplicationHandler extends HttpHandler {
.getModel();
}
- private HttpResponse triggerReindexing(HttpRequest request, ApplicationId applicationId) {
+ private HttpResponse triggerReindexing(ApplicationId applicationId, HttpRequest request) {
Model model = getActiveModelOrThrow(applicationId);
Map<String, Set<String>> documentTypes = model.documentTypesByCluster();
Map<String, Set<String>> indexedDocumentTypes = model.indexedDocumentTypesByCluster();
@@ -274,7 +230,7 @@ public class ApplicationHandler extends HttpHandler {
return reindexing;
});
- return createMessageResponse(reindexed.entrySet().stream()
+ return new MessageResponse(reindexed.entrySet().stream()
.filter(cluster -> ! cluster.getValue().isEmpty())
.map(cluster -> "[" + String.join(", ", cluster.getValue()) + "] in '" + cluster.getKey() + "'")
.reduce(new StringJoiner(", ", "Reindexing document types ", " of application " + applicationId)
@@ -284,6 +240,16 @@ public class ApplicationHandler extends HttpHandler {
.toString());
}
+ public HttpResponse disableReindexing(ApplicationId applicationId) {
+ applicationRepository.modifyReindexing(applicationId, reindexing -> reindexing.enabled(false));
+ return new MessageResponse("Reindexing disabled");
+ }
+
+ private HttpResponse enableReindexing(ApplicationId applicationId) {
+ applicationRepository.modifyReindexing(applicationId, reindexing -> reindexing.enabled(true));
+ return new MessageResponse("Reindexing enabled");
+ }
+
private HttpResponse getReindexingStatus(ApplicationId applicationId) {
Tenant tenant = applicationRepository.getTenant(applicationId);
if (tenant == null)
@@ -294,239 +260,42 @@ public class ApplicationHandler extends HttpHandler {
applicationRepository.getClusterReindexingStatus(applicationId));
}
- private HttpResponse restart(HttpRequest request, ApplicationId applicationId) {
- if (getBindingMatch(request).groupCount() != 7)
- throw new NotFoundException("Illegal POST restart request '" + request.getUri() +
- "': Must have 6 arguments but had " + (getBindingMatch(request).groupCount() - 1));
- applicationRepository.restart(applicationId, hostFilterFrom(request));
- return new JSONResponse(Response.Status.OK); // return empty
- }
-
- private HostFilter hostFilterFrom(HttpRequest request) {
- return HostFilter.from(request.getProperty("hostname"),
- request.getProperty("flavor"),
- request.getProperty("clusterType"),
- request.getProperty("clusterId"));
- }
-
- private static BindingMatch<?> getBindingMatch(HttpRequest request) {
- return URI_PATTERNS.stream()
- .map(pattern -> {
- UriPattern.Match match = pattern.match(request.getUri());
- if (match == null) return null;
- return new BindingMatch<>(match, new Object(), pattern);
- })
- .filter(Objects::nonNull)
- .findFirst()
- .orElseThrow(() -> new IllegalArgumentException("Illegal url for config request: " + request.getUri()));
- }
-
- private static boolean isRestartRequest(HttpRequest request) {
- return getBindingMatch(request).groupCount() == 7 &&
- request.getUri().getPath().endsWith("/restart");
- }
-
- private static boolean isReindexRequest(HttpRequest request) {
- return getBindingMatch(request).groupCount() == 7 &&
- request.getUri().getPath().endsWith("/reindex");
- }
-
- private static boolean isReindexingRequest(HttpRequest request) {
- return getBindingMatch(request).groupCount() == 7 &&
- request.getUri().getPath().endsWith("/reindexing");
- }
-
- private static boolean isIsSuspendedRequest(HttpRequest request) {
- return getBindingMatch(request).groupCount() == 7 &&
- request.getUri().getPath().endsWith("/suspended");
- }
-
- private static boolean isProtonMetricsRequest(HttpRequest request) {
- return getBindingMatch(request).groupCount() == 8 &&
- request.getUri().getPath().endsWith("/metrics/proton");
- }
-
- private static boolean isDeploymentMetricsRequest(HttpRequest request) {
- return getBindingMatch(request).groupCount() == 8 &&
- request.getUri().getPath().endsWith("/metrics/deployment");
- }
-
- private static boolean isLogRequest(HttpRequest request) {
- return getBindingMatch(request).groupCount() == 7 &&
- request.getUri().getPath().endsWith("/logs");
- }
-
- private static boolean isValidateSecretStoreRequest(HttpRequest request) {
- return getBindingMatch(request).groupCount() == 7 &&
- request.getUri().getPath().endsWith("/validate-secret-store");
- }
-
- private static boolean isServiceConvergeListRequest(HttpRequest request) {
- return getBindingMatch(request).groupCount() == 7 &&
- request.getUri().getPath().endsWith("/serviceconverge");
- }
-
- private static boolean isServiceConvergeRequest(HttpRequest request) {
- return getBindingMatch(request).groupCount() == 8 &&
- request.getUri().getPath().contains("/serviceconverge/");
- }
-
- private static boolean isClusterControllerStatusRequest(HttpRequest request) {
- return getBindingMatch(request).groupCount() == 9 &&
- request.getUri().getPath().contains("/clustercontroller/");
- }
-
- private static boolean isContentRequest(HttpRequest request) {
- return getBindingMatch(request).groupCount() > 7 &&
- request.getUri().getPath().contains("/content/");
- }
-
- private static boolean isFiledistributionStatusRequest(HttpRequest request) {
- return getBindingMatch(request).groupCount() == 7 &&
- request.getUri().getPath().contains("/filedistributionstatus");
- }
-
- private static boolean isTesterRequest(HttpRequest request) {
- return getBindingMatch(request).groupCount() == 8 &&
- request.getUri().getPath().contains("/tester");
- }
-
- private static boolean isTesterStartTestsRequest(HttpRequest request) {
- return getBindingMatch(request).groupCount() == 9 &&
- request.getUri().getPath().contains("/tester/run/");
- }
-
- private static boolean isQuotaUsageRequest(HttpRequest request) {
- return getBindingMatch(request).groupCount() == 7 &&
- request.getUri().getPath().endsWith("/quota");
- }
-
- private static String getHostNameFromRequest(HttpRequest req) {
- BindingMatch<?> bm = getBindingMatch(req);
- return bm.group(7);
- }
-
- private static String getTesterCommandFromRequest(HttpRequest req) {
- BindingMatch<?> bm = getBindingMatch(req);
- return bm.group(7);
- }
-
- private static String getSuiteFromRequest(HttpRequest req) {
- BindingMatch<?> bm = getBindingMatch(req);
- return bm.group(8);
- }
-
- private static String getPathSuffix(HttpRequest req) {
- BindingMatch<?> bm = getBindingMatch(req);
- return bm.group(8);
- }
-
- private static ApplicationId getApplicationIdFromRequest(HttpRequest req) {
- // Two bindings for this: with full app id or only application name
- BindingMatch<?> bm = getBindingMatch(req);
- if (bm.groupCount() > 4) return createFromRequestFullAppId(bm);
- return createFromRequestSimpleAppId(bm);
- }
-
- // The URL pattern with only tenant and application given
- private static ApplicationId createFromRequestSimpleAppId(BindingMatch<?> bm) {
- TenantName tenant = TenantName.from(bm.group(2));
- ApplicationName application = ApplicationName.from(bm.group(3));
- return new ApplicationId.Builder().tenant(tenant).applicationName(application).build();
- }
-
- // The URL pattern with full app id given
- private static ApplicationId createFromRequestFullAppId(BindingMatch<?> bm) {
- String tenant = bm.group(2);
- String application = bm.group(3);
- String instance = bm.group(6);
- return new ApplicationId.Builder()
- .tenant(tenant)
- .applicationName(application).instanceName(instance)
- .build();
- }
-
- private static Optional<Version> getVespaVersionFromRequest(HttpRequest request) {
- String vespaVersion = request.getProperty("vespaVersion");
- return (vespaVersion == null || vespaVersion.isEmpty())
- ? Optional.empty()
- : Optional.of(Version.fromString(vespaVersion));
+ private HttpResponse restart(ApplicationId applicationId, HttpRequest request) {
+ HostFilter filter = HostFilter.from(request.getProperty("hostname"),
+ request.getProperty("flavor"),
+ request.getProperty("clusterType"),
+ request.getProperty("clusterId"));
+ applicationRepository.restart(applicationId, filter);
+ return new MessageResponse("Success");
}
- private static class DeleteApplicationResponse extends JSONResponse {
- DeleteApplicationResponse(int status, ApplicationId applicationId) {
- super(status);
- object.setString("message", "Application '" + applicationId + "' deleted");
+ private HttpResponse testerStartTests(ApplicationId applicationId, String suite, HttpRequest request) {
+ byte[] data;
+ try {
+ data = IOUtils.readBytes(request.getData(), 1024 * 1000);
+ } catch (IOException e) {
+ throw new IllegalArgumentException("Could not read data in request " + request);
}
+ return applicationRepository.startTests(applicationId, suite, data);
}
- private static class GetApplicationResponse extends JSONResponse {
- GetApplicationResponse(int status, long generation, List<Version> modelVersions, Optional<String> applicationPackageReference) {
- super(status);
- object.setLong("generation", generation);
- object.setString("applicationPackageFileReference", applicationPackageReference.orElse(""));
- Cursor modelVersionArray = object.setArray("modelVersions");
- modelVersions.forEach(version -> modelVersionArray.addString(version.toFullString()));
- }
+ private HttpResponse validateSecretStore(ApplicationId applicationId, HttpRequest request) {
+ var slime = uncheck(() -> SlimeUtils.jsonToSlime(request.getData().readAllBytes()));
+ return applicationRepository.validateSecretStore(applicationId, zone.system(), slime);
}
- private static class ApplicationSuspendedResponse extends JSONResponse {
- ApplicationSuspendedResponse(boolean suspended) {
- super(Response.Status.OK);
- object.setBool("suspended", suspended);
- }
+ private static ApplicationId applicationId(Path path) {
+ return ApplicationId.from(path.get("tenant"), path.get("application"), path.get("instance"));
}
- private static class QuotaUsageResponse extends JSONResponse {
- QuotaUsageResponse(double usageRate) {
- super(Response.Status.OK);
- object.setDouble("rate", usageRate);
- }
+ private static Duration getTimeoutFromRequest(HttpRequest request) {
+ return HttpHandler.getRequestTimeout(request, Duration.ofSeconds(5));
}
- static class ReindexingResponse extends JSONResponse {
- ReindexingResponse(Map<String, Set<String>> documentTypes, ApplicationReindexing reindexing,
- Map<String, ClusterReindexing> clusters) {
- super(Response.Status.OK);
- object.setBool("enabled", reindexing.enabled());
- Cursor clustersObject = object.setObject("clusters");
- documentTypes.forEach((cluster, types) -> {
- Cursor clusterObject = clustersObject.setObject(cluster);
- Cursor pendingObject = clusterObject.setObject("pending");
- Cursor readyObject = clusterObject.setObject("ready");
-
- for (String type : types) {
- Cursor statusObject = readyObject.setObject(type);
- if (reindexing.clusters().containsKey(cluster)) {
- if (reindexing.clusters().get(cluster).pending().containsKey(type))
- pendingObject.setLong(type, reindexing.clusters().get(cluster).pending().get(type));
-
- if (reindexing.clusters().get(cluster).ready().containsKey(type))
- setStatus(statusObject, reindexing.clusters().get(cluster).ready().get(type));
- }
- if (clusters.containsKey(cluster))
- if (clusters.get(cluster).documentTypeStatus().containsKey(type))
- setStatus(statusObject, clusters.get(cluster).documentTypeStatus().get(type));
- }
- });
- }
-
- private static void setStatus(Cursor object, ApplicationReindexing.Status readyStatus) {
- object.setLong("readyMillis", readyStatus.ready().toEpochMilli());
- }
-
- private static void setStatus(Cursor object, ClusterReindexing.Status status) {
- object.setLong("startedMillis", status.startedAt().toEpochMilli());
- status.endedAt().ifPresent(endedAt -> object.setLong("endedMillis", endedAt.toEpochMilli()));
- status.state().map(ClusterReindexing.State::asString).ifPresent(state -> object.setString("state", state));
- status.message().ifPresent(message -> object.setString("message", message));
- status.progress().ifPresent(progress -> object.setDouble("progress", progress));
- }
-
- }
-
- private static JSONResponse createMessageResponse(String message) {
- return new JSONResponse(Response.Status.OK) { { object.setString("message", message); } };
+ private static Optional<Version> getVespaVersionFromRequest(HttpRequest request) {
+ return Optional.ofNullable(request.getProperty("vespaVersion"))
+ .filter(s -> !s.isEmpty())
+ .map(Version::fromString);
}
}
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/HostHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/HostHandler.java
index 1ea41b85983..0a69c7ed904 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/HostHandler.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/HostHandler.java
@@ -12,6 +12,7 @@ import com.yahoo.vespa.config.server.ApplicationRepository;
import com.yahoo.vespa.config.server.http.HttpErrorResponse;
import com.yahoo.vespa.config.server.http.HttpHandler;
import com.yahoo.vespa.config.server.http.JSONResponse;
+import com.yahoo.vespa.config.server.http.v2.request.HttpConfigRequests;
/**
* Handler for getting tenant and application for a given hostname.
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/HttpGetConfigHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/HttpGetConfigHandler.java
index 1787431e841..76a39bbcadb 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/HttpGetConfigHandler.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/HttpGetConfigHandler.java
@@ -7,6 +7,7 @@ import com.yahoo.container.jdisc.HttpResponse;
import java.util.logging.Level;
import com.yahoo.vespa.config.protocol.ConfigResponse;
import com.yahoo.vespa.config.server.RequestHandler;
+import com.yahoo.vespa.config.server.http.v2.request.HttpConfigRequests;
import com.yahoo.vespa.config.server.tenant.TenantRepository;
import com.yahoo.vespa.config.server.http.HttpConfigRequest;
import com.yahoo.vespa.config.server.http.HttpConfigResponse;
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/HttpListConfigsHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/HttpListConfigsHandler.java
index 71be471193a..3210f5eb8e5 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/HttpListConfigsHandler.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/HttpListConfigsHandler.java
@@ -15,6 +15,8 @@ import com.yahoo.slime.JsonFormat;
import com.yahoo.slime.Slime;
import com.yahoo.vespa.config.ConfigKey;
import com.yahoo.vespa.config.server.RequestHandler;
+import com.yahoo.vespa.config.server.http.v2.request.HttpConfigRequests;
+import com.yahoo.vespa.config.server.http.v2.request.HttpListConfigsRequest;
import com.yahoo.vespa.config.server.tenant.TenantRepository;
import com.yahoo.config.provision.ApplicationId;
import com.yahoo.vespa.config.server.http.HttpConfigResponse;
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/HttpListNamedConfigsHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/HttpListNamedConfigsHandler.java
index bbc2efe9c36..425aa1c2bd2 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/HttpListNamedConfigsHandler.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/HttpListNamedConfigsHandler.java
@@ -10,6 +10,8 @@ import com.yahoo.container.jdisc.HttpRequest;
import com.yahoo.container.jdisc.HttpResponse;
import com.yahoo.vespa.config.ConfigKey;
import com.yahoo.vespa.config.server.RequestHandler;
+import com.yahoo.vespa.config.server.http.v2.request.HttpConfigRequests;
+import com.yahoo.vespa.config.server.http.v2.request.HttpListConfigsRequest;
import com.yahoo.vespa.config.server.tenant.TenantRepository;
import com.yahoo.config.provision.ApplicationId;
import com.yahoo.vespa.config.server.http.HttpConfigRequest;
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ListApplicationsHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ListApplicationsHandler.java
index b06d5c31ac6..20688f4f2f1 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ListApplicationsHandler.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ListApplicationsHandler.java
@@ -10,6 +10,7 @@ import com.yahoo.container.jdisc.HttpRequest;
import com.yahoo.container.jdisc.HttpResponse;
import com.yahoo.jdisc.Response;
import com.yahoo.vespa.config.server.application.TenantApplications;
+import com.yahoo.vespa.config.server.http.v2.response.ListApplicationsResponse;
import com.yahoo.vespa.config.server.tenant.Tenant;
import com.yahoo.vespa.config.server.tenant.TenantRepository;
import com.yahoo.config.provision.ApplicationId;
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionActiveHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionActiveHandler.java
index 4e75234620f..f97e272b132 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionActiveHandler.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionActiveHandler.java
@@ -14,6 +14,7 @@ import com.yahoo.jdisc.Request;
import com.yahoo.jdisc.handler.ResponseHandler;
import java.util.logging.Level;
import com.yahoo.vespa.config.server.ApplicationRepository;
+import com.yahoo.vespa.config.server.http.v2.response.SessionActiveResponse;
import com.yahoo.vespa.config.server.tenant.Tenant;
import com.yahoo.vespa.config.server.tenant.TenantRepository;
import com.yahoo.vespa.config.server.TimeoutBudget;
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionContentHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionContentHandler.java
index 9f526bbdce8..4664fdf3557 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionContentHandler.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionContentHandler.java
@@ -11,6 +11,7 @@ import com.yahoo.vespa.config.server.http.ContentRequest;
import com.yahoo.vespa.config.server.http.ContentHandler;
import com.yahoo.vespa.config.server.http.SessionHandler;
import com.yahoo.vespa.config.server.http.Utils;
+import com.yahoo.vespa.config.server.http.v2.request.SessionContentRequestV2;
/**
* A handler that will return content or content status for files or directories
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionCreateHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionCreateHandler.java
index 61f099fb8ea..ea7d11d9802 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionCreateHandler.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionCreateHandler.java
@@ -9,7 +9,6 @@ import com.yahoo.config.provision.InstanceName;
import com.yahoo.config.provision.TenantName;
import com.yahoo.container.jdisc.HttpRequest;
import com.yahoo.container.jdisc.HttpResponse;
-import com.yahoo.jdisc.HeaderFields;
import com.yahoo.jdisc.application.UriPattern;
import com.yahoo.vespa.config.server.ApplicationRepository;
import com.yahoo.vespa.config.server.deploy.DeployHandlerLogger;
@@ -17,13 +16,12 @@ import com.yahoo.vespa.config.server.TimeoutBudget;
import com.yahoo.vespa.config.server.http.BadRequestException;
import com.yahoo.vespa.config.server.http.SessionHandler;
import com.yahoo.vespa.config.server.http.Utils;
-import com.yahoo.vespa.model.content.Content;
+import com.yahoo.vespa.config.server.http.v2.response.SessionCreateResponse;
import org.apache.hc.core5.http.ContentType;
import java.net.URI;
import java.time.Duration;
import java.util.List;
-import java.util.stream.Collectors;
/**
* A handler that is able to create a session from an application package,
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionPrepareHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionPrepareHandler.java
index 6fa2075807f..0d6d5ed943a 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionPrepareHandler.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionPrepareHandler.java
@@ -7,6 +7,7 @@ import com.yahoo.config.provision.TenantName;
import com.yahoo.container.jdisc.HttpRequest;
import com.yahoo.container.jdisc.HttpResponse;
import com.yahoo.vespa.config.server.ApplicationRepository;
+import com.yahoo.vespa.config.server.http.v2.response.SessionPrepareResponse;
import com.yahoo.vespa.config.server.session.PrepareParams;
import com.yahoo.vespa.config.server.tenant.Tenant;
import com.yahoo.vespa.config.server.tenant.TenantRepository;
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/TenantHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/TenantHandler.java
index bb94f8d442a..a9961762f8b 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/TenantHandler.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/TenantHandler.java
@@ -10,6 +10,10 @@ import com.yahoo.restapi.RestApi;
import com.yahoo.restapi.RestApiException;
import com.yahoo.restapi.RestApiRequestHandler;
import com.yahoo.vespa.config.server.ApplicationRepository;
+import com.yahoo.vespa.config.server.http.v2.response.ListTenantsResponse;
+import com.yahoo.vespa.config.server.http.v2.response.TenantCreateResponse;
+import com.yahoo.vespa.config.server.http.v2.response.TenantDeleteResponse;
+import com.yahoo.vespa.config.server.http.v2.response.TenantGetResponse;
import com.yahoo.vespa.config.server.tenant.TenantRepository;
import com.yahoo.yolean.Exceptions;
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationContentRequest.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/request/ApplicationContentRequest.java
index af3ab0b1b83..b7ec6272c3a 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationContentRequest.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/request/ApplicationContentRequest.java
@@ -1,5 +1,5 @@
// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.config.server.http.v2;
+package com.yahoo.vespa.config.server.http.v2.request;
import com.yahoo.config.application.api.ApplicationFile;
import com.yahoo.config.provision.ApplicationId;
@@ -18,12 +18,12 @@ public class ApplicationContentRequest extends ContentRequest {
private final ApplicationId applicationId;
private final Zone zone;
- ApplicationContentRequest(HttpRequest request,
- long sessionId,
- ApplicationId applicationId,
- Zone zone,
- String contentPath,
- ApplicationFile applicationFile) {
+ public ApplicationContentRequest(HttpRequest request,
+ long sessionId,
+ ApplicationId applicationId,
+ Zone zone,
+ String contentPath,
+ ApplicationFile applicationFile) {
super(request, sessionId, contentPath, applicationFile);
this.applicationId = applicationId;
this.zone = zone;
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/HttpConfigRequests.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/request/HttpConfigRequests.java
index 8b482997044..d190ccaec12 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/HttpConfigRequests.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/request/HttpConfigRequests.java
@@ -1,5 +1,5 @@
// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.config.server.http.v2;
+package com.yahoo.vespa.config.server.http.v2.request;
import java.net.URI;
@@ -45,7 +45,7 @@ public class HttpConfigRequests {
}
- static RequestHandler getRequestHandler(TenantRepository tenantRepository, TenantRequest request) {
+ public static RequestHandler getRequestHandler(TenantRepository tenantRepository, TenantRequest request) {
Tenant tenant = tenantRepository.getTenant(request.getApplicationId().tenant());
if (tenant==null) throw new NotFoundException("No such tenant: "+request.getApplicationId().tenant());
return tenant.getRequestHandler();
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/HttpListConfigsRequest.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/request/HttpListConfigsRequest.java
index 208b682df9a..cd9c44db85b 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/HttpListConfigsRequest.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/request/HttpListConfigsRequest.java
@@ -1,5 +1,5 @@
// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.config.server.http.v2;
+package com.yahoo.vespa.config.server.http.v2.request;
import com.yahoo.collections.Tuple2;
import com.yahoo.config.provision.ApplicationName;
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionContentRequestV2.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/request/SessionContentRequestV2.java
index c5a9d36f493..4f34b2fb332 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionContentRequestV2.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/request/SessionContentRequestV2.java
@@ -1,5 +1,5 @@
// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.config.server.http.v2;
+package com.yahoo.vespa.config.server.http.v2.request;
import com.yahoo.config.application.api.ApplicationFile;
import com.yahoo.config.provision.TenantName;
@@ -20,7 +20,7 @@ public class SessionContentRequestV2 extends ContentRequest {
private final TenantName tenantName;
private final long sessionId;
- SessionContentRequestV2(HttpRequest request,
+ public SessionContentRequestV2(HttpRequest request,
long sessionId,
TenantName tenantName,
String path,
@@ -35,7 +35,7 @@ public class SessionContentRequestV2 extends ContentRequest {
return "/application/v2/tenant/" + tenantName.value() + "/session/" + sessionId;
}
- static String getContentPath(HttpRequest request) {
+ public static String getContentPath(HttpRequest request) {
BindingMatch<?> bm = Utils.getBindingMatch(request, uriPattern);
return bm.group(4);
}
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/TenantRequest.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/request/TenantRequest.java
index 0fae091aa78..84071ca2a98 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/TenantRequest.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/request/TenantRequest.java
@@ -1,5 +1,5 @@
// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.config.server.http.v2;
+package com.yahoo.vespa.config.server.http.v2.request;
import com.yahoo.config.provision.ApplicationId;
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/response/ApplicationSuspendedResponse.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/response/ApplicationSuspendedResponse.java
new file mode 100644
index 00000000000..c2449d9a31d
--- /dev/null
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/response/ApplicationSuspendedResponse.java
@@ -0,0 +1,12 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.config.server.http.v2.response;
+
+import com.yahoo.jdisc.Response;
+import com.yahoo.vespa.config.server.http.JSONResponse;
+
+public class ApplicationSuspendedResponse extends JSONResponse {
+ public ApplicationSuspendedResponse(boolean suspended) {
+ super(Response.Status.OK);
+ object.setBool("suspended", suspended);
+ }
+}
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/response/DeleteApplicationResponse.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/response/DeleteApplicationResponse.java
new file mode 100644
index 00000000000..375a6d5e57f
--- /dev/null
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/response/DeleteApplicationResponse.java
@@ -0,0 +1,11 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.config.server.http.v2.response;
+
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.restapi.MessageResponse;
+
+public class DeleteApplicationResponse extends MessageResponse {
+ public DeleteApplicationResponse(ApplicationId applicationId) {
+ super("Application '" + applicationId + "' deleted");
+ }
+}
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/DeploymentMetricsResponse.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/response/DeploymentMetricsResponse.java
index cdfdce91500..253e37f2b0a 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/DeploymentMetricsResponse.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/response/DeploymentMetricsResponse.java
@@ -1,5 +1,5 @@
// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.config.server.http.v2;
+package com.yahoo.vespa.config.server.http.v2.response;
import com.yahoo.config.provision.ApplicationId;
import com.yahoo.restapi.SlimeJsonResponse;
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/response/GetApplicationResponse.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/response/GetApplicationResponse.java
new file mode 100644
index 00000000000..dc124891540
--- /dev/null
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/response/GetApplicationResponse.java
@@ -0,0 +1,19 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.config.server.http.v2.response;
+
+import com.yahoo.component.Version;
+import com.yahoo.slime.Cursor;
+import com.yahoo.vespa.config.server.http.JSONResponse;
+
+import java.util.List;
+import java.util.Optional;
+
+public class GetApplicationResponse extends JSONResponse {
+ public GetApplicationResponse(int status, long generation, List<Version> modelVersions, Optional<String> applicationPackageReference) {
+ super(status);
+ object.setLong("generation", generation);
+ object.setString("applicationPackageFileReference", applicationPackageReference.orElse(""));
+ Cursor modelVersionArray = object.setArray("modelVersions");
+ modelVersions.forEach(version -> modelVersionArray.addString(version.toFullString()));
+ }
+}
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ListApplicationsResponse.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/response/ListApplicationsResponse.java
index a4527305abb..6b1f7031d81 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ListApplicationsResponse.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/response/ListApplicationsResponse.java
@@ -1,5 +1,5 @@
// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.config.server.http.v2;
+package com.yahoo.vespa.config.server.http.v2.response;
import com.yahoo.container.jdisc.HttpResponse;
import com.yahoo.slime.Cursor;
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ListTenantsResponse.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/response/ListTenantsResponse.java
index 3789939429c..ec1b3c83604 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ListTenantsResponse.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/response/ListTenantsResponse.java
@@ -1,11 +1,12 @@
// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.config.server.http.v2;
+package com.yahoo.vespa.config.server.http.v2.response;
-import com.google.common.collect.ImmutableSet;
import com.yahoo.config.provision.TenantName;
import com.yahoo.restapi.SlimeJsonResponse;
import com.yahoo.slime.Cursor;
+import java.util.Set;
+
/**
* Tenant list response
*
@@ -13,7 +14,7 @@ import com.yahoo.slime.Cursor;
*/
public class ListTenantsResponse extends SlimeJsonResponse {
- ListTenantsResponse(ImmutableSet<TenantName> tenants) {
+ public ListTenantsResponse(Set<TenantName> tenants) {
Cursor tenantArray = slime.setObject().setArray("tenants");
tenants.forEach(tenantName -> tenantArray.addString(tenantName.value()));
}
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ProtonMetricsResponse.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/response/ProtonMetricsResponse.java
index 99b95f9244c..25e770c7211 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ProtonMetricsResponse.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/response/ProtonMetricsResponse.java
@@ -1,5 +1,5 @@
// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.config.server.http.v2;
+package com.yahoo.vespa.config.server.http.v2.response;
import com.yahoo.config.provision.ApplicationId;
import com.yahoo.restapi.SlimeJsonResponse;
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/response/QuotaUsageResponse.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/response/QuotaUsageResponse.java
new file mode 100644
index 00000000000..e30f0f45098
--- /dev/null
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/response/QuotaUsageResponse.java
@@ -0,0 +1,12 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.config.server.http.v2.response;
+
+import com.yahoo.jdisc.Response;
+import com.yahoo.vespa.config.server.http.JSONResponse;
+
+public class QuotaUsageResponse extends JSONResponse {
+ public QuotaUsageResponse(double usageRate) {
+ super(Response.Status.OK);
+ object.setDouble("rate", usageRate);
+ }
+}
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/response/ReindexingResponse.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/response/ReindexingResponse.java
new file mode 100644
index 00000000000..e73d4a9cb56
--- /dev/null
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/response/ReindexingResponse.java
@@ -0,0 +1,52 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.config.server.http.v2.response;
+
+import com.yahoo.jdisc.Response;
+import com.yahoo.slime.Cursor;
+import com.yahoo.vespa.config.server.application.ApplicationReindexing;
+import com.yahoo.vespa.config.server.application.ClusterReindexing;
+import com.yahoo.vespa.config.server.http.JSONResponse;
+
+import java.util.Map;
+import java.util.Set;
+
+public class ReindexingResponse extends JSONResponse {
+ public ReindexingResponse(Map<String, Set<String>> documentTypes, ApplicationReindexing reindexing,
+ Map<String, ClusterReindexing> clusters) {
+ super(Response.Status.OK);
+ object.setBool("enabled", reindexing.enabled());
+ Cursor clustersObject = object.setObject("clusters");
+ documentTypes.forEach((cluster, types) -> {
+ Cursor clusterObject = clustersObject.setObject(cluster);
+ Cursor pendingObject = clusterObject.setObject("pending");
+ Cursor readyObject = clusterObject.setObject("ready");
+
+ for (String type : types) {
+ Cursor statusObject = readyObject.setObject(type);
+ if (reindexing.clusters().containsKey(cluster)) {
+ if (reindexing.clusters().get(cluster).pending().containsKey(type))
+ pendingObject.setLong(type, reindexing.clusters().get(cluster).pending().get(type));
+
+ if (reindexing.clusters().get(cluster).ready().containsKey(type))
+ setStatus(statusObject, reindexing.clusters().get(cluster).ready().get(type));
+ }
+ if (clusters.containsKey(cluster))
+ if (clusters.get(cluster).documentTypeStatus().containsKey(type))
+ setStatus(statusObject, clusters.get(cluster).documentTypeStatus().get(type));
+ }
+ });
+ }
+
+ private static void setStatus(Cursor object, ApplicationReindexing.Status readyStatus) {
+ object.setLong("readyMillis", readyStatus.ready().toEpochMilli());
+ }
+
+ private static void setStatus(Cursor object, ClusterReindexing.Status status) {
+ object.setLong("startedMillis", status.startedAt().toEpochMilli());
+ status.endedAt().ifPresent(endedAt -> object.setLong("endedMillis", endedAt.toEpochMilli()));
+ status.state().map(ClusterReindexing.State::asString).ifPresent(state -> object.setString("state", state));
+ status.message().ifPresent(message -> object.setString("message", message));
+ status.progress().ifPresent(progress -> object.setDouble("progress", progress));
+ }
+
+}
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionActiveResponse.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/response/SessionActiveResponse.java
index 9c0fbdf2613..f14f9cc6575 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionActiveResponse.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/response/SessionActiveResponse.java
@@ -1,5 +1,5 @@
// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.config.server.http.v2;
+package com.yahoo.vespa.config.server.http.v2.response;
import com.yahoo.config.provision.TenantName;
import com.yahoo.config.provision.Zone;
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionCreateResponse.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/response/SessionCreateResponse.java
index faf02a1ea4f..047962ea16b 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionCreateResponse.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/response/SessionCreateResponse.java
@@ -1,5 +1,5 @@
// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.config.server.http.v2;
+package com.yahoo.vespa.config.server.http.v2.response;
import com.yahoo.config.provision.TenantName;
import com.yahoo.restapi.SlimeJsonResponse;
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionPrepareAndActivateResponse.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/response/SessionPrepareAndActivateResponse.java
index 7bace4749a8..d219ff64b4e 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionPrepareAndActivateResponse.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/response/SessionPrepareAndActivateResponse.java
@@ -1,5 +1,5 @@
// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.config.server.http.v2;
+package com.yahoo.vespa.config.server.http.v2.response;
import com.yahoo.config.provision.ApplicationId;
import com.yahoo.config.provision.TenantName;
@@ -8,15 +8,16 @@ import com.yahoo.container.jdisc.HttpRequest;
import com.yahoo.restapi.SlimeJsonResponse;
import com.yahoo.slime.Cursor;
import com.yahoo.vespa.config.server.configchange.ConfigChangeActionsSlimeConverter;
+import com.yahoo.vespa.config.server.http.v2.PrepareResult;
/**
* Creates a response for SessionPrepareHandler.
*
* @author hmusum
*/
-class SessionPrepareAndActivateResponse extends SlimeJsonResponse {
+public class SessionPrepareAndActivateResponse extends SlimeJsonResponse {
- SessionPrepareAndActivateResponse(PrepareResult result, HttpRequest request, ApplicationId applicationId, Zone zone) {
+ public SessionPrepareAndActivateResponse(PrepareResult result, HttpRequest request, ApplicationId applicationId, Zone zone) {
super(result.deployLogger().slime());
TenantName tenantName = applicationId.tenant();
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionPrepareResponse.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/response/SessionPrepareResponse.java
index a97cd37d3b4..5f50a5236fb 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionPrepareResponse.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/response/SessionPrepareResponse.java
@@ -1,5 +1,5 @@
// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.config.server.http.v2;
+package com.yahoo.vespa.config.server.http.v2.response;
import com.yahoo.config.provision.TenantName;
import com.yahoo.container.jdisc.HttpRequest;
@@ -9,19 +9,20 @@ import com.yahoo.slime.Slime;
import com.yahoo.slime.Type;
import com.yahoo.vespa.config.server.configchange.ConfigChangeActions;
import com.yahoo.vespa.config.server.configchange.ConfigChangeActionsSlimeConverter;
+import com.yahoo.vespa.config.server.http.v2.PrepareResult;
/**
* Creates a response for SessionPrepareHandler.
*
* @author hmusum
*/
-class SessionPrepareResponse extends SlimeJsonResponse {
+public class SessionPrepareResponse extends SlimeJsonResponse {
- SessionPrepareResponse(TenantName tenantName, HttpRequest request, long sessionId) {
+ public SessionPrepareResponse(TenantName tenantName, HttpRequest request, long sessionId) {
this(new Slime(), tenantName, request, sessionId, new ConfigChangeActions());
}
- SessionPrepareResponse(PrepareResult result, TenantName tenantName, HttpRequest request) {
+ public SessionPrepareResponse(PrepareResult result, TenantName tenantName, HttpRequest request) {
this(result.deployLogger().slime(), tenantName, request, result.sessionId(), result.configChangeActions());
}
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/TenantCreateResponse.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/response/TenantCreateResponse.java
index 6ff2b30075d..395a52df9f2 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/TenantCreateResponse.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/response/TenantCreateResponse.java
@@ -1,5 +1,5 @@
// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.config.server.http.v2;
+package com.yahoo.vespa.config.server.http.v2.response;
import com.yahoo.config.provision.TenantName;
import com.yahoo.restapi.MessageResponse;
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/TenantDeleteResponse.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/response/TenantDeleteResponse.java
index d21584c8cdc..fbfb4d581c6 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/TenantDeleteResponse.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/response/TenantDeleteResponse.java
@@ -1,5 +1,5 @@
// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.config.server.http.v2;
+package com.yahoo.vespa.config.server.http.v2.response;
import com.yahoo.config.provision.TenantName;
import com.yahoo.restapi.MessageResponse;
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/TenantGetResponse.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/response/TenantGetResponse.java
index b918cab7828..3a50a68cb21 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/TenantGetResponse.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/response/TenantGetResponse.java
@@ -1,5 +1,5 @@
// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.config.server.http.v2;
+package com.yahoo.vespa.config.server.http.v2.response;
import com.yahoo.config.provision.TenantName;
import com.yahoo.restapi.MessageResponse;
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/metrics/DeploymentMetricsRetriever.java b/configserver/src/main/java/com/yahoo/vespa/config/server/metrics/DeploymentMetricsRetriever.java
index 7fc2c47c06c..2681952cae9 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/metrics/DeploymentMetricsRetriever.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/metrics/DeploymentMetricsRetriever.java
@@ -4,7 +4,7 @@ package com.yahoo.vespa.config.server.metrics;
import com.yahoo.config.model.api.HostInfo;
import com.yahoo.config.model.api.ServiceInfo;
import com.yahoo.vespa.config.server.application.Application;
-import com.yahoo.vespa.config.server.http.v2.DeploymentMetricsResponse;
+import com.yahoo.vespa.config.server.http.v2.response.DeploymentMetricsResponse;
import java.net.URI;
import java.util.Collection;
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/metrics/ProtonMetricsRetriever.java b/configserver/src/main/java/com/yahoo/vespa/config/server/metrics/ProtonMetricsRetriever.java
index 5078fba8b38..4c111687c41 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/metrics/ProtonMetricsRetriever.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/metrics/ProtonMetricsRetriever.java
@@ -4,7 +4,7 @@ package com.yahoo.vespa.config.server.metrics;
import com.yahoo.config.model.api.HostInfo;
import com.yahoo.config.model.api.ServiceInfo;
import com.yahoo.vespa.config.server.application.Application;
-import com.yahoo.vespa.config.server.http.v2.ProtonMetricsResponse;
+import com.yahoo.vespa.config.server.http.v2.response.ProtonMetricsResponse;
import java.net.URI;
import java.util.Collection;
import java.util.function.Predicate;
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClient.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClient.java
index 52fbb8e6781..a46b1cc9bdc 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClient.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClient.java
@@ -90,7 +90,7 @@ public class SessionZooKeeperClient {
return data.map(d -> Session.Status.parse(Utf8.toString(d))).orElse(Session.Status.NONE);
} catch (Exception e) {
log.log(Level.INFO, "Failed to read session status at " + sessionStatusPath.getAbsolute() +
- ", will assume session has been removed: " + e.getMessage());
+ ", will assume session has been removed: ", e);
return Session.Status.NONE;
}
}
diff --git a/configserver/src/main/sh/start-configserver b/configserver/src/main/sh/start-configserver
index 24acf79705d..f9eb5e3a0a5 100755
--- a/configserver/src/main/sh/start-configserver
+++ b/configserver/src/main/sh/start-configserver
@@ -175,6 +175,7 @@ vespa-run-as-vespa-user vespa-runserver -s configserver -r 30 -p $pidfile -- \
--add-opens=java.base/java.io=ALL-UNNAMED \
--add-opens=java.base/java.lang=ALL-UNNAMED \
--add-opens=java.base/java.net=ALL-UNNAMED \
+ --add-opens=java.base/java.nio=ALL-UNNAMED \
--add-opens=java.base/jdk.internal.loader=ALL-UNNAMED \
--add-opens=java.base/sun.security.ssl=ALL-UNNAMED \
-Djava.io.tmpdir=${VESPA_HOME}/tmp \
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java
index 22a5b3795d9..46d94ec476b 100644
--- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java
@@ -32,7 +32,7 @@ import com.yahoo.vespa.config.server.http.HttpErrorResponse;
import com.yahoo.vespa.config.server.http.SecretStoreValidator;
import com.yahoo.vespa.config.server.http.SessionHandlerTest;
import com.yahoo.vespa.config.server.http.StaticResponse;
-import com.yahoo.vespa.config.server.http.v2.ApplicationHandler.ReindexingResponse;
+import com.yahoo.vespa.config.server.http.v2.response.ReindexingResponse;
import com.yahoo.vespa.config.server.modelfactory.ModelFactoryRegistry;
import com.yahoo.vespa.config.server.provision.HostProvisionerProvider;
import com.yahoo.vespa.config.server.session.PrepareParams;
@@ -60,7 +60,6 @@ import java.util.TreeMap;
import java.util.TreeSet;
import java.util.stream.Stream;
-import static com.yahoo.config.model.api.container.ContainerServiceType.CLUSTERCONTROLLER_CONTAINER;
import static com.yahoo.container.jdisc.HttpRequest.createTestRequest;
import static com.yahoo.jdisc.http.HttpRequest.Method.DELETE;
import static com.yahoo.jdisc.http.HttpRequest.Method.GET;
@@ -72,9 +71,8 @@ import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import static org.mockito.ArgumentMatchers.any;
-import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.doAnswer;
import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
/**
* @author hmusum
@@ -343,10 +341,9 @@ public class ApplicationHandlerTest {
}
@Test
- public void testClusterControllerStatus() throws Exception {
+ public void testServiceStatus() throws Exception {
applicationRepository.deploy(testApp, prepareParams(applicationId));
String host = "foo.yahoo.com";
- String url = toUrlPath(applicationId, Zone.defaultZone(), true) + "/clustercontroller/" + host + "/status/v1/clusterName1";
HttpProxy mockHttpProxy = mock(HttpProxy.class);
ApplicationRepository applicationRepository = new ApplicationRepository.Builder()
.withTenantRepository(tenantRepository)
@@ -356,11 +353,19 @@ public class ApplicationHandlerTest {
.withHttpProxy(mockHttpProxy)
.build();
ApplicationHandler mockHandler = createApplicationHandler(applicationRepository);
- when(mockHttpProxy.get(any(), eq(host), eq(CLUSTERCONTROLLER_CONTAINER.serviceName), eq("clustercontroller-status/v1/clusterName1")))
- .thenReturn(new StaticResponse(200, "text/html", "<html>...</html>"));
+ doAnswer(invoc -> new StaticResponse(200, "text/html", "<html>" +
+ "host=" + invoc.getArgument(1, String.class) + "," +
+ "service=" + invoc.getArgument(2, String.class) + "," +
+ "path=" + invoc.getArgument(3, String.class) + "</html>")).when(mockHttpProxy).get(any(), any(), any(), any());
- HttpResponse response = mockHandler.handle(createTestRequest(url, GET));
- assertHttpStatusCodeAndMessage(response, 200, "text/html", "<html>...</html>");
+ HttpResponse response = mockHandler.handle(createTestRequest(toUrlPath(applicationId, Zone.defaultZone(), true) + "/service/container-clustercontroller/" + host + "/status/some/path/clusterName1", GET));
+ assertHttpStatusCodeAndMessage(response, 200, "text/html", "<html>host=foo.yahoo.com,service=container-clustercontroller,path=clustercontroller-status/v1/some/path/clusterName1</html>");
+
+ response = mockHandler.handle(createTestRequest(toUrlPath(applicationId, Zone.defaultZone(), true) + "/service/distributor/" + host + "/status/something", GET));
+ assertHttpStatusCodeAndMessage(response, 200, "text/html", "<html>host=foo.yahoo.com,service=distributor,path=something</html>");
+
+ response = mockHandler.handle(createTestRequest(toUrlPath(applicationId, Zone.defaultZone(), true) + "/service/fake-service/" + host + "/status/something", GET));
+ assertHttpStatusCodeAndMessage(response, 404, "{\"error-code\":\"NOT_FOUND\",\"message\":\"No status page for service: fake-service\"}");
}
@Test
diff --git a/container-apache-http-client-bundle/src/main/java/org/apache/http/client/entity/package-info.java b/container-apache-http-client-bundle/src/main/java/org/apache/http/client/entity/package-info.java
new file mode 100644
index 00000000000..28199fd727b
--- /dev/null
+++ b/container-apache-http-client-bundle/src/main/java/org/apache/http/client/entity/package-info.java
@@ -0,0 +1,8 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+/**
+ * @author bjorncs
+ */
+@ExportPackage
+package org.apache.http.client.entity;
+
+import com.yahoo.osgi.annotation.ExportPackage; \ No newline at end of file
diff --git a/container-core/pom.xml b/container-core/pom.xml
index 2b87d79daa4..c7fe2998530 100644
--- a/container-core/pom.xml
+++ b/container-core/pom.xml
@@ -140,10 +140,6 @@
<version>${project.version}</version>
</dependency>
<dependency>
- <groupId>org.apache.httpcomponents</groupId>
- <artifactId>httpclient</artifactId>
- </dependency>
- <dependency>
<groupId>org.hdrhistogram</groupId>
<artifactId>HdrHistogram</artifactId>
</dependency>
@@ -211,6 +207,12 @@
</dependency>
<dependency>
<groupId>com.yahoo.vespa</groupId>
+ <artifactId>container-apache-http-client-bundle</artifactId>
+ <version>${project.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>com.yahoo.vespa</groupId>
<artifactId>defaults</artifactId>
<version>${project.version}</version>
<scope>provided</scope>
@@ -262,11 +264,6 @@
<scope>test</scope>
</dependency>
<dependency>
- <groupId>org.apache.httpcomponents</groupId>
- <artifactId>httpmime</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
<groupId>org.assertj</groupId>
<artifactId>assertj-core</artifactId>
<scope>test</scope>
@@ -302,17 +299,6 @@
<artifactId>mockito-core</artifactId>
<scope>test</scope>
</dependency>
- <dependency>
- <groupId>org.apache.httpcomponents.client5</groupId>
- <artifactId>httpclient5</artifactId>
- <scope>test</scope>
- <exclusions>
- <exclusion>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-api</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
</dependencies>
<build>
<plugins>
diff --git a/container-core/src/main/java/com/yahoo/container/core/config/HandlersConfigurerDi.java b/container-core/src/main/java/com/yahoo/container/core/config/HandlersConfigurerDi.java
index 9f6f5cf7ea1..08a468a3031 100644
--- a/container-core/src/main/java/com/yahoo/container/core/config/HandlersConfigurerDi.java
+++ b/container-core/src/main/java/com/yahoo/container/core/config/HandlersConfigurerDi.java
@@ -117,8 +117,11 @@ public class HandlersConfigurerDi {
@Override
public void installPlatformBundles(Collection<String> bundlePaths) {
- log.fine("Installing platform bundles.");
- platformBundleLoader.useBundles(new ArrayList<>(bundlePaths));
+ // Don't install physical bundles for test frameworks, where all platform bundles are on the classpath.
+ if (osgiFramework.isFelixFramework()) {
+ log.fine("Installing platform bundles.");
+ platformBundleLoader.useBundles(new ArrayList<>(bundlePaths));
+ }
}
@Override
diff --git a/container-core/src/test/java/com/yahoo/jdisc/http/server/jetty/EchoRequestHandler.java b/container-core/src/test/java/com/yahoo/jdisc/http/server/jetty/EchoRequestHandler.java
new file mode 100644
index 00000000000..9610648ad41
--- /dev/null
+++ b/container-core/src/test/java/com/yahoo/jdisc/http/server/jetty/EchoRequestHandler.java
@@ -0,0 +1,24 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.jdisc.http.server.jetty;
+
+import com.yahoo.jdisc.Request;
+import com.yahoo.jdisc.Response;
+import com.yahoo.jdisc.handler.AbstractRequestHandler;
+import com.yahoo.jdisc.handler.ContentChannel;
+import com.yahoo.jdisc.handler.ResponseHandler;
+
+import static com.yahoo.jdisc.Response.Status.OK;
+
+/**
+ * @author bjorncs
+ */
+class EchoRequestHandler extends AbstractRequestHandler {
+ @Override
+ public ContentChannel handleRequest(final Request request, final ResponseHandler handler) {
+ int port = request.getUri().getPort();
+ Response response = new Response(OK);
+ response.headers().put("Jdisc-Local-Port", Integer.toString(port));
+ return handler.handleResponse(response);
+ }
+}
+
diff --git a/container-core/src/test/java/com/yahoo/jdisc/http/server/jetty/HttpServerTest.java b/container-core/src/test/java/com/yahoo/jdisc/http/server/jetty/HttpServerTest.java
index 0f625b5c3df..bd3000a0188 100644
--- a/container-core/src/test/java/com/yahoo/jdisc/http/server/jetty/HttpServerTest.java
+++ b/container-core/src/test/java/com/yahoo/jdisc/http/server/jetty/HttpServerTest.java
@@ -5,14 +5,12 @@ import com.google.inject.AbstractModule;
import com.google.inject.Module;
import com.yahoo.container.logging.ConnectionLog;
import com.yahoo.container.logging.ConnectionLogEntry;
-import com.yahoo.container.logging.ConnectionLogEntry.SslHandshakeFailure.ExceptionEntry;
import com.yahoo.container.logging.RequestLog;
import com.yahoo.container.logging.RequestLogEntry;
import com.yahoo.jdisc.References;
import com.yahoo.jdisc.Request;
import com.yahoo.jdisc.Response;
import com.yahoo.jdisc.application.BindingSetSelector;
-import com.yahoo.jdisc.application.MetricConsumer;
import com.yahoo.jdisc.handler.AbstractRequestHandler;
import com.yahoo.jdisc.handler.CompletionHandler;
import com.yahoo.jdisc.handler.ContentChannel;
@@ -28,12 +26,7 @@ import com.yahoo.jdisc.http.HttpResponse;
import com.yahoo.jdisc.http.ServerConfig;
import com.yahoo.jdisc.http.server.jetty.JettyTestDriver.TlsClientAuth;
import com.yahoo.jdisc.service.BindingSetNotFoundException;
-import com.yahoo.security.KeyUtils;
-import com.yahoo.security.Pkcs10Csr;
-import com.yahoo.security.Pkcs10CsrBuilder;
import com.yahoo.security.SslContextBuilder;
-import com.yahoo.security.X509CertificateBuilder;
-import com.yahoo.security.X509CertificateUtils;
import com.yahoo.security.tls.TlsContext;
import org.apache.hc.client5.http.async.methods.SimpleHttpResponse;
import org.apache.hc.client5.http.async.methods.SimpleRequestBuilder;
@@ -42,39 +35,22 @@ import org.apache.hc.client5.http.entity.mime.FormBodyPartBuilder;
import org.apache.hc.client5.http.entity.mime.StringBody;
import org.apache.hc.client5.http.impl.async.CloseableHttpAsyncClient;
import org.apache.hc.client5.http.impl.async.H2AsyncClientBuilder;
-import org.apache.hc.client5.http.impl.async.HttpAsyncClientBuilder;
-import org.apache.hc.client5.http.impl.nio.PoolingAsyncClientConnectionManagerBuilder;
import org.apache.hc.client5.http.ssl.ClientTlsStrategyBuilder;
-import org.apache.hc.client5.http.ssl.NoopHostnameVerifier;
import org.apache.hc.core5.http.ContentType;
import org.apache.hc.core5.http.nio.ssl.TlsStrategy;
-import org.apache.hc.core5.http2.HttpVersionPolicy;
import org.assertj.core.api.Assertions;
-import org.eclipse.jetty.client.HttpClient;
-import org.eclipse.jetty.client.ProxyProtocolClientConnectionFactory.V1;
-import org.eclipse.jetty.client.ProxyProtocolClientConnectionFactory.V2;
-import org.eclipse.jetty.client.api.ContentResponse;
import org.eclipse.jetty.server.handler.AbstractHandlerContainer;
-import org.eclipse.jetty.util.ssl.SslContextFactory;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
import javax.net.ssl.SSLContext;
-import javax.net.ssl.SSLException;
-import javax.net.ssl.SSLHandshakeException;
-import javax.security.auth.x500.X500Principal;
import java.io.IOException;
-import java.math.BigInteger;
import java.net.BindException;
import java.net.URI;
import java.nio.ByteBuffer;
import java.nio.charset.StandardCharsets;
-import java.nio.file.Files;
import java.nio.file.Path;
-import java.security.KeyPair;
-import java.security.PrivateKey;
-import java.security.cert.X509Certificate;
import java.time.Duration;
import java.time.Instant;
import java.time.temporal.ChronoUnit;
@@ -85,10 +61,7 @@ import java.util.List;
import java.util.Map;
import java.util.TreeMap;
import java.util.UUID;
-import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
-import java.util.logging.Level;
-import java.util.logging.Logger;
import java.util.regex.Pattern;
import static com.yahoo.jdisc.Response.Status.GATEWAY_TIMEOUT;
@@ -105,8 +78,8 @@ import static com.yahoo.jdisc.http.HttpHeaders.Names.X_DISABLE_CHUNKING;
import static com.yahoo.jdisc.http.HttpHeaders.Values.APPLICATION_X_WWW_FORM_URLENCODED;
import static com.yahoo.jdisc.http.HttpHeaders.Values.CLOSE;
import static com.yahoo.jdisc.http.server.jetty.SimpleHttpClient.ResponseValidator;
-import static com.yahoo.security.KeyAlgorithm.EC;
-import static com.yahoo.security.SignatureAlgorithm.SHA256_WITH_ECDSA;
+import static com.yahoo.jdisc.http.server.jetty.Utils.createSslTestDriver;
+import static com.yahoo.jdisc.http.server.jetty.Utils.generatePrivateKeyAndCertificate;
import static org.cthul.matchers.CthulMatchers.containsPattern;
import static org.cthul.matchers.CthulMatchers.matchesPattern;
import static org.hamcrest.CoreMatchers.containsString;
@@ -114,13 +87,10 @@ import static org.hamcrest.CoreMatchers.instanceOf;
import static org.hamcrest.CoreMatchers.is;
import static org.hamcrest.CoreMatchers.startsWith;
import static org.hamcrest.MatcherAssert.assertThat;
-import static org.hamcrest.Matchers.anyOf;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotEquals;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
-import static org.junit.Assume.assumeTrue;
import static org.mockito.Mockito.atLeast;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.verify;
@@ -133,8 +103,6 @@ import static org.mockito.Mockito.when;
*/
public class HttpServerTest {
- private static final Logger log = Logger.getLogger(HttpServerTest.class.getName());
-
@Rule
public TemporaryFolder tmpFolder = new TemporaryFolder();
@@ -666,205 +634,6 @@ public class HttpServerTest {
}
@Test
- public void requireThatMetricIsIncrementedWhenClientIsMissingCertificateOnHandshake() throws IOException {
- Path privateKeyFile = tmpFolder.newFile().toPath();
- Path certificateFile = tmpFolder.newFile().toPath();
- generatePrivateKeyAndCertificate(privateKeyFile, certificateFile);
- var metricConsumer = new MetricConsumerMock();
- InMemoryConnectionLog connectionLog = new InMemoryConnectionLog();
- JettyTestDriver driver = createSslTestDriver(certificateFile, privateKeyFile, metricConsumer, connectionLog);
-
- SSLContext clientCtx = new SslContextBuilder()
- .withTrustStore(certificateFile)
- .build();
- assertHttpsRequestTriggersSslHandshakeException(
- driver, clientCtx, null, null, "Received fatal alert: bad_certificate");
- verify(metricConsumer.mockitoMock(), atLeast(1))
- .add(MetricDefinitions.SSL_HANDSHAKE_FAILURE_MISSING_CLIENT_CERT, 1L, MetricConsumerMock.STATIC_CONTEXT);
- assertTrue(driver.close());
- Assertions.assertThat(connectionLog.logEntries()).hasSize(1);
- assertSslHandshakeFailurePresent(
- connectionLog.logEntries().get(0), SSLHandshakeException.class, SslHandshakeFailure.MISSING_CLIENT_CERT.failureType());
- }
-
- @Test
- public void requireThatMetricIsIncrementedWhenClientUsesIncompatibleTlsVersion() throws IOException {
- Path privateKeyFile = tmpFolder.newFile().toPath();
- Path certificateFile = tmpFolder.newFile().toPath();
- generatePrivateKeyAndCertificate(privateKeyFile, certificateFile);
- var metricConsumer = new MetricConsumerMock();
- InMemoryConnectionLog connectionLog = new InMemoryConnectionLog();
- JettyTestDriver driver = createSslTestDriver(certificateFile, privateKeyFile, metricConsumer, connectionLog);
-
- SSLContext clientCtx = new SslContextBuilder()
- .withTrustStore(certificateFile)
- .withKeyStore(privateKeyFile, certificateFile)
- .build();
-
- boolean tlsv11Enabled = List.of(clientCtx.getDefaultSSLParameters().getProtocols()).contains("TLSv1.1");
- assumeTrue("TLSv1.1 must be enabled in installed JDK", tlsv11Enabled);
-
- assertHttpsRequestTriggersSslHandshakeException(driver, clientCtx, "TLSv1.1", null, "protocol");
- verify(metricConsumer.mockitoMock(), atLeast(1))
- .add(MetricDefinitions.SSL_HANDSHAKE_FAILURE_INCOMPATIBLE_PROTOCOLS, 1L, MetricConsumerMock.STATIC_CONTEXT);
- assertTrue(driver.close());
- Assertions.assertThat(connectionLog.logEntries()).hasSize(1);
- assertSslHandshakeFailurePresent(
- connectionLog.logEntries().get(0), SSLHandshakeException.class, SslHandshakeFailure.INCOMPATIBLE_PROTOCOLS.failureType());
- }
-
- @Test
- public void requireThatMetricIsIncrementedWhenClientUsesIncompatibleCiphers() throws IOException {
- Path privateKeyFile = tmpFolder.newFile().toPath();
- Path certificateFile = tmpFolder.newFile().toPath();
- generatePrivateKeyAndCertificate(privateKeyFile, certificateFile);
- var metricConsumer = new MetricConsumerMock();
- InMemoryConnectionLog connectionLog = new InMemoryConnectionLog();
- JettyTestDriver driver = createSslTestDriver(certificateFile, privateKeyFile, metricConsumer, connectionLog);
-
- SSLContext clientCtx = new SslContextBuilder()
- .withTrustStore(certificateFile)
- .withKeyStore(privateKeyFile, certificateFile)
- .build();
-
- assertHttpsRequestTriggersSslHandshakeException(
- driver, clientCtx, null, "TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA", "Received fatal alert: handshake_failure");
- verify(metricConsumer.mockitoMock(), atLeast(1))
- .add(MetricDefinitions.SSL_HANDSHAKE_FAILURE_INCOMPATIBLE_CIPHERS, 1L, MetricConsumerMock.STATIC_CONTEXT);
- assertTrue(driver.close());
- Assertions.assertThat(connectionLog.logEntries()).hasSize(1);
- assertSslHandshakeFailurePresent(
- connectionLog.logEntries().get(0), SSLHandshakeException.class, SslHandshakeFailure.INCOMPATIBLE_CIPHERS.failureType());
- }
-
- @Test
- public void requireThatMetricIsIncrementedWhenClientUsesInvalidCertificateInHandshake() throws IOException {
- Path serverPrivateKeyFile = tmpFolder.newFile().toPath();
- Path serverCertificateFile = tmpFolder.newFile().toPath();
- generatePrivateKeyAndCertificate(serverPrivateKeyFile, serverCertificateFile);
- var metricConsumer = new MetricConsumerMock();
- InMemoryConnectionLog connectionLog = new InMemoryConnectionLog();
- JettyTestDriver driver = createSslTestDriver(serverCertificateFile, serverPrivateKeyFile, metricConsumer, connectionLog);
-
- Path clientPrivateKeyFile = tmpFolder.newFile().toPath();
- Path clientCertificateFile = tmpFolder.newFile().toPath();
- generatePrivateKeyAndCertificate(clientPrivateKeyFile, clientCertificateFile);
-
- SSLContext clientCtx = new SslContextBuilder()
- .withKeyStore(clientPrivateKeyFile, clientCertificateFile)
- .withTrustStore(serverCertificateFile)
- .build();
-
- assertHttpsRequestTriggersSslHandshakeException(
- driver, clientCtx, null, null, "Received fatal alert: certificate_unknown");
- verify(metricConsumer.mockitoMock(), atLeast(1))
- .add(MetricDefinitions.SSL_HANDSHAKE_FAILURE_INVALID_CLIENT_CERT, 1L, MetricConsumerMock.STATIC_CONTEXT);
- assertTrue(driver.close());
- Assertions.assertThat(connectionLog.logEntries()).hasSize(1);
- assertSslHandshakeFailurePresent(
- connectionLog.logEntries().get(0), SSLHandshakeException.class, SslHandshakeFailure.INVALID_CLIENT_CERT.failureType());
- }
-
- @Test
- public void requireThatMetricIsIncrementedWhenClientUsesExpiredCertificateInHandshake() throws IOException {
- Path rootPrivateKeyFile = tmpFolder.newFile().toPath();
- Path rootCertificateFile = tmpFolder.newFile().toPath();
- Path privateKeyFile = tmpFolder.newFile().toPath();
- Path certificateFile = tmpFolder.newFile().toPath();
- Instant notAfter = Instant.now().minus(100, ChronoUnit.DAYS);
- generatePrivateKeyAndCertificate(rootPrivateKeyFile, rootCertificateFile, privateKeyFile, certificateFile, notAfter);
- var metricConsumer = new MetricConsumerMock();
- InMemoryConnectionLog connectionLog = new InMemoryConnectionLog();
- JettyTestDriver driver = createSslTestDriver(rootCertificateFile, rootPrivateKeyFile, metricConsumer, connectionLog);
-
- SSLContext clientCtx = new SslContextBuilder()
- .withTrustStore(rootCertificateFile)
- .withKeyStore(privateKeyFile, certificateFile)
- .build();
-
- assertHttpsRequestTriggersSslHandshakeException(
- driver, clientCtx, null, null, "Received fatal alert: certificate_unknown");
- verify(metricConsumer.mockitoMock(), atLeast(1))
- .add(MetricDefinitions.SSL_HANDSHAKE_FAILURE_EXPIRED_CLIENT_CERT, 1L, MetricConsumerMock.STATIC_CONTEXT);
- assertTrue(driver.close());
- Assertions.assertThat(connectionLog.logEntries()).hasSize(1);
-
- }
-
- @Test
- public void requireThatProxyProtocolIsAcceptedAndActualRemoteAddressStoredInAccessLog() throws Exception {
- Path privateKeyFile = tmpFolder.newFile().toPath();
- Path certificateFile = tmpFolder.newFile().toPath();
- generatePrivateKeyAndCertificate(privateKeyFile, certificateFile);
- InMemoryRequestLog requestLogMock = new InMemoryRequestLog();
- InMemoryConnectionLog connectionLog = new InMemoryConnectionLog();
- JettyTestDriver driver = createSslWithProxyProtocolTestDriver(certificateFile, privateKeyFile, requestLogMock, /*mixedMode*/connectionLog, false);
-
- String proxiedRemoteAddress = "192.168.0.100";
- int proxiedRemotePort = 12345;
- sendJettyClientRequest(driver, certificateFile, new V1.Tag(proxiedRemoteAddress, proxiedRemotePort));
- sendJettyClientRequest(driver, certificateFile, new V2.Tag(proxiedRemoteAddress, proxiedRemotePort));
- assertTrue(driver.close());
-
- assertEquals(2, requestLogMock.entries().size());
- assertLogEntryHasRemote(requestLogMock.entries().get(0), proxiedRemoteAddress, proxiedRemotePort);
- assertLogEntryHasRemote(requestLogMock.entries().get(1), proxiedRemoteAddress, proxiedRemotePort);
- Assertions.assertThat(connectionLog.logEntries()).hasSize(2);
- assertLogEntryHasRemote(connectionLog.logEntries().get(0), proxiedRemoteAddress, proxiedRemotePort);
- assertEquals("v1", connectionLog.logEntries().get(0).proxyProtocolVersion().get());
- assertLogEntryHasRemote(connectionLog.logEntries().get(1), proxiedRemoteAddress, proxiedRemotePort);
- assertEquals("v2", connectionLog.logEntries().get(1).proxyProtocolVersion().get());
- }
-
- @Test
- public void requireThatConnectorWithProxyProtocolMixedEnabledAcceptsBothProxyProtocolAndHttps() throws Exception {
- Path privateKeyFile = tmpFolder.newFile().toPath();
- Path certificateFile = tmpFolder.newFile().toPath();
- generatePrivateKeyAndCertificate(privateKeyFile, certificateFile);
- InMemoryRequestLog requestLogMock = new InMemoryRequestLog();
- InMemoryConnectionLog connectionLog = new InMemoryConnectionLog();
- JettyTestDriver driver = createSslWithProxyProtocolTestDriver(certificateFile, privateKeyFile, requestLogMock, /*mixedMode*/connectionLog, true);
-
- String proxiedRemoteAddress = "192.168.0.100";
- sendJettyClientRequest(driver, certificateFile, null);
- sendJettyClientRequest(driver, certificateFile, new V1.Tag(proxiedRemoteAddress, 12345));
- sendJettyClientRequest(driver, certificateFile, new V2.Tag(proxiedRemoteAddress, 12345));
- assertTrue(driver.close());
-
- assertEquals(3, requestLogMock.entries().size());
- assertLogEntryHasRemote(requestLogMock.entries().get(0), "127.0.0.1", 0);
- assertLogEntryHasRemote(requestLogMock.entries().get(1), proxiedRemoteAddress, 0);
- assertLogEntryHasRemote(requestLogMock.entries().get(2), proxiedRemoteAddress, 0);
- Assertions.assertThat(connectionLog.logEntries()).hasSize(3);
- assertLogEntryHasRemote(connectionLog.logEntries().get(0), null, 0);
- assertLogEntryHasRemote(connectionLog.logEntries().get(1), proxiedRemoteAddress, 12345);
- assertLogEntryHasRemote(connectionLog.logEntries().get(2), proxiedRemoteAddress, 12345);
- }
-
- @Test
- public void requireThatJdiscLocalPortPropertyIsNotOverriddenByProxyProtocol() throws Exception {
- Path privateKeyFile = tmpFolder.newFile().toPath();
- Path certificateFile = tmpFolder.newFile().toPath();
- generatePrivateKeyAndCertificate(privateKeyFile, certificateFile);
- InMemoryRequestLog requestLogMock = new InMemoryRequestLog();
- InMemoryConnectionLog connectionLog = new InMemoryConnectionLog();
- JettyTestDriver driver = createSslWithProxyProtocolTestDriver(certificateFile, privateKeyFile, requestLogMock, connectionLog, /*mixedMode*/false);
-
- String proxiedRemoteAddress = "192.168.0.100";
- int proxiedRemotePort = 12345;
- String proxyLocalAddress = "10.0.0.10";
- int proxyLocalPort = 23456;
- V2.Tag v2Tag = new V2.Tag(V2.Tag.Command.PROXY, null, V2.Tag.Protocol.STREAM,
- proxiedRemoteAddress, proxiedRemotePort, proxyLocalAddress, proxyLocalPort, null);
- ContentResponse response = sendJettyClientRequest(driver, certificateFile, v2Tag);
- assertTrue(driver.close());
-
- int clientPort = Integer.parseInt(response.getHeaders().get("Jdisc-Local-Port"));
- assertNotEquals(proxyLocalPort, clientPort);
- assertNotEquals(proxyLocalPort, connectionLog.logEntries().get(0).localPort().get().intValue());
- }
-
- @Test
public void requireThatConnectionIsTrackedInConnectionLog() throws Exception {
Path privateKeyFile = tmpFolder.newFile().toPath();
Path certificateFile = tmpFolder.newFile().toPath();
@@ -932,41 +701,6 @@ public class HttpServerTest {
.set(MetricDefinitions.REQUESTS_PER_CONNECTION, 1L, MetricConsumerMock.STATIC_CONTEXT);
}
- private ContentResponse sendJettyClientRequest(JettyTestDriver testDriver, Path certificateFile, Object tag)
- throws Exception {
- HttpClient client = createJettyHttpClient(certificateFile);
- try {
- int maxAttempts = 3;
- for (int attempt = 0; attempt < maxAttempts; attempt++) {
- try {
- ContentResponse response = client.newRequest(URI.create("https://localhost:" + testDriver.server().getListenPort() + "/"))
- .tag(tag)
- .send();
- assertEquals(200, response.getStatus());
- return response;
- } catch (ExecutionException e) {
- // Retry when the server closes the connection before the TLS handshake is completed. This have been observed in CI.
- // We have been unable to reproduce this locally. The cause is therefor currently unknown.
- log.log(Level.WARNING, String.format("Attempt %d failed: %s", attempt, e.getMessage()), e);
- Thread.sleep(10);
- }
- }
- throw new AssertionError("Failed to send request, see log for details");
- } finally {
- client.stop();
- }
- }
-
- // Using Jetty's http client as Apache httpclient does not support the proxy-protocol v1/v2.
- private static HttpClient createJettyHttpClient(Path certificateFile) throws Exception {
- SslContextFactory.Client clientSslCtxFactory = new SslContextFactory.Client();
- clientSslCtxFactory.setHostnameVerifier(NoopHostnameVerifier.INSTANCE);
- clientSslCtxFactory.setSslContext(new SslContextBuilder().withTrustStore(certificateFile).build());
-
- HttpClient client = new HttpClient(clientSslCtxFactory);
- client.start();
- return client;
- }
private static CloseableHttpAsyncClient createHttp2Client(JettyTestDriver driver) {
TlsStrategy tlsStrategy = ClientTlsStrategyBuilder.create()
@@ -980,58 +714,6 @@ public class HttpServerTest {
return client;
}
- private static void assertLogEntryHasRemote(RequestLogEntry entry, String expectedAddress, int expectedPort) {
- assertEquals(expectedAddress, entry.peerAddress().get());
- if (expectedPort > 0) {
- assertEquals(expectedPort, entry.peerPort().getAsInt());
- }
- }
-
- private static void assertLogEntryHasRemote(ConnectionLogEntry entry, String expectedAddress, int expectedPort) {
- if (expectedAddress != null) {
- Assertions.assertThat(entry.remoteAddress()).hasValue(expectedAddress);
- } else {
- Assertions.assertThat(entry.remoteAddress()).isEmpty();
- }
- if (expectedPort > 0) {
- Assertions.assertThat(entry.remotePort()).hasValue(expectedPort);
- } else {
- Assertions.assertThat(entry.remotePort()).isEmpty();
- }
- }
-
- private static void assertSslHandshakeFailurePresent(
- ConnectionLogEntry entry, Class<? extends SSLHandshakeException> expectedException, String expectedType) {
- Assertions.assertThat(entry.sslHandshakeFailure()).isPresent();
- ConnectionLogEntry.SslHandshakeFailure failure = entry.sslHandshakeFailure().get();
- assertEquals(expectedType, failure.type());
- ExceptionEntry exceptionEntry = failure.exceptionChain().get(0);
- assertEquals(expectedException.getName(), exceptionEntry.name());
- }
-
- private static JettyTestDriver createSslWithProxyProtocolTestDriver(
- Path certificateFile, Path privateKeyFile, RequestLog requestLog,
- ConnectionLog connectionLog, boolean mixedMode) {
- ConnectorConfig.Builder connectorConfig = new ConnectorConfig.Builder()
- .http2Enabled(true)
- .proxyProtocol(new ConnectorConfig.ProxyProtocol.Builder()
- .enabled(true)
- .mixedMode(mixedMode))
- .ssl(new ConnectorConfig.Ssl.Builder()
- .enabled(true)
- .privateKeyFile(privateKeyFile.toString())
- .certificateFile(certificateFile.toString())
- .caCertificateFile(certificateFile.toString()));
- return JettyTestDriver.newConfiguredInstance(
- new EchoRequestHandler(),
- new ServerConfig.Builder().connectionLog(new ServerConfig.ConnectionLog.Builder().enabled(true)),
- connectorConfig,
- binder -> {
- binder.bind(RequestLog.class).toInstance(requestLog);
- binder.bind(ConnectionLog.class).toInstance(connectionLog);
- });
- }
-
private static JettyTestDriver createSslWithTlsClientAuthenticationEnforcer(Path certificateFile, Path privateKeyFile) {
ConnectorConfig.Builder connectorConfig = new ConnectorConfig.Builder()
.tlsClientAuthEnforcer(
@@ -1051,63 +733,6 @@ public class HttpServerTest {
binder -> {});
}
- private static JettyTestDriver createSslTestDriver(
- Path serverCertificateFile, Path serverPrivateKeyFile, MetricConsumerMock metricConsumer, InMemoryConnectionLog connectionLog) throws IOException {
- Module extraModule = binder -> {
- binder.bind(MetricConsumer.class).toInstance(metricConsumer.mockitoMock());
- binder.bind(ConnectionLog.class).toInstance(connectionLog);
- };
- return JettyTestDriver.newInstanceWithSsl(
- new EchoRequestHandler(), serverCertificateFile, serverPrivateKeyFile, TlsClientAuth.NEED, extraModule);
- }
-
- private static void assertHttpsRequestTriggersSslHandshakeException(
- JettyTestDriver testDriver,
- SSLContext sslContext,
- String protocolOverride,
- String cipherOverride,
- String expectedExceptionSubstring) throws IOException {
- List<String> protocols = protocolOverride != null ? List.of(protocolOverride) : null;
- List<String> ciphers = cipherOverride != null ? List.of(cipherOverride) : null;
- try (var client = new SimpleHttpClient(sslContext, protocols, ciphers, testDriver.server().getListenPort(), false)) {
- client.get("/status.html");
- fail("SSLHandshakeException expected");
- } catch (SSLHandshakeException e) {
- assertThat(e.getMessage(), containsString(expectedExceptionSubstring));
- } catch (SSLException e) {
- // This exception is thrown if Apache httpclient's write thread detects the handshake failure before the read thread.
- log.log(Level.WARNING, "Client failed to get a proper TLS handshake response: " + e.getMessage(), e);
- // Only ignore a subset of exceptions
- assertThat(e.getMessage(), anyOf(containsString("readHandshakeRecord"), containsString("Broken pipe")));
- }
- }
-
- private static void generatePrivateKeyAndCertificate(Path privateKeyFile, Path certificateFile) throws IOException {
- KeyPair keyPair = KeyUtils.generateKeypair(EC);
- Files.writeString(privateKeyFile, KeyUtils.toPem(keyPair.getPrivate()));
-
- X509Certificate certificate = X509CertificateBuilder
- .fromKeypair(
- keyPair, new X500Principal("CN=localhost"), Instant.EPOCH, Instant.EPOCH.plus(100_000, ChronoUnit.DAYS), SHA256_WITH_ECDSA, BigInteger.ONE)
- .build();
- Files.writeString(certificateFile, X509CertificateUtils.toPem(certificate));
- }
-
- private static void generatePrivateKeyAndCertificate(Path rootPrivateKeyFile, Path rootCertificateFile,
- Path privateKeyFile, Path certificateFile, Instant notAfter) throws IOException {
- generatePrivateKeyAndCertificate(rootPrivateKeyFile, rootCertificateFile);
- X509Certificate rootCertificate = X509CertificateUtils.fromPem(Files.readString(rootCertificateFile));
- PrivateKey privateKey = KeyUtils.fromPemEncodedPrivateKey(Files.readString(rootPrivateKeyFile));
-
- KeyPair keyPair = KeyUtils.generateKeypair(EC);
- Files.writeString(privateKeyFile, KeyUtils.toPem(keyPair.getPrivate()));
- Pkcs10Csr csr = Pkcs10CsrBuilder.fromKeypair(new X500Principal("CN=myclient"), keyPair, SHA256_WITH_ECDSA).build();
- X509Certificate certificate = X509CertificateBuilder
- .fromCsr(csr, rootCertificate.getSubjectX500Principal(), Instant.EPOCH, notAfter, privateKey, SHA256_WITH_ECDSA, BigInteger.ONE)
- .build();
- Files.writeString(certificateFile, X509CertificateUtils.toPem(certificate));
- }
-
private static RequestHandler mockRequestHandler() {
final RequestHandler mockRequestHandler = mock(RequestHandler.class);
when(mockRequestHandler.refer()).thenReturn(References.NOOP_REFERENCE);
@@ -1237,17 +862,6 @@ public class HttpServerTest {
}
}
- private static class EchoRequestHandler extends AbstractRequestHandler {
-
- @Override
- public ContentChannel handleRequest(final Request request, final ResponseHandler handler) {
- int port = request.getUri().getPort();
- Response response = new Response(OK);
- response.headers().put("Jdisc-Local-Port", Integer.toString(port));
- return handler.handleResponse(response);
- }
- }
-
private static class OkRequestHandler extends AbstractRequestHandler {
@Override
public ContentChannel handleRequest(Request request, ResponseHandler handler) {
diff --git a/container-core/src/test/java/com/yahoo/jdisc/http/server/jetty/ProxyProtocolTest.java b/container-core/src/test/java/com/yahoo/jdisc/http/server/jetty/ProxyProtocolTest.java
new file mode 100644
index 00000000000..d29abea024e
--- /dev/null
+++ b/container-core/src/test/java/com/yahoo/jdisc/http/server/jetty/ProxyProtocolTest.java
@@ -0,0 +1,198 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.jdisc.http.server.jetty;
+
+import com.yahoo.container.logging.ConnectionLog;
+import com.yahoo.container.logging.ConnectionLogEntry;
+import com.yahoo.container.logging.RequestLog;
+import com.yahoo.container.logging.RequestLogEntry;
+import com.yahoo.jdisc.http.ConnectorConfig;
+import com.yahoo.jdisc.http.ServerConfig;
+import com.yahoo.security.SslContextBuilder;
+import org.apache.hc.client5.http.ssl.NoopHostnameVerifier;
+import org.assertj.core.api.Assertions;
+import org.eclipse.jetty.client.HttpClient;
+import org.eclipse.jetty.client.ProxyProtocolClientConnectionFactory;
+import org.eclipse.jetty.client.api.ContentResponse;
+import org.eclipse.jetty.util.ssl.SslContextFactory;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+
+import java.io.IOException;
+import java.net.URI;
+import java.nio.file.Path;
+import java.util.concurrent.ExecutionException;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+import static com.yahoo.jdisc.http.server.jetty.Utils.generatePrivateKeyAndCertificate;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+/**
+ * @author bjorncs
+ */
+class ProxyProtocolTest {
+
+ private static final Logger log = Logger.getLogger(ProxyProtocolTest.class.getName());
+
+ private static Path privateKeyFile;
+ private static Path certificateFile;
+ private InMemoryConnectionLog connectionLog;
+ private InMemoryRequestLog requestLogMock;
+ private JettyTestDriver driver;
+
+ @BeforeAll
+ static void generateCrypto(@TempDir Path tmpFolder) throws IOException {
+ privateKeyFile = tmpFolder.resolve("key.pem");
+ certificateFile = tmpFolder.resolve("cert.pem");
+ generatePrivateKeyAndCertificate(privateKeyFile, certificateFile);
+ }
+
+ @BeforeEach
+ void initializeServer() {
+ requestLogMock = new InMemoryRequestLog();
+ connectionLog = new InMemoryConnectionLog();
+ driver = createSslWithProxyProtocolTestDriver(certificateFile, privateKeyFile, requestLogMock, /*mixedMode*/connectionLog, false);
+ }
+
+ @Test
+ void requireThatProxyProtocolIsAcceptedAndActualRemoteAddressStoredInAccessLog() throws Exception {
+ String proxiedRemoteAddress = "192.168.0.100";
+ int proxiedRemotePort = 12345;
+ sendJettyClientRequest(driver, certificateFile, new ProxyProtocolClientConnectionFactory.V1.Tag(proxiedRemoteAddress, proxiedRemotePort));
+ sendJettyClientRequest(driver, certificateFile, new ProxyProtocolClientConnectionFactory.V2.Tag(proxiedRemoteAddress, proxiedRemotePort));
+ assertTrue(driver.close());
+
+ assertEquals(2, requestLogMock.entries().size());
+ assertLogEntryHasRemote(requestLogMock.entries().get(0), proxiedRemoteAddress, proxiedRemotePort);
+ assertLogEntryHasRemote(requestLogMock.entries().get(1), proxiedRemoteAddress, proxiedRemotePort);
+ Assertions.assertThat(connectionLog.logEntries()).hasSize(2);
+ assertLogEntryHasRemote(connectionLog.logEntries().get(0), proxiedRemoteAddress, proxiedRemotePort);
+ assertEquals("v1", connectionLog.logEntries().get(0).proxyProtocolVersion().get());
+ assertLogEntryHasRemote(connectionLog.logEntries().get(1), proxiedRemoteAddress, proxiedRemotePort);
+ assertEquals("v2", connectionLog.logEntries().get(1).proxyProtocolVersion().get());
+ }
+
+ @Test
+ void requireThatConnectorWithProxyProtocolMixedEnabledAcceptsBothProxyProtocolAndHttps() throws Exception {
+ generatePrivateKeyAndCertificate(privateKeyFile, certificateFile);
+ InMemoryRequestLog requestLogMock = new InMemoryRequestLog();
+ InMemoryConnectionLog connectionLog = new InMemoryConnectionLog();
+ JettyTestDriver driver = createSslWithProxyProtocolTestDriver(certificateFile, privateKeyFile, requestLogMock, /*mixedMode*/connectionLog, true);
+
+ String proxiedRemoteAddress = "192.168.0.100";
+ sendJettyClientRequest(driver, certificateFile, null);
+ sendJettyClientRequest(driver, certificateFile, new ProxyProtocolClientConnectionFactory.V1.Tag(proxiedRemoteAddress, 12345));
+ sendJettyClientRequest(driver, certificateFile, new ProxyProtocolClientConnectionFactory.V2.Tag(proxiedRemoteAddress, 12345));
+ assertTrue(driver.close());
+
+ assertEquals(3, requestLogMock.entries().size());
+ assertLogEntryHasRemote(requestLogMock.entries().get(0), "127.0.0.1", 0);
+ assertLogEntryHasRemote(requestLogMock.entries().get(1), proxiedRemoteAddress, 0);
+ assertLogEntryHasRemote(requestLogMock.entries().get(2), proxiedRemoteAddress, 0);
+ Assertions.assertThat(connectionLog.logEntries()).hasSize(3);
+ assertLogEntryHasRemote(connectionLog.logEntries().get(0), null, 0);
+ assertLogEntryHasRemote(connectionLog.logEntries().get(1), proxiedRemoteAddress, 12345);
+ assertLogEntryHasRemote(connectionLog.logEntries().get(2), proxiedRemoteAddress, 12345);
+ }
+
+ @Test
+ void requireThatJdiscLocalPortPropertyIsNotOverriddenByProxyProtocol() throws Exception {
+ String proxiedRemoteAddress = "192.168.0.100";
+ int proxiedRemotePort = 12345;
+ String proxyLocalAddress = "10.0.0.10";
+ int proxyLocalPort = 23456;
+ ProxyProtocolClientConnectionFactory.V2.Tag v2Tag = new ProxyProtocolClientConnectionFactory.V2.Tag(ProxyProtocolClientConnectionFactory.V2.Tag.Command.PROXY, null, ProxyProtocolClientConnectionFactory.V2.Tag.Protocol.STREAM,
+ proxiedRemoteAddress, proxiedRemotePort, proxyLocalAddress, proxyLocalPort, null);
+ ContentResponse response = sendJettyClientRequest(driver, certificateFile, v2Tag);
+ assertTrue(driver.close());
+
+ int clientPort = Integer.parseInt(response.getHeaders().get("Jdisc-Local-Port"));
+ assertNotEquals(proxyLocalPort, clientPort);
+ assertNotEquals(proxyLocalPort, connectionLog.logEntries().get(0).localPort().get().intValue());
+ }
+
+ private static JettyTestDriver createSslWithProxyProtocolTestDriver(
+ Path certificateFile, Path privateKeyFile, RequestLog requestLog,
+ ConnectionLog connectionLog, boolean mixedMode) {
+ ConnectorConfig.Builder connectorConfig = new ConnectorConfig.Builder()
+ .http2Enabled(true)
+ .proxyProtocol(new ConnectorConfig.ProxyProtocol.Builder()
+ .enabled(true)
+ .mixedMode(mixedMode))
+ .ssl(new ConnectorConfig.Ssl.Builder()
+ .enabled(true)
+ .privateKeyFile(privateKeyFile.toString())
+ .certificateFile(certificateFile.toString())
+ .caCertificateFile(certificateFile.toString()));
+ return JettyTestDriver.newConfiguredInstance(
+ new EchoRequestHandler(),
+ new ServerConfig.Builder().connectionLog(new ServerConfig.ConnectionLog.Builder().enabled(true)),
+ connectorConfig,
+ binder -> {
+ binder.bind(RequestLog.class).toInstance(requestLog);
+ binder.bind(ConnectionLog.class).toInstance(connectionLog);
+ });
+ }
+
+ private ContentResponse sendJettyClientRequest(JettyTestDriver testDriver, Path certificateFile, Object tag)
+ throws Exception {
+ HttpClient client = createJettyHttpClient(certificateFile);
+ try {
+ int maxAttempts = 3;
+ for (int attempt = 0; attempt < maxAttempts; attempt++) {
+ try {
+ ContentResponse response = client.newRequest(URI.create("https://localhost:" + testDriver.server().getListenPort() + "/"))
+ .tag(tag)
+ .send();
+ assertEquals(200, response.getStatus());
+ return response;
+ } catch (ExecutionException e) {
+ // Retry when the server closes the connection before the TLS handshake is completed. This have been observed in CI.
+ // We have been unable to reproduce this locally. The cause is therefor currently unknown.
+ log.log(Level.WARNING, String.format("Attempt %d failed: %s", attempt, e.getMessage()), e);
+ Thread.sleep(10);
+ }
+ }
+ throw new AssertionError("Failed to send request, see log for details");
+ } finally {
+ client.stop();
+ }
+ }
+
+ // Using Jetty's http client as Apache httpclient does not support the proxy-protocol v1/v2.
+ private static HttpClient createJettyHttpClient(Path certificateFile) throws Exception {
+ SslContextFactory.Client clientSslCtxFactory = new SslContextFactory.Client();
+ clientSslCtxFactory.setHostnameVerifier(NoopHostnameVerifier.INSTANCE);
+ clientSslCtxFactory.setSslContext(new SslContextBuilder().withTrustStore(certificateFile).build());
+
+ HttpClient client = new HttpClient(clientSslCtxFactory);
+ client.start();
+ return client;
+ }
+
+ private static void assertLogEntryHasRemote(RequestLogEntry entry, String expectedAddress, int expectedPort) {
+ assertEquals(expectedAddress, entry.peerAddress().get());
+ if (expectedPort > 0) {
+ assertEquals(expectedPort, entry.peerPort().getAsInt());
+ }
+ }
+
+ private static void assertLogEntryHasRemote(ConnectionLogEntry entry, String expectedAddress, int expectedPort) {
+ if (expectedAddress != null) {
+ Assertions.assertThat(entry.remoteAddress()).hasValue(expectedAddress);
+ } else {
+ Assertions.assertThat(entry.remoteAddress()).isEmpty();
+ }
+ if (expectedPort > 0) {
+ Assertions.assertThat(entry.remotePort()).hasValue(expectedPort);
+ } else {
+ Assertions.assertThat(entry.remotePort()).isEmpty();
+ }
+ }
+
+
+}
diff --git a/container-core/src/test/java/com/yahoo/jdisc/http/server/jetty/SslHandshakeMetricsTest.java b/container-core/src/test/java/com/yahoo/jdisc/http/server/jetty/SslHandshakeMetricsTest.java
new file mode 100644
index 00000000000..0f71dd87d00
--- /dev/null
+++ b/container-core/src/test/java/com/yahoo/jdisc/http/server/jetty/SslHandshakeMetricsTest.java
@@ -0,0 +1,189 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.jdisc.http.server.jetty;
+
+import com.yahoo.container.logging.ConnectionLogEntry;
+import com.yahoo.security.SslContextBuilder;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+
+import javax.net.ssl.SSLContext;
+import javax.net.ssl.SSLException;
+import javax.net.ssl.SSLHandshakeException;
+import java.io.IOException;
+import java.nio.file.Path;
+import java.time.Instant;
+import java.time.temporal.ChronoUnit;
+import java.util.List;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+import static com.yahoo.jdisc.http.server.jetty.Utils.createSslTestDriver;
+import static com.yahoo.jdisc.http.server.jetty.Utils.generatePrivateKeyAndCertificate;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.junit.jupiter.api.Assertions.fail;
+import static org.junit.jupiter.api.Assumptions.assumeTrue;
+import static org.mockito.Mockito.atLeast;
+import static org.mockito.Mockito.verify;
+
+/**
+ * @author bjorncs
+ */
+class SslHandshakeMetricsTest {
+
+ private static final Logger log = Logger.getLogger(SslHandshakeMetricsTest.class.getName());
+ private static Path privateKeyFile;
+ private static Path certificateFile;
+
+ @BeforeAll
+ static void generateCrypto(@TempDir Path tmpFolder) throws IOException {
+ privateKeyFile = tmpFolder.resolve("private-key.pem");
+ certificateFile = tmpFolder.resolve("certificate.pem");
+ generatePrivateKeyAndCertificate(privateKeyFile, certificateFile);
+ }
+
+ @Test
+ void requireThatMetricIsIncrementedWhenClientIsMissingCertificateOnHandshake() throws IOException {
+ var metricConsumer = new MetricConsumerMock();
+ InMemoryConnectionLog connectionLog = new InMemoryConnectionLog();
+ JettyTestDriver driver = createSslTestDriver(certificateFile, privateKeyFile, metricConsumer, connectionLog);
+
+ SSLContext clientCtx = new SslContextBuilder()
+ .withTrustStore(certificateFile)
+ .build();
+ assertHttpsRequestTriggersSslHandshakeException(
+ driver, clientCtx, null, null, "Received fatal alert: bad_certificate");
+ verify(metricConsumer.mockitoMock(), atLeast(1))
+ .add(MetricDefinitions.SSL_HANDSHAKE_FAILURE_MISSING_CLIENT_CERT, 1L, MetricConsumerMock.STATIC_CONTEXT);
+ assertTrue(driver.close());
+ assertThat(connectionLog.logEntries()).hasSize(1);
+ assertSslHandshakeFailurePresent(
+ connectionLog.logEntries().get(0), SSLHandshakeException.class, SslHandshakeFailure.MISSING_CLIENT_CERT.failureType());
+ }
+
+ @Test
+ void requireThatMetricIsIncrementedWhenClientUsesIncompatibleTlsVersion() throws IOException {
+ var metricConsumer = new MetricConsumerMock();
+ InMemoryConnectionLog connectionLog = new InMemoryConnectionLog();
+ JettyTestDriver driver = createSslTestDriver(certificateFile, privateKeyFile, metricConsumer, connectionLog);
+
+ SSLContext clientCtx = new SslContextBuilder()
+ .withTrustStore(certificateFile)
+ .withKeyStore(privateKeyFile, certificateFile)
+ .build();
+
+ boolean tlsv11Enabled = List.of(clientCtx.getDefaultSSLParameters().getProtocols()).contains("TLSv1.1");
+ assumeTrue(tlsv11Enabled, "TLSv1.1 must be enabled in installed JDK");
+
+ assertHttpsRequestTriggersSslHandshakeException(driver, clientCtx, "TLSv1.1", null, "protocol");
+ verify(metricConsumer.mockitoMock(), atLeast(1))
+ .add(MetricDefinitions.SSL_HANDSHAKE_FAILURE_INCOMPATIBLE_PROTOCOLS, 1L, MetricConsumerMock.STATIC_CONTEXT);
+ assertTrue(driver.close());
+ assertThat(connectionLog.logEntries()).hasSize(1);
+ assertSslHandshakeFailurePresent(
+ connectionLog.logEntries().get(0), SSLHandshakeException.class, SslHandshakeFailure.INCOMPATIBLE_PROTOCOLS.failureType());
+ }
+
+ @Test
+ void requireThatMetricIsIncrementedWhenClientUsesIncompatibleCiphers() throws IOException {
+ var metricConsumer = new MetricConsumerMock();
+ InMemoryConnectionLog connectionLog = new InMemoryConnectionLog();
+ JettyTestDriver driver = createSslTestDriver(certificateFile, privateKeyFile, metricConsumer, connectionLog);
+
+ SSLContext clientCtx = new SslContextBuilder()
+ .withTrustStore(certificateFile)
+ .withKeyStore(privateKeyFile, certificateFile)
+ .build();
+
+ assertHttpsRequestTriggersSslHandshakeException(
+ driver, clientCtx, null, "TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA", "Received fatal alert: handshake_failure");
+ verify(metricConsumer.mockitoMock(), atLeast(1))
+ .add(MetricDefinitions.SSL_HANDSHAKE_FAILURE_INCOMPATIBLE_CIPHERS, 1L, MetricConsumerMock.STATIC_CONTEXT);
+ assertTrue(driver.close());
+ assertThat(connectionLog.logEntries()).hasSize(1);
+ assertSslHandshakeFailurePresent(
+ connectionLog.logEntries().get(0), SSLHandshakeException.class, SslHandshakeFailure.INCOMPATIBLE_CIPHERS.failureType());
+ }
+
+ @Test
+ void requireThatMetricIsIncrementedWhenClientUsesInvalidCertificateInHandshake(@TempDir Path tmpFolder) throws IOException {
+ var metricConsumer = new MetricConsumerMock();
+ InMemoryConnectionLog connectionLog = new InMemoryConnectionLog();
+ JettyTestDriver driver = createSslTestDriver(certificateFile, privateKeyFile, metricConsumer, connectionLog);
+
+ Path clientPrivateKeyFile = tmpFolder.resolve("client-key.pem");
+ Path clientCertificateFile = tmpFolder.resolve("client-cert.pem");
+ generatePrivateKeyAndCertificate(clientPrivateKeyFile, clientCertificateFile);
+
+ SSLContext clientCtx = new SslContextBuilder()
+ .withKeyStore(clientPrivateKeyFile, clientCertificateFile)
+ .withTrustStore(certificateFile)
+ .build();
+
+ assertHttpsRequestTriggersSslHandshakeException(
+ driver, clientCtx, null, null, "Received fatal alert: certificate_unknown");
+ verify(metricConsumer.mockitoMock(), atLeast(1))
+ .add(MetricDefinitions.SSL_HANDSHAKE_FAILURE_INVALID_CLIENT_CERT, 1L, MetricConsumerMock.STATIC_CONTEXT);
+ assertTrue(driver.close());
+ assertThat(connectionLog.logEntries()).hasSize(1);
+ assertSslHandshakeFailurePresent(
+ connectionLog.logEntries().get(0), SSLHandshakeException.class, SslHandshakeFailure.INVALID_CLIENT_CERT.failureType());
+ }
+
+ @Test
+ void requireThatMetricIsIncrementedWhenClientUsesExpiredCertificateInHandshake(@TempDir Path tmpFolder) throws IOException {
+ Path endEntityKeyFile = tmpFolder.resolve("client-key.pem");
+ Path endEntitycertificateFile = tmpFolder.resolve("client-cert.pem");
+ Instant notAfter = Instant.now().minus(100, ChronoUnit.DAYS);
+ generatePrivateKeyAndCertificate(privateKeyFile, certificateFile, endEntityKeyFile, endEntitycertificateFile, notAfter);
+ var metricConsumer = new MetricConsumerMock();
+ InMemoryConnectionLog connectionLog = new InMemoryConnectionLog();
+ JettyTestDriver driver = createSslTestDriver(certificateFile, privateKeyFile, metricConsumer, connectionLog);
+
+ SSLContext clientCtx = new SslContextBuilder()
+ .withTrustStore(certificateFile)
+ .withKeyStore(endEntityKeyFile, endEntitycertificateFile)
+ .build();
+
+ assertHttpsRequestTriggersSslHandshakeException(
+ driver, clientCtx, null, null, "Received fatal alert: certificate_unknown");
+ verify(metricConsumer.mockitoMock(), atLeast(1))
+ .add(MetricDefinitions.SSL_HANDSHAKE_FAILURE_EXPIRED_CLIENT_CERT, 1L, MetricConsumerMock.STATIC_CONTEXT);
+ assertTrue(driver.close());
+ assertThat(connectionLog.logEntries()).hasSize(1);
+ }
+
+
+ private static void assertHttpsRequestTriggersSslHandshakeException(
+ JettyTestDriver testDriver,
+ SSLContext sslContext,
+ String protocolOverride,
+ String cipherOverride,
+ String expectedExceptionSubstring) throws IOException {
+ List<String> protocols = protocolOverride != null ? List.of(protocolOverride) : null;
+ List<String> ciphers = cipherOverride != null ? List.of(cipherOverride) : null;
+ try (var client = new SimpleHttpClient(sslContext, protocols, ciphers, testDriver.server().getListenPort(), false)) {
+ client.get("/status.html");
+ fail("SSLHandshakeException expected");
+ } catch (SSLHandshakeException e) {
+ assertThat(e.getMessage()).contains(expectedExceptionSubstring);
+ } catch (SSLException e) {
+ // This exception is thrown if Apache httpclient's write thread detects the handshake failure before the read thread.
+ log.log(Level.WARNING, "Client failed to get a proper TLS handshake response: " + e.getMessage(), e);
+ // Only ignore a subset of exceptions
+ assertTrue(e.getMessage().contains("readHandshakeRecord") || e.getMessage().contains("Broken pipe"), e.getMessage());
+ }
+ }
+
+ private static void assertSslHandshakeFailurePresent(
+ ConnectionLogEntry entry, Class<? extends SSLHandshakeException> expectedException, String expectedType) {
+ assertThat(entry.sslHandshakeFailure()).isPresent();
+ ConnectionLogEntry.SslHandshakeFailure failure = entry.sslHandshakeFailure().get();
+ assertEquals(expectedType, failure.type());
+ ConnectionLogEntry.SslHandshakeFailure.ExceptionEntry exceptionEntry = failure.exceptionChain().get(0);
+ assertEquals(expectedException.getName(), exceptionEntry.name());
+ }
+
+}
diff --git a/container-core/src/test/java/com/yahoo/jdisc/http/server/jetty/Utils.java b/container-core/src/test/java/com/yahoo/jdisc/http/server/jetty/Utils.java
new file mode 100644
index 00000000000..626ab521773
--- /dev/null
+++ b/container-core/src/test/java/com/yahoo/jdisc/http/server/jetty/Utils.java
@@ -0,0 +1,68 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.jdisc.http.server.jetty;
+
+import com.google.inject.Module;
+import com.yahoo.container.logging.ConnectionLog;
+import com.yahoo.jdisc.application.MetricConsumer;
+import com.yahoo.security.KeyUtils;
+import com.yahoo.security.Pkcs10Csr;
+import com.yahoo.security.Pkcs10CsrBuilder;
+import com.yahoo.security.X509CertificateBuilder;
+import com.yahoo.security.X509CertificateUtils;
+
+import javax.security.auth.x500.X500Principal;
+import java.io.IOException;
+import java.math.BigInteger;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.security.KeyPair;
+import java.security.PrivateKey;
+import java.security.cert.X509Certificate;
+import java.time.Instant;
+import java.time.temporal.ChronoUnit;
+
+import static com.yahoo.security.KeyAlgorithm.EC;
+import static com.yahoo.security.SignatureAlgorithm.SHA256_WITH_ECDSA;
+
+/**
+ * @author bjorncs
+ */
+class Utils {
+
+ private Utils() {}
+
+ static JettyTestDriver createSslTestDriver(
+ Path serverCertificateFile, Path serverPrivateKeyFile, MetricConsumerMock metricConsumer, InMemoryConnectionLog connectionLog) {
+ Module extraModule = binder -> {
+ binder.bind(MetricConsumer.class).toInstance(metricConsumer.mockitoMock());
+ binder.bind(ConnectionLog.class).toInstance(connectionLog);
+ };
+ return JettyTestDriver.newInstanceWithSsl(
+ new EchoRequestHandler(), serverCertificateFile, serverPrivateKeyFile, JettyTestDriver.TlsClientAuth.NEED, extraModule);
+ }
+
+ static void generatePrivateKeyAndCertificate(Path privateKeyFile, Path certificateFile) throws IOException {
+ KeyPair keyPair = KeyUtils.generateKeypair(EC);
+ Files.writeString(privateKeyFile, KeyUtils.toPem(keyPair.getPrivate()));
+
+ X509Certificate certificate = X509CertificateBuilder
+ .fromKeypair(
+ keyPair, new X500Principal("CN=localhost"), Instant.EPOCH, Instant.EPOCH.plus(100_000, ChronoUnit.DAYS), SHA256_WITH_ECDSA, BigInteger.ONE)
+ .build();
+ Files.writeString(certificateFile, X509CertificateUtils.toPem(certificate));
+ }
+
+ static void generatePrivateKeyAndCertificate(Path rootPrivateKeyFile, Path rootCertificateFile,
+ Path privateKeyFile, Path certificateFile, Instant notAfter) throws IOException {
+ X509Certificate rootCertificate = X509CertificateUtils.fromPem(Files.readString(rootCertificateFile));
+ PrivateKey privateKey = KeyUtils.fromPemEncodedPrivateKey(Files.readString(rootPrivateKeyFile));
+
+ KeyPair keyPair = KeyUtils.generateKeypair(EC);
+ Files.writeString(privateKeyFile, KeyUtils.toPem(keyPair.getPrivate()));
+ Pkcs10Csr csr = Pkcs10CsrBuilder.fromKeypair(new X500Principal("CN=myclient"), keyPair, SHA256_WITH_ECDSA).build();
+ X509Certificate certificate = X509CertificateBuilder
+ .fromCsr(csr, rootCertificate.getSubjectX500Principal(), Instant.EPOCH, notAfter, privateKey, SHA256_WITH_ECDSA, BigInteger.ONE)
+ .build();
+ Files.writeString(certificateFile, X509CertificateUtils.toPem(certificate));
+ }
+}
diff --git a/container-disc/pom.xml b/container-disc/pom.xml
index 5446c9e1698..b255b6af02a 100644
--- a/container-disc/pom.xml
+++ b/container-disc/pom.xml
@@ -170,14 +170,11 @@
configgen.jar,
config-bundle-jar-with-dependencies.jar,
configdefinitions-jar-with-dependencies.jar,
- container-jersey2-jar-with-dependencies.jar,
container-search-and-docproc-jar-with-dependencies.jar,
container-search-gui-jar-with-dependencies.jar,
docprocs-jar-with-dependencies.jar,
hosted-zone-api-jar-with-dependencies.jar,
jdisc-security-filters-jar-with-dependencies.jar,
- model-evaluation-jar-with-dependencies.jar,
- model-integration-jar-with-dependencies.jar,
vespaclient-container-plugin-jar-with-dependencies.jar,
vespa-athenz-jar-with-dependencies.jar,
security-utils-jar-with-dependencies.jar,
diff --git a/container-disc/src/main/java/com/yahoo/container/jdisc/ConfiguredApplication.java b/container-disc/src/main/java/com/yahoo/container/jdisc/ConfiguredApplication.java
index f43c1fcffd6..0e4c4446778 100644
--- a/container-disc/src/main/java/com/yahoo/container/jdisc/ConfiguredApplication.java
+++ b/container-disc/src/main/java/com/yahoo/container/jdisc/ConfiguredApplication.java
@@ -18,6 +18,7 @@ import com.yahoo.container.di.config.Subscriber;
import com.yahoo.container.di.config.SubscriberFactory;
import com.yahoo.container.http.filter.FilterChainRepository;
import com.yahoo.container.jdisc.component.Deconstructor;
+import com.yahoo.container.jdisc.messagebus.SessionCache;
import com.yahoo.container.jdisc.metric.DisableGuiceMetric;
import com.yahoo.jdisc.Metric;
import com.yahoo.jdisc.application.Application;
@@ -76,6 +77,7 @@ public final class ConfiguredApplication implements Application {
// to config to make sure that container will be registered in slobrok (by {@link com.yahoo.jrt.slobrok.api.Register})
// if slobrok config changes (typically slobroks moving to other nodes)
private final Optional<SlobrokConfigSubscriber> slobrokConfigSubscriber;
+ private final SessionCache sessionCache;
//TODO: FilterChainRepository should instead always be set up in the model.
private final FilterChainRepository defaultFilterChainRepository =
@@ -125,6 +127,7 @@ public final class ConfiguredApplication implements Application {
this.slobrokConfigSubscriber = (subscriberFactory instanceof CloudSubscriberFactory)
? Optional.of(new SlobrokConfigSubscriber(configId))
: Optional.empty();
+ this.sessionCache = new SessionCache(configId);
this.restrictedOsgiFramework = new DisableOsgiFramework(new RestrictedBundleContext(osgiFramework.bundleContext()));
}
@@ -346,6 +349,7 @@ public final class ConfiguredApplication implements Application {
bind(OsgiFramework.class).toInstance(restrictedOsgiFramework);
bind(com.yahoo.jdisc.Timer.class).toInstance(timerSingleton);
bind(FilterChainRepository.class).toInstance(defaultFilterChainRepository);
+ bind(SessionCache.class).toInstance(sessionCache); // Needed by e.g. FeedHandler
}
});
}
diff --git a/container-disc/src/main/sh/vespa-start-container-daemon.sh b/container-disc/src/main/sh/vespa-start-container-daemon.sh
index 4518868cb81..223124c29d6 100755
--- a/container-disc/src/main/sh/vespa-start-container-daemon.sh
+++ b/container-disc/src/main/sh/vespa-start-container-daemon.sh
@@ -215,6 +215,7 @@ exec $numactlcmd $envcmd java \
--add-opens=java.base/java.io=ALL-UNNAMED \
--add-opens=java.base/java.lang=ALL-UNNAMED \
--add-opens=java.base/java.net=ALL-UNNAMED \
+ --add-opens=java.base/java.nio=ALL-UNNAMED \
--add-opens=java.base/jdk.internal.loader=ALL-UNNAMED \
--add-opens=java.base/sun.security.ssl=ALL-UNNAMED \
-Djava.io.tmpdir="${VESPA_HOME}/tmp" \
diff --git a/container-messagebus/src/main/java/com/yahoo/container/jdisc/messagebus/SessionCache.java b/container-messagebus/src/main/java/com/yahoo/container/jdisc/messagebus/SessionCache.java
index ab3d09af178..d65b2f7cc12 100644
--- a/container-messagebus/src/main/java/com/yahoo/container/jdisc/messagebus/SessionCache.java
+++ b/container-messagebus/src/main/java/com/yahoo/container/jdisc/messagebus/SessionCache.java
@@ -1,24 +1,22 @@
// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.container.jdisc.messagebus;
-import com.google.inject.Inject;
-import com.yahoo.cloud.config.SlobroksConfig;
import com.yahoo.component.AbstractComponent;
+import com.yahoo.config.subscription.ConfigGetter;
import com.yahoo.container.jdisc.ContainerMbusConfig;
import com.yahoo.document.DocumentTypeManager;
import com.yahoo.document.DocumentUtil;
-import com.yahoo.document.config.DocumentmanagerConfig;
import com.yahoo.documentapi.messagebus.loadtypes.LoadTypeSet;
import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol;
-import com.yahoo.documentapi.messagebus.protocol.DocumentProtocolPoliciesConfig;
import com.yahoo.jdisc.ReferencedResource;
import com.yahoo.jdisc.References;
import com.yahoo.jdisc.ResourceReference;
import com.yahoo.jdisc.SharedResource;
+import java.util.logging.Level;
+import com.yahoo.messagebus.ConfigAgent;
import com.yahoo.messagebus.DynamicThrottlePolicy;
import com.yahoo.messagebus.IntermediateSessionParams;
import com.yahoo.messagebus.MessageBusParams;
-import com.yahoo.messagebus.MessagebusConfig;
import com.yahoo.messagebus.Protocol;
import com.yahoo.messagebus.SourceSessionParams;
import com.yahoo.messagebus.StaticThrottlePolicy;
@@ -28,12 +26,9 @@ import com.yahoo.messagebus.network.rpc.RPCNetworkParams;
import com.yahoo.messagebus.shared.SharedIntermediateSession;
import com.yahoo.messagebus.shared.SharedMessageBus;
import com.yahoo.messagebus.shared.SharedSourceSession;
-import com.yahoo.vespa.config.content.DistributionConfig;
-import com.yahoo.vespa.config.content.LoadTypeConfig;
import java.util.HashMap;
import java.util.Map;
-import java.util.logging.Level;
import java.util.logging.Logger;
/**
@@ -42,19 +37,24 @@ import java.util.logging.Logger;
* @author Steinar Knutsen
* @author Einar Rosenvinge
*/
-// TODO jonmv: Remove this: only used with more than one entry by FeedHandlerV3, where only timeout varies.
-// rant: This whole construct is because DI at one point didn't exist, so getting hold of a shared resource
-// or session was hard(?), and one resorted to routing through the Container, using URIs, to the correct
-// MbusClient, with or without throttling. This introduced the problem of ownership during shutdown,
-// which was solved with manual reference counting. This is all much better solved with DI, which (now)
-// owns everything, and does component shutdown in reverse construction order, which is always right.
-// So for the sake everyone's mental health, this should all just be removed now! I suspect this is
-// even the case for Request; we can track in handlers, and warn when requests have been misplaced.
+// TODO jonmv: Remove this? Only used sensibly by FeedHandlerV3, where only timeout varies.
public final class SessionCache extends AbstractComponent {
private static final Logger log = Logger.getLogger(SessionCache.class.getName());
- private final SharedMessageBus messageBus;
+ //config
+ private final String messagebusConfigId;
+ private final String slobrokConfigId;
+ private final String identity;
+ private final String containerMbusConfigId;
+ private final String documentManagerConfigId;
+ private final String loadTypeConfigId;
+ private final DocumentTypeManager documentTypeManager;
+
+ // initialized in start()
+ private ConfigAgent configAgent;
+ private SharedMessageBus messageBus;
+
private final Object intermediateLock = new Object();
private final Map<String, SharedIntermediateSession> intermediates = new HashMap<>();
private final IntermediateSessionCreator intermediatesCreator = new IntermediateSessionCreator();
@@ -63,48 +63,52 @@ public final class SessionCache extends AbstractComponent {
private final Map<SourceSessionKey, SharedSourceSession> sources = new HashMap<>();
private final SourceSessionCreator sourcesCreator = new SourceSessionCreator();
- @Inject
- public SessionCache(ContainerMbusConfig containerMbusConfig, DocumentmanagerConfig documentmanagerConfig,
- LoadTypeConfig loadTypeConfig, SlobroksConfig slobroksConfig,
- MessagebusConfig messagebusConfig, DocumentProtocolPoliciesConfig policiesConfig,
- DistributionConfig distributionConfig) {
- this(containerMbusConfig, documentmanagerConfig, loadTypeConfig, slobroksConfig,
- messagebusConfig, policiesConfig, distributionConfig, System.getProperty("config.id")); //:
+ public SessionCache(String messagebusConfigId, String slobrokConfigId, String identity,
+ String containerMbusConfigId, String documentManagerConfigId,
+ String loadTypeConfigId,
+ DocumentTypeManager documentTypeManager) {
+ this.messagebusConfigId = messagebusConfigId;
+ this.slobrokConfigId = slobrokConfigId;
+ this.identity = identity;
+ this.containerMbusConfigId = containerMbusConfigId;
+ this.documentManagerConfigId = documentManagerConfigId;
+ this.loadTypeConfigId = loadTypeConfigId;
+ this.documentTypeManager = documentTypeManager;
}
- public SessionCache(ContainerMbusConfig containerMbusConfig, DocumentmanagerConfig documentmanagerConfig,
- LoadTypeConfig loadTypeConfig, SlobroksConfig slobroksConfig,
- MessagebusConfig messagebusConfig, DocumentProtocolPoliciesConfig policiesConfig,
- DistributionConfig distributionConfig, String identity) {
- this.messageBus = createSharedMessageBus(containerMbusConfig,
- messagebusConfig,
- slobroksConfig,
- identity,
- new DocumentProtocol(new DocumentTypeManager(documentmanagerConfig),
- new LoadTypeSet(loadTypeConfig),
- policiesConfig,
- distributionConfig));
+ public SessionCache(final String identity) {
+ this(identity, identity, identity, identity, identity, identity, new DocumentTypeManager());
}
- public SessionCache(ContainerMbusConfig containerMbusConfig, SlobroksConfig slobroksConfig,
- MessagebusConfig messagebusConfig, String identity, Protocol protocol) {
- this.messageBus = createSharedMessageBus(containerMbusConfig,
- messagebusConfig,
- slobroksConfig,
- identity,
- protocol);
+ public void deconstruct() {
+ if (configAgent != null) {
+ configAgent.shutdown();
+ }
}
- public void deconstruct() {
- messageBus.release();
+
+ private void start() {
+ ContainerMbusConfig mbusConfig = ConfigGetter.getConfig(ContainerMbusConfig.class, containerMbusConfigId);
+ if (documentManagerConfigId != null) {
+ documentTypeManager.configure(documentManagerConfigId);
+ }
+ LoadTypeSet loadTypeSet = new LoadTypeSet(loadTypeConfigId);
+ DocumentProtocol protocol = new DocumentProtocol(documentTypeManager, identity, loadTypeSet);
+ messageBus = createSharedMessageBus(mbusConfig, slobrokConfigId, identity, protocol);
+ // TODO: stop doing subscriptions to config when that is to be solved in slobrok as well
+ configAgent = new ConfigAgent(messagebusConfigId, messageBus.messageBus());
+ configAgent.subscribe();
+ }
+
+
+ private boolean isStarted() {
+ return messageBus != null;
}
private static SharedMessageBus createSharedMessageBus(ContainerMbusConfig mbusConfig,
- MessagebusConfig messagebusConfig,
- SlobroksConfig slobroksConfig, String identity,
+ String slobrokConfigId, String identity,
Protocol protocol) {
- MessageBusParams mbusParams = new MessageBusParams().addProtocol(protocol)
- .setMessageBusConfig(messagebusConfig);
+ MessageBusParams mbusParams = new MessageBusParams().addProtocol(protocol);
int maxPendingSize = DocumentUtil
.calculateMaxPendingSize(mbusConfig.maxConcurrentFactor(), mbusConfig.documentExpansionFactor(),
@@ -115,7 +119,7 @@ public final class SessionCache extends AbstractComponent {
mbusParams.setMaxPendingSize(maxPendingSize);
RPCNetworkParams netParams = new RPCNetworkParams()
- .setSlobroksConfig(slobroksConfig)
+ .setSlobrokConfigId(slobrokConfigId)
.setIdentity(new Identity(identity))
.setListenPort(mbusConfig.port())
.setNumTargetsPerSpec(mbusConfig.numconnectionspertarget())
@@ -140,10 +144,20 @@ public final class SessionCache extends AbstractComponent {
}
ReferencedResource<SharedIntermediateSession> retainIntermediate(final IntermediateSessionParams p) {
+ synchronized (this) {
+ if (!isStarted()) {
+ start();
+ }
+ }
return intermediatesCreator.retain(intermediateLock, intermediates, p);
}
public ReferencedResource<SharedSourceSession> retainSource(final SourceSessionParams p) {
+ synchronized (this) {
+ if (!isStarted()) {
+ start();
+ }
+ }
return sourcesCreator.retain(sourceLock, sources, p);
}
diff --git a/container-messagebus/src/main/java/com/yahoo/messagebus/jdisc/test/ClientTestDriver.java b/container-messagebus/src/main/java/com/yahoo/messagebus/jdisc/test/ClientTestDriver.java
index ea0ed7eadc8..111805d61b0 100644
--- a/container-messagebus/src/main/java/com/yahoo/messagebus/jdisc/test/ClientTestDriver.java
+++ b/container-messagebus/src/main/java/com/yahoo/messagebus/jdisc/test/ClientTestDriver.java
@@ -33,7 +33,7 @@ public class ClientTestDriver {
this.server = server;
MessageBusParams mbusParams = new MessageBusParams().addProtocol(protocol);
- RPCNetworkParams netParams = new RPCNetworkParams().setSlobroksConfig(server.slobroksConfig());
+ RPCNetworkParams netParams = new RPCNetworkParams().setSlobrokConfigId(server.slobrokId());
SharedMessageBus mbus = SharedMessageBus.newInstance(mbusParams, netParams);
session = mbus.newSourceSession(new SourceSessionParams());
client = new MbusClient(session);
@@ -128,4 +128,7 @@ public class ClientTestDriver {
return new ClientTestDriver(RemoteServer.newInstanceWithInternSlobrok(), protocol);
}
+ public static ClientTestDriver newInstanceWithExternSlobrok(String slobrokId) {
+ return new ClientTestDriver(RemoteServer.newInstanceWithExternSlobrok(slobrokId), new SimpleProtocol());
+ }
}
diff --git a/container-messagebus/src/main/java/com/yahoo/messagebus/jdisc/test/RemoteClient.java b/container-messagebus/src/main/java/com/yahoo/messagebus/jdisc/test/RemoteClient.java
index 6cd8fb8f34d..57d0abd980b 100644
--- a/container-messagebus/src/main/java/com/yahoo/messagebus/jdisc/test/RemoteClient.java
+++ b/container-messagebus/src/main/java/com/yahoo/messagebus/jdisc/test/RemoteClient.java
@@ -1,17 +1,9 @@
// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.messagebus.jdisc.test;
-import com.yahoo.cloud.config.SlobroksConfig;
import com.yahoo.jrt.ListenFailedException;
import com.yahoo.jrt.slobrok.server.Slobrok;
-import com.yahoo.messagebus.Message;
-import com.yahoo.messagebus.MessageBus;
-import com.yahoo.messagebus.MessageBusParams;
-import com.yahoo.messagebus.Protocol;
-import com.yahoo.messagebus.Reply;
-import com.yahoo.messagebus.Result;
-import com.yahoo.messagebus.SourceSession;
-import com.yahoo.messagebus.SourceSessionParams;
+import com.yahoo.messagebus.*;
import com.yahoo.messagebus.network.local.LocalNetwork;
import com.yahoo.messagebus.network.rpc.RPCNetwork;
import com.yahoo.messagebus.network.rpc.RPCNetworkParams;
@@ -25,14 +17,16 @@ import java.util.concurrent.TimeUnit;
public class RemoteClient {
private final Slobrok slobrok;
+ private final String slobrokId;
private final MessageBus mbus;
private final ReplyQueue queue = new ReplyQueue();
private final SourceSession session;
- private RemoteClient(Protocol protocol, boolean network) {
- this.slobrok = newSlobrok();
+ private RemoteClient(Slobrok slobrok, String slobrokId, Protocol protocol, boolean network) {
+ this.slobrok = slobrok;
+ this.slobrokId = slobrok != null ? slobrok.configId() : slobrokId;
mbus = network
- ? new MessageBus(new RPCNetwork(new RPCNetworkParams().setSlobroksConfig(slobroksConfig())),
+ ? new MessageBus(new RPCNetwork(new RPCNetworkParams().setSlobrokConfigId(this.slobrokId)),
new MessageBusParams().addProtocol(protocol))
: new MessageBus(new LocalNetwork(), new MessageBusParams().addProtocol(protocol));
session = mbus.createSourceSession(new SourceSessionParams().setThrottlePolicy(null).setReplyHandler(queue));
@@ -46,22 +40,28 @@ public class RemoteClient {
return queue.awaitReply(timeout, unit);
}
- public SlobroksConfig slobroksConfig() {
- return TestUtils.configFor(slobrok);
+ public String slobrokId() {
+ return slobrokId;
}
public void close() {
session.destroy();
mbus.destroy();
- slobrok.stop();
+ if (slobrok != null) {
+ slobrok.stop();
+ }
}
public static RemoteClient newInstanceWithInternSlobrok(boolean network) {
- return new RemoteClient(new SimpleProtocol(), network);
+ return new RemoteClient(newSlobrok(), null, new SimpleProtocol(), network);
+ }
+
+ public static RemoteClient newInstanceWithExternSlobrok(String slobrokId, boolean network) {
+ return new RemoteClient(null, slobrokId, new SimpleProtocol(), network);
}
public static RemoteClient newInstanceWithProtocolAndInternSlobrok(Protocol protocol, boolean network) {
- return new RemoteClient(protocol, network);
+ return new RemoteClient(newSlobrok(), null, protocol, network);
}
private static Slobrok newSlobrok() {
diff --git a/container-messagebus/src/main/java/com/yahoo/messagebus/jdisc/test/RemoteServer.java b/container-messagebus/src/main/java/com/yahoo/messagebus/jdisc/test/RemoteServer.java
index 0fb23e33709..1f0f82c4903 100644
--- a/container-messagebus/src/main/java/com/yahoo/messagebus/jdisc/test/RemoteServer.java
+++ b/container-messagebus/src/main/java/com/yahoo/messagebus/jdisc/test/RemoteServer.java
@@ -1,16 +1,9 @@
// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.messagebus.jdisc.test;
-import com.yahoo.cloud.config.SlobroksConfig;
import com.yahoo.jrt.ListenFailedException;
import com.yahoo.jrt.slobrok.server.Slobrok;
-import com.yahoo.messagebus.DestinationSession;
-import com.yahoo.messagebus.DestinationSessionParams;
-import com.yahoo.messagebus.Message;
-import com.yahoo.messagebus.MessageBus;
-import com.yahoo.messagebus.MessageBusParams;
-import com.yahoo.messagebus.Protocol;
-import com.yahoo.messagebus.Reply;
+import com.yahoo.messagebus.*;
import com.yahoo.messagebus.network.Identity;
import com.yahoo.messagebus.network.rpc.RPCNetwork;
import com.yahoo.messagebus.network.rpc.RPCNetworkParams;
@@ -24,14 +17,16 @@ import java.util.concurrent.TimeUnit;
public class RemoteServer {
private final Slobrok slobrok;
+ private final String slobrokId;
private final MessageBus mbus;
private final MessageQueue queue = new MessageQueue();
private final DestinationSession session;
- private RemoteServer(Protocol protocol, String identity) {
- this.slobrok = newSlobrok();
+ private RemoteServer(Slobrok slobrok, String slobrokId, Protocol protocol, String identity) {
+ this.slobrok = slobrok;
+ this.slobrokId = slobrok != null ? slobrok.configId() : slobrokId;
mbus = new MessageBus(new RPCNetwork(new RPCNetworkParams()
- .setSlobroksConfig(slobroksConfig())
+ .setSlobrokConfigId(this.slobrokId)
.setIdentity(new Identity(identity))),
new MessageBusParams().addProtocol(protocol));
session = mbus.createDestinationSession(new DestinationSessionParams().setMessageHandler(queue));
@@ -53,22 +48,32 @@ public class RemoteServer {
session.reply(reply);
}
- public SlobroksConfig slobroksConfig() {
- return TestUtils.configFor(slobrok);
+ public String slobrokId() {
+ return slobrokId;
}
public void close() {
session.destroy();
mbus.destroy();
- slobrok.stop();
+ if (slobrok != null) {
+ slobrok.stop();
+ }
}
public static RemoteServer newInstanceWithInternSlobrok() {
- return new RemoteServer(new SimpleProtocol(), "remote");
+ return new RemoteServer(newSlobrok(), null, new SimpleProtocol(), "remote");
+ }
+
+ public static RemoteServer newInstanceWithExternSlobrok(String slobrokId) {
+ return new RemoteServer(null, slobrokId, new SimpleProtocol(), "remote");
+ }
+
+ public static RemoteServer newInstance(String slobrokId, String identity, Protocol protocol) {
+ return new RemoteServer(null, slobrokId, protocol, identity);
}
- public static RemoteServer newInstance(String identity, Protocol protocol) {
- return new RemoteServer(protocol, identity);
+ public static RemoteServer newInstanceWithProtocol(Protocol protocol) {
+ return new RemoteServer(newSlobrok(), null, protocol, "remote");
}
private static Slobrok newSlobrok() {
diff --git a/container-messagebus/src/main/java/com/yahoo/messagebus/jdisc/test/ServerTestDriver.java b/container-messagebus/src/main/java/com/yahoo/messagebus/jdisc/test/ServerTestDriver.java
index fa0dd37ed13..e59db28e886 100644
--- a/container-messagebus/src/main/java/com/yahoo/messagebus/jdisc/test/ServerTestDriver.java
+++ b/container-messagebus/src/main/java/com/yahoo/messagebus/jdisc/test/ServerTestDriver.java
@@ -42,7 +42,7 @@ public class ServerTestDriver {
}
MessageBusParams mbusParams = new MessageBusParams().addProtocol(protocol);
- RPCNetworkParams netParams = new RPCNetworkParams().setSlobroksConfig(client.slobroksConfig());
+ RPCNetworkParams netParams = new RPCNetworkParams().setSlobrokConfigId(client.slobrokId());
SharedMessageBus mbus = SharedMessageBus.newInstance(mbusParams, netParams);
ServerSession session = mbus.newDestinationSession(new DestinationSessionParams());
server = new MbusServer(driver, session);
@@ -130,6 +130,13 @@ public class ServerTestDriver {
guiceModules);
}
+ public static ServerTestDriver newInstanceWithExternSlobrok(String slobrokId, RequestHandler requestHandler,
+ boolean network, Module... guiceModules)
+ {
+ return new ServerTestDriver(RemoteClient.newInstanceWithExternSlobrok(slobrokId, network),
+ true, requestHandler, new SimpleProtocol(), guiceModules);
+ }
+
public static ServerTestDriver newInactiveInstance(boolean network, Module... guiceModules) {
return new ServerTestDriver(RemoteClient.newInstanceWithInternSlobrok(network), false, null,
new SimpleProtocol(), guiceModules);
diff --git a/container-messagebus/src/main/java/com/yahoo/messagebus/jdisc/test/TestUtils.java b/container-messagebus/src/main/java/com/yahoo/messagebus/jdisc/test/TestUtils.java
deleted file mode 100644
index 85ad241259f..00000000000
--- a/container-messagebus/src/main/java/com/yahoo/messagebus/jdisc/test/TestUtils.java
+++ /dev/null
@@ -1,21 +0,0 @@
-// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.messagebus.jdisc.test;
-
-import com.yahoo.cloud.config.SlobroksConfig;
-import com.yahoo.jrt.Spec;
-import com.yahoo.jrt.slobrok.server.Slobrok;
-
-/**
- * @author jonmv
- */
-public class TestUtils {
-
- private TestUtils() { }
-
- public static SlobroksConfig configFor(Slobrok slobrok) {
- return new SlobroksConfig.Builder().slobrok(new SlobroksConfig.Slobrok.Builder()
- .connectionspec(new Spec("localhost", slobrok.port()).toString()))
- .build();
- }
-
-}
diff --git a/container-messagebus/src/main/java/com/yahoo/messagebus/shared/SharedMessageBus.java b/container-messagebus/src/main/java/com/yahoo/messagebus/shared/SharedMessageBus.java
index 259dd788244..dd135a51378 100644
--- a/container-messagebus/src/main/java/com/yahoo/messagebus/shared/SharedMessageBus.java
+++ b/container-messagebus/src/main/java/com/yahoo/messagebus/shared/SharedMessageBus.java
@@ -3,8 +3,6 @@ package com.yahoo.messagebus.shared;
import com.yahoo.config.subscription.ConfigGetter;
import com.yahoo.jdisc.AbstractResource;
-
-import java.util.Objects;
import java.util.logging.Level;
import com.yahoo.messagebus.DestinationSessionParams;
import com.yahoo.messagebus.IntermediateSessionParams;
@@ -27,7 +25,8 @@ public class SharedMessageBus extends AbstractResource {
private final MessageBus mbus;
public SharedMessageBus(MessageBus mbus) {
- this.mbus = Objects.requireNonNull(mbus);
+ mbus.getClass(); // throws NullPointerException
+ this.mbus = mbus;
}
public MessageBus messageBus() {
diff --git a/container-messagebus/src/test/java/com/yahoo/container/jdisc/messagebus/MbusClientProviderTest.java b/container-messagebus/src/test/java/com/yahoo/container/jdisc/messagebus/MbusClientProviderTest.java
index 9512c1a4873..6335cf01d8c 100644
--- a/container-messagebus/src/test/java/com/yahoo/container/jdisc/messagebus/MbusClientProviderTest.java
+++ b/container-messagebus/src/test/java/com/yahoo/container/jdisc/messagebus/MbusClientProviderTest.java
@@ -1,16 +1,9 @@
// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.container.jdisc.messagebus;
-import com.yahoo.cloud.config.SlobroksConfig;
-import com.yahoo.container.jdisc.ContainerMbusConfig;
import com.yahoo.container.jdisc.config.SessionConfig;
import com.yahoo.container.jdisc.messagebus.MbusClientProvider;
import com.yahoo.container.jdisc.messagebus.SessionCache;
-import com.yahoo.document.config.DocumentmanagerConfig;
-import com.yahoo.documentapi.messagebus.protocol.DocumentProtocolPoliciesConfig;
-import com.yahoo.messagebus.MessagebusConfig;
-import com.yahoo.vespa.config.content.DistributionConfig;
-import com.yahoo.vespa.config.content.LoadTypeConfig;
import org.junit.Test;
import static org.junit.Assert.assertNotNull;
@@ -37,15 +30,7 @@ public class MbusClientProviderTest {
}
private void testClient(SessionConfig config) {
- SessionCache cache = new SessionCache(new ContainerMbusConfig.Builder().build(),
- new DocumentmanagerConfig.Builder().build(),
- new LoadTypeConfig.Builder().build(),
- new SlobroksConfig.Builder().build(),
- new MessagebusConfig.Builder().build(),
- new DocumentProtocolPoliciesConfig.Builder().build(),
- new DistributionConfig.Builder().build(),
- "test");
- MbusClientProvider p = new MbusClientProvider(cache, config);
+ MbusClientProvider p = new MbusClientProvider(new SessionCache("dir:src/test/resources/config/clientprovider"), config);
assertNotNull(p.get());
p.deconstruct();
}
diff --git a/container-messagebus/src/test/java/com/yahoo/messagebus/jdisc/test/ClientTestDriverTestCase.java b/container-messagebus/src/test/java/com/yahoo/messagebus/jdisc/test/ClientTestDriverTestCase.java
index d24acb9d0d8..ef290a070cb 100644
--- a/container-messagebus/src/test/java/com/yahoo/messagebus/jdisc/test/ClientTestDriverTestCase.java
+++ b/container-messagebus/src/test/java/com/yahoo/messagebus/jdisc/test/ClientTestDriverTestCase.java
@@ -23,6 +23,10 @@ public class ClientTestDriverTestCase {
driver = ClientTestDriver.newInstanceWithProtocol(new SimpleProtocol());
assertNotNull(driver);
assertTrue(driver.close());
- }
+ Slobrok slobrok = new Slobrok();
+ driver = ClientTestDriver.newInstanceWithExternSlobrok(slobrok.configId());
+ assertNotNull(driver);
+ assertTrue(driver.close());
+ }
}
diff --git a/container-messagebus/src/test/java/com/yahoo/messagebus/jdisc/test/ServerTestDriverTestCase.java b/container-messagebus/src/test/java/com/yahoo/messagebus/jdisc/test/ServerTestDriverTestCase.java
index 7b2c1b68549..f6ae2335d12 100644
--- a/container-messagebus/src/test/java/com/yahoo/messagebus/jdisc/test/ServerTestDriverTestCase.java
+++ b/container-messagebus/src/test/java/com/yahoo/messagebus/jdisc/test/ServerTestDriverTestCase.java
@@ -24,6 +24,11 @@ public class ServerTestDriverTestCase {
driver = ServerTestDriver.newInstanceWithProtocol(new SimpleProtocol(), new NonWorkingRequestHandler(), false);
assertNotNull(driver);
assertTrue(driver.close());
+
+ Slobrok slobrok = new Slobrok();
+ driver = ServerTestDriver.newInstanceWithExternSlobrok(slobrok.configId(), new NonWorkingRequestHandler(), false);
+ assertNotNull(driver);
+ assertTrue(driver.close());
}
}
diff --git a/container-messagebus/src/test/java/com/yahoo/messagebus/shared/SharedDestinationSessionTestCase.java b/container-messagebus/src/test/java/com/yahoo/messagebus/shared/SharedDestinationSessionTestCase.java
index 759509946ce..78e79da4b9f 100644
--- a/container-messagebus/src/test/java/com/yahoo/messagebus/shared/SharedDestinationSessionTestCase.java
+++ b/container-messagebus/src/test/java/com/yahoo/messagebus/shared/SharedDestinationSessionTestCase.java
@@ -1,14 +1,12 @@
// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.messagebus.shared;
-import com.yahoo.cloud.config.SlobroksConfig;
import com.yahoo.jrt.ListenFailedException;
import com.yahoo.jrt.slobrok.server.Slobrok;
import com.yahoo.messagebus.*;
import com.yahoo.messagebus.jdisc.test.MessageQueue;
import com.yahoo.messagebus.jdisc.test.RemoteClient;
import com.yahoo.messagebus.jdisc.test.ReplyQueue;
-import com.yahoo.messagebus.jdisc.test.TestUtils;
import com.yahoo.messagebus.network.rpc.RPCNetworkParams;
import com.yahoo.messagebus.routing.Route;
import com.yahoo.messagebus.test.SimpleMessage;
@@ -100,7 +98,7 @@ public class SharedDestinationSessionTestCase {
RemoteClient client = RemoteClient.newInstanceWithInternSlobrok(true);
MessageQueue queue = new MessageQueue();
DestinationSessionParams params = new DestinationSessionParams().setMessageHandler(queue);
- SharedDestinationSession session = newDestinationSession(client.slobroksConfig(), params);
+ SharedDestinationSession session = newDestinationSession(client.slobrokId(), params);
Route route = Route.parse(session.connectionSpec());
assertTrue(client.sendMessage(new SimpleMessage("foo").setRoute(route)).isAccepted());
@@ -122,11 +120,11 @@ public class SharedDestinationSessionTestCase {
} catch (ListenFailedException e) {
fail();
}
- return newDestinationSession(TestUtils.configFor(slobrok), new DestinationSessionParams());
+ return newDestinationSession(slobrok.configId(), new DestinationSessionParams());
}
- private static SharedDestinationSession newDestinationSession(SlobroksConfig slobroksConfig, DestinationSessionParams params) {
- RPCNetworkParams netParams = new RPCNetworkParams().setSlobroksConfig(slobroksConfig);
+ private static SharedDestinationSession newDestinationSession(String slobrokId, DestinationSessionParams params) {
+ RPCNetworkParams netParams = new RPCNetworkParams().setSlobrokConfigId(slobrokId);
MessageBusParams mbusParams = new MessageBusParams().addProtocol(new SimpleProtocol());
SharedMessageBus mbus = SharedMessageBus.newInstance(mbusParams, netParams);
SharedDestinationSession session = mbus.newDestinationSession(params);
diff --git a/container-messagebus/src/test/java/com/yahoo/messagebus/shared/SharedIntermediateSessionTestCase.java b/container-messagebus/src/test/java/com/yahoo/messagebus/shared/SharedIntermediateSessionTestCase.java
index ca8301d1ea9..87958415149 100644
--- a/container-messagebus/src/test/java/com/yahoo/messagebus/shared/SharedIntermediateSessionTestCase.java
+++ b/container-messagebus/src/test/java/com/yahoo/messagebus/shared/SharedIntermediateSessionTestCase.java
@@ -1,7 +1,6 @@
// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.messagebus.shared;
-import com.yahoo.cloud.config.SlobroksConfig;
import com.yahoo.jrt.ListenFailedException;
import com.yahoo.jrt.slobrok.server.Slobrok;
import com.yahoo.messagebus.*;
@@ -9,7 +8,6 @@ import com.yahoo.messagebus.jdisc.test.MessageQueue;
import com.yahoo.messagebus.jdisc.test.RemoteClient;
import com.yahoo.messagebus.jdisc.test.RemoteServer;
import com.yahoo.messagebus.jdisc.test.ReplyQueue;
-import com.yahoo.messagebus.jdisc.test.TestUtils;
import com.yahoo.messagebus.network.local.LocalNetwork;
import com.yahoo.messagebus.network.local.LocalWire;
import com.yahoo.messagebus.network.rpc.RPCNetworkParams;
@@ -79,7 +77,7 @@ public class SharedIntermediateSessionTestCase {
public void requireThatReplyHandlerCanNotBeSet() throws ListenFailedException {
Slobrok slobrok = new Slobrok();
try {
- newIntermediateSession(TestUtils.configFor(slobrok),
+ newIntermediateSession(slobrok.configId(),
new IntermediateSessionParams().setReplyHandler(new ReplyQueue()),
false);
fail();
@@ -113,7 +111,7 @@ public class SharedIntermediateSessionTestCase {
@Test
public void requireThatSessionCanSendMessage() throws InterruptedException {
RemoteServer server = RemoteServer.newInstanceWithInternSlobrok();
- SharedIntermediateSession session = newIntermediateSession(server.slobroksConfig(),
+ SharedIntermediateSession session = newIntermediateSession(server.slobrokId(),
new IntermediateSessionParams(),
true);
ReplyQueue queue = new ReplyQueue();
@@ -136,7 +134,7 @@ public class SharedIntermediateSessionTestCase {
RemoteClient client = RemoteClient.newInstanceWithInternSlobrok(true);
MessageQueue queue = new MessageQueue();
IntermediateSessionParams params = new IntermediateSessionParams().setMessageHandler(queue);
- SharedIntermediateSession session = newIntermediateSession(client.slobroksConfig(), params, true);
+ SharedIntermediateSession session = newIntermediateSession(client.slobrokId(), params, true);
Route route = Route.parse(session.connectionSpec());
assertTrue(client.sendMessage(new SimpleMessage("foo").setRoute(route)).isAccepted());
@@ -158,13 +156,13 @@ public class SharedIntermediateSessionTestCase {
} catch (ListenFailedException e) {
fail();
}
- return newIntermediateSession(TestUtils.configFor(slobrok), new IntermediateSessionParams(), network);
+ return newIntermediateSession(slobrok.configId(), new IntermediateSessionParams(), network);
}
- private static SharedIntermediateSession newIntermediateSession(SlobroksConfig slobroksConfig,
+ private static SharedIntermediateSession newIntermediateSession(String slobrokId,
IntermediateSessionParams params,
boolean network) {
- RPCNetworkParams netParams = new RPCNetworkParams().setSlobroksConfig(slobroksConfig);
+ RPCNetworkParams netParams = new RPCNetworkParams().setSlobrokConfigId(slobrokId);
MessageBusParams mbusParams = new MessageBusParams().addProtocol(new SimpleProtocol());
SharedMessageBus mbus = network
? SharedMessageBus.newInstance(mbusParams, netParams)
diff --git a/container-messagebus/src/test/java/com/yahoo/messagebus/shared/SharedSourceSessionTestCase.java b/container-messagebus/src/test/java/com/yahoo/messagebus/shared/SharedSourceSessionTestCase.java
index 7a4a3d17170..1f0966fc961 100644
--- a/container-messagebus/src/test/java/com/yahoo/messagebus/shared/SharedSourceSessionTestCase.java
+++ b/container-messagebus/src/test/java/com/yahoo/messagebus/shared/SharedSourceSessionTestCase.java
@@ -1,7 +1,6 @@
// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.messagebus.shared;
-import com.yahoo.cloud.config.SlobroksConfig;
import com.yahoo.jrt.ListenFailedException;
import com.yahoo.jrt.slobrok.server.Slobrok;
import com.yahoo.messagebus.Message;
@@ -9,7 +8,6 @@ import com.yahoo.messagebus.MessageBusParams;
import com.yahoo.messagebus.SourceSessionParams;
import com.yahoo.messagebus.jdisc.test.RemoteServer;
import com.yahoo.messagebus.jdisc.test.ReplyQueue;
-import com.yahoo.messagebus.jdisc.test.TestUtils;
import com.yahoo.messagebus.network.rpc.RPCNetworkParams;
import com.yahoo.messagebus.routing.Route;
import com.yahoo.messagebus.test.SimpleMessage;
@@ -61,7 +59,7 @@ public class SharedSourceSessionTestCase {
@Test
public void requireThatSessionCanSendMessage() throws InterruptedException {
RemoteServer server = RemoteServer.newInstanceWithInternSlobrok();
- SharedSourceSession session = newSourceSession(server.slobroksConfig(),
+ SharedSourceSession session = newSourceSession(server.slobrokId(),
new SourceSessionParams());
ReplyQueue queue = new ReplyQueue();
Message msg = new SimpleMessage("foo").setRoute(Route.parse(server.connectionSpec()));
@@ -82,11 +80,11 @@ public class SharedSourceSessionTestCase {
} catch (ListenFailedException e) {
fail();
}
- return newSourceSession(TestUtils.configFor(slobrok), params);
+ return newSourceSession(slobrok.configId(), params);
}
- private static SharedSourceSession newSourceSession(SlobroksConfig slobroksConfig, SourceSessionParams params) {
- RPCNetworkParams netParams = new RPCNetworkParams().setSlobroksConfig(slobroksConfig);
+ private static SharedSourceSession newSourceSession(String slobrokId, SourceSessionParams params) {
+ RPCNetworkParams netParams = new RPCNetworkParams().setSlobrokConfigId(slobrokId);
MessageBusParams mbusParams = new MessageBusParams().addProtocol(new SimpleProtocol());
SharedMessageBus mbus = SharedMessageBus.newInstance(mbusParams, netParams);
SharedSourceSession session = mbus.newSourceSession(params);
diff --git a/container-messagebus/src/test/resources/config/clientprovider/container-mbus.cfg b/container-messagebus/src/test/resources/config/clientprovider/container-mbus.cfg
new file mode 100644
index 00000000000..e69de29bb2d
--- /dev/null
+++ b/container-messagebus/src/test/resources/config/clientprovider/container-mbus.cfg
diff --git a/container-messagebus/src/test/resources/config/clientprovider/documentmanager.cfg b/container-messagebus/src/test/resources/config/clientprovider/documentmanager.cfg
new file mode 100644
index 00000000000..e69de29bb2d
--- /dev/null
+++ b/container-messagebus/src/test/resources/config/clientprovider/documentmanager.cfg
diff --git a/container-messagebus/src/test/resources/config/clientprovider/load-type.cfg b/container-messagebus/src/test/resources/config/clientprovider/load-type.cfg
new file mode 100644
index 00000000000..e69de29bb2d
--- /dev/null
+++ b/container-messagebus/src/test/resources/config/clientprovider/load-type.cfg
diff --git a/container-messagebus/src/test/resources/config/clientprovider/messagebus.cfg b/container-messagebus/src/test/resources/config/clientprovider/messagebus.cfg
new file mode 100644
index 00000000000..e69de29bb2d
--- /dev/null
+++ b/container-messagebus/src/test/resources/config/clientprovider/messagebus.cfg
diff --git a/container-messagebus/src/test/resources/config/clientprovider/slobroks.cfg b/container-messagebus/src/test/resources/config/clientprovider/slobroks.cfg
new file mode 100644
index 00000000000..e69de29bb2d
--- /dev/null
+++ b/container-messagebus/src/test/resources/config/clientprovider/slobroks.cfg
diff --git a/controller-api/pom.xml b/controller-api/pom.xml
index 2f01f45edaa..02a7028b8ca 100644
--- a/controller-api/pom.xml
+++ b/controller-api/pom.xml
@@ -27,6 +27,13 @@
<dependency>
<groupId>com.yahoo.vespa</groupId>
+ <artifactId>container-apache-http-client-bundle</artifactId>
+ <version>${project.version}</version>
+ <scope>provided</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>com.yahoo.vespa</groupId>
<artifactId>serviceview</artifactId>
<scope>provided</scope>
<version>${project.version}</version>
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServer.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServer.java
index 81c4d7be483..8d9f20a7cee 100644
--- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServer.java
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServer.java
@@ -55,7 +55,7 @@ public interface ConfigServer {
Map<?,?> getServiceApiResponse(DeploymentId deployment, String serviceName, String restPath);
- String getClusterControllerStatus(DeploymentId deployment, String node, String subPath);
+ String getServiceStatusPage(DeploymentId deployment, String serviceName, String node, String subPath);
/**
* Gets the Vespa logs of the given deployment.
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/ApplicationSummary.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/ApplicationSummary.java
index 1d23cd52d23..01045eec020 100644
--- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/ApplicationSummary.java
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/ApplicationSummary.java
@@ -3,6 +3,7 @@ package com.yahoo.vespa.hosted.controller.api.integration.organization;
import com.yahoo.config.provision.ApplicationId;
import com.yahoo.config.provision.zone.ZoneId;
+import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId;
import java.time.Instant;
import java.util.Map;
@@ -20,10 +21,10 @@ public class ApplicationSummary {
private final Optional<Instant> lastQueried;
private final Optional<Instant> lastWritten;
private final Optional<Instant> lastBuilt;
- private final Map<ZoneId, Metric> metrics;
+ private final Map<DeploymentId, Metric> metrics;
public ApplicationSummary(ApplicationId application, Optional<Instant> lastQueried, Optional<Instant> lastWritten,
- Optional<Instant> lastBuilt, Map<ZoneId, Metric> metrics) {
+ Optional<Instant> lastBuilt, Map<DeploymentId, Metric> metrics) {
this.application = Objects.requireNonNull(application);
this.lastQueried = Objects.requireNonNull(lastQueried);
this.lastWritten = Objects.requireNonNull(lastWritten);
@@ -47,7 +48,7 @@ public class ApplicationSummary {
return lastBuilt;
}
- public Map<ZoneId, Metric> metrics() {
+ public Map<DeploymentId, Metric> metrics() {
return metrics;
}
diff --git a/controller-server/pom.xml b/controller-server/pom.xml
index 0c05f7d70bb..a0d05588a0d 100644
--- a/controller-server/pom.xml
+++ b/controller-server/pom.xml
@@ -20,6 +20,13 @@
<!-- provided -->
<dependency>
+ <groupId>com.yahoo.vespa</groupId>
+ <artifactId>container-apache-http-client-bundle</artifactId>
+ <version>${project.version}</version>
+ <scope>provided</scope>
+ </dependency>
+
+ <dependency>
<groupId>com.yahoo.vespa</groupId>
<artifactId>controller-api</artifactId>
<version>${project.version}</version>
@@ -128,16 +135,6 @@
</dependency>
<dependency>
- <groupId>org.apache.httpcomponents</groupId>
- <artifactId>httpcore</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.apache.httpcomponents</groupId>
- <artifactId>httpclient</artifactId>
- </dependency>
-
- <dependency>
<groupId>com.auth0</groupId>
<artifactId>java-jwt</artifactId>
<exclusions>
@@ -167,6 +164,14 @@
<scope>compile</scope>
</dependency>
+ <dependency>
+ <!-- required by java-jwt -->
+ <groupId>commons-codec</groupId>
+ <artifactId>commons-codec</artifactId>
+ <version>${commons.codec.version}</version>
+ <scope>compile</scope>
+ </dependency>
+
<!-- test -->
<dependency>
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java
index a1566a1b5a1..3b6a03a76c9 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java
@@ -66,7 +66,6 @@ import java.time.Instant;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
-import java.util.Locale;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
@@ -89,12 +88,15 @@ import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.installatio
import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.outOfCapacity;
import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.running;
import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.testFailure;
+import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.succeeded;
+import static com.yahoo.vespa.hosted.controller.deployment.Step.copyVespaLogs;
import static com.yahoo.vespa.hosted.controller.deployment.Step.deactivateReal;
import static com.yahoo.vespa.hosted.controller.deployment.Step.deactivateTester;
import static com.yahoo.vespa.hosted.controller.deployment.Step.deployInitialReal;
import static com.yahoo.vespa.hosted.controller.deployment.Step.deployReal;
import static com.yahoo.vespa.hosted.controller.deployment.Step.deployTester;
import static com.yahoo.vespa.hosted.controller.deployment.Step.installTester;
+import static com.yahoo.vespa.hosted.controller.deployment.Step.report;
import static java.nio.charset.StandardCharsets.UTF_8;
import static java.util.Objects.requireNonNull;
import static java.util.logging.Level.INFO;
@@ -188,11 +190,20 @@ public class InternalStepRunner implements StepRunner {
}
private Optional<RunStatus> deployReal(RunId id, boolean setTheStage, DualLogger logger) {
+ Optional<X509Certificate> testerCertificate = controller.jobController().run(id).get().testerCertificate();
return deploy(() -> controller.applications().deploy(id.job(), setTheStage),
controller.jobController().run(id).get()
.stepInfo(setTheStage ? deployInitialReal : deployReal).get()
.startTime().get(),
- logger);
+ logger)
+ .filter(result -> {
+ // If no tester cert, or deployment failed, propagate original result.
+ if (testerCertificate.isEmpty() || result != running)
+ return true;
+ // If tester cert, ensure real is deployed with the tester cert whose key was successfully deployed.
+ return controller.jobController().run(id).get().stepStatus(deployTester).get() == succeeded
+ && testerCertificate.equals(controller.jobController().run(id).get().testerCertificate());
+ });
}
private Optional<RunStatus> deployTester(RunId id, DualLogger logger) {
@@ -249,7 +260,7 @@ public class InternalStepRunner implements StepRunner {
case OUT_OF_CAPACITY:
logger.log(e.message());
return controller.system().isCd() && startTime.plus(timeouts.capacity()).isAfter(controller.clock().instant())
- ? Optional.empty()
+ ? result
: Optional.of(outOfCapacity);
case INVALID_APPLICATION_PACKAGE:
case BAD_REQUEST:
@@ -636,6 +647,19 @@ public class InternalStepRunner implements StepRunner {
try {
controller.jobController().updateVespaLog(id);
}
+ // Hitting a config server which doesn't have this particular app loaded causes a 404.
+ catch (ConfigServerException e) {
+ Instant doom = controller.jobController().run(id).get().stepInfo(copyVespaLogs).get().startTime().get()
+ .plus(Duration.ofMinutes(3));
+ if (e.code() == ConfigServerException.ErrorCode.NOT_FOUND && controller.clock().instant().isBefore(doom)) {
+ logger.log(INFO, "Found no logs, but will retry");
+ return Optional.empty();
+ }
+ else {
+ logger.log(INFO, "Failure getting vespa logs for " + id, e);
+ return Optional.of(error);
+ }
+ }
catch (Exception e) {
logger.log(INFO, "Failure getting vespa logs for " + id, e);
return Optional.of(error);
@@ -685,6 +709,12 @@ public class InternalStepRunner implements StepRunner {
logger.log(INFO, "Job '" + id.type() + "' no longer supposed to run?", e);
return Optional.of(error);
}
+ catch (RuntimeException e) {
+ Instant start = controller.jobController().run(id).get().stepInfo(report).get().startTime().get();
+ return (controller.clock().instant().isAfter(start.plusSeconds(180)))
+ ? Optional.empty()
+ : Optional.of(error);
+ }
return Optional.of(running);
}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java
index 25bc21a0076..d9d82b41e71 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java
@@ -48,6 +48,7 @@ import java.util.concurrent.atomic.AtomicReference;
import java.util.function.Consumer;
import java.util.function.UnaryOperator;
import java.util.logging.Level;
+import java.util.stream.Collectors;
import java.util.stream.Stream;
import static com.google.common.collect.ImmutableList.copyOf;
@@ -230,6 +231,14 @@ public class JobController {
return runs(id.application(), id.type());
}
+ /** Lists the start time of non-redeployment runs of the given job, in order of increasing age. */
+ public List<Instant> jobStarts(JobId id) {
+ return runs(id).descendingMap().values().stream()
+ .filter(run -> ! run.isRedeployment())
+ .map(Run::start)
+ .collect(toUnmodifiableList());
+ }
+
/** Returns an immutable map of all known runs for the given application and job type. */
public NavigableMap<RunId, Run> runs(ApplicationId id, JobType type) {
ImmutableSortedMap.Builder<RunId, Run> runs = ImmutableSortedMap.orderedBy(Comparator.comparing(RunId::number));
@@ -359,7 +368,7 @@ public class JobController {
List<Lock> locks = new ArrayList<>();
try {
// Ensure no step is still running before we finish the run — report depends transitively on all the other steps.
- for (Step step : report.allPrerequisites())
+ for (Step step : report.allPrerequisites(run(id).get().steps().keySet()))
locks.add(curator.lock(id.application(), id.type(), step));
locked(id, run -> { // Store the modified run after it has been written to history, in case the latter fails.
@@ -440,18 +449,23 @@ public class JobController {
/** Orders a run of the given type, or throws an IllegalStateException if that job type is already running. */
public void start(ApplicationId id, JobType type, Versions versions) {
- start(id, type, versions, JobProfile.of(type));
+ start(id, type, versions, false);
+ }
+
+ /** Orders a run of the given type, or throws an IllegalStateException if that job type is already running. */
+ public void start(ApplicationId id, JobType type, Versions versions, boolean isRedeployment) {
+ start(id, type, versions, isRedeployment, JobProfile.of(type));
}
/** Orders a run of the given type, or throws an IllegalStateException if that job type is already running. */
- public void start(ApplicationId id, JobType type, Versions versions, JobProfile profile) {
+ public void start(ApplicationId id, JobType type, Versions versions, boolean isRedeployment, JobProfile profile) {
locked(id, type, __ -> {
Optional<Run> last = last(id, type);
if (last.flatMap(run -> active(run.id())).isPresent())
throw new IllegalStateException("Can not start " + type + " for " + id + "; it is already running!");
RunId newId = new RunId(id, type, last.map(run -> run.id().number()).orElse(0L) + 1);
- curator.writeLastRun(Run.initial(newId, versions, controller.clock().instant(), profile));
+ curator.writeLastRun(Run.initial(newId, versions, isRedeployment, controller.clock().instant(), profile));
metric.jobStarted(newId.job());
});
}
@@ -477,6 +491,7 @@ public class JobController {
ApplicationVersion.unknown,
Optional.empty(),
Optional.empty()),
+ false,
JobProfile.development);
});
@@ -573,7 +588,7 @@ public class JobController {
/** Locks the given step and checks none of its prerequisites are running, then performs the given actions. */
public void locked(ApplicationId id, JobType type, Step step, Consumer<LockedStep> action) throws TimeoutException {
try (Lock lock = curator.lock(id, type, step)) {
- for (Step prerequisite : step.allPrerequisites()) // Check that no prerequisite is still running.
+ for (Step prerequisite : step.allPrerequisites(last(id, type).get().steps().keySet())) // Check that no prerequisite is still running.
try (Lock __ = curator.lock(id, type, prerequisite)) { ; }
action.accept(new LockedStep(lock, step));
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Run.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Run.java
index d2481cd97ad..d93a0133c97 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Run.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Run.java
@@ -30,6 +30,7 @@ public class Run {
private final RunId id;
private final Map<Step, StepInfo> steps;
private final Versions versions;
+ private final boolean isRedeployment;
private final Instant start;
private final Optional<Instant> end;
private final RunStatus status;
@@ -40,12 +41,13 @@ public class Run {
private final Optional<X509Certificate> testerCertificate;
// For deserialisation only -- do not use!
- public Run(RunId id, Map<Step, StepInfo> steps, Versions versions, Instant start, Optional<Instant> end,
+ public Run(RunId id, Map<Step, StepInfo> steps, Versions versions, boolean isRedeployment, Instant start, Optional<Instant> end,
RunStatus status, long lastTestRecord, Instant lastVespaLogTimestamp, Optional<Instant> noNodesDownSince,
Optional<ConvergenceSummary> convergenceSummary, Optional<X509Certificate> testerCertificate) {
this.id = id;
this.steps = Collections.unmodifiableMap(new EnumMap<>(steps));
this.versions = versions;
+ this.isRedeployment = isRedeployment;
this.start = start;
this.end = end;
this.status = status;
@@ -56,10 +58,10 @@ public class Run {
this.testerCertificate = testerCertificate;
}
- public static Run initial(RunId id, Versions versions, Instant now, JobProfile profile) {
+ public static Run initial(RunId id, Versions versions, boolean isRedeployment, Instant now, JobProfile profile) {
EnumMap<Step, StepInfo> steps = new EnumMap<>(Step.class);
profile.steps().forEach(step -> steps.put(step, StepInfo.initial(step)));
- return new Run(id, steps, requireNonNull(versions), requireNonNull(now), Optional.empty(), running,
+ return new Run(id, steps, requireNonNull(versions), isRedeployment, requireNonNull(now), Optional.empty(), running,
-1, Instant.EPOCH, Optional.empty(), Optional.empty(), Optional.empty());
}
@@ -73,7 +75,7 @@ public class Run {
EnumMap<Step, StepInfo> steps = new EnumMap<>(this.steps);
steps.put(step.get(), stepInfo.with(Step.Status.of(status)));
- return new Run(id, steps, versions, start, end, this.status == running ? status : this.status,
+ return new Run(id, steps, versions, isRedeployment, start, end, this.status == running ? status : this.status,
lastTestRecord, lastVespaLogTimestamp, noNodesDownSince, convergenceSummary, testerCertificate);
}
@@ -88,49 +90,49 @@ public class Run {
EnumMap<Step, StepInfo> steps = new EnumMap<>(this.steps);
steps.put(step.get(), stepInfo.with(startTime));
- return new Run(id, steps, versions, start, end, status, lastTestRecord, lastVespaLogTimestamp,
+ return new Run(id, steps, versions, isRedeployment, start, end, status, lastTestRecord, lastVespaLogTimestamp,
noNodesDownSince, convergenceSummary, testerCertificate);
}
public Run finished(Instant now) {
requireActive();
- return new Run(id, steps, versions, start, Optional.of(now), status == running ? success : status,
+ return new Run(id, steps, versions, isRedeployment, start, Optional.of(now), status == running ? success : status,
lastTestRecord, lastVespaLogTimestamp, noNodesDownSince, convergenceSummary, Optional.empty());
}
public Run aborted() {
requireActive();
- return new Run(id, steps, versions, start, end, aborted, lastTestRecord, lastVespaLogTimestamp,
+ return new Run(id, steps, versions, isRedeployment, start, end, aborted, lastTestRecord, lastVespaLogTimestamp,
noNodesDownSince, convergenceSummary, testerCertificate);
}
public Run with(long lastTestRecord) {
requireActive();
- return new Run(id, steps, versions, start, end, status, lastTestRecord, lastVespaLogTimestamp,
+ return new Run(id, steps, versions, isRedeployment, start, end, status, lastTestRecord, lastVespaLogTimestamp,
noNodesDownSince, convergenceSummary, testerCertificate);
}
public Run with(Instant lastVespaLogTimestamp) {
requireActive();
- return new Run(id, steps, versions, start, end, status, lastTestRecord, lastVespaLogTimestamp,
+ return new Run(id, steps, versions, isRedeployment, start, end, status, lastTestRecord, lastVespaLogTimestamp,
noNodesDownSince, convergenceSummary, testerCertificate);
}
public Run noNodesDownSince(Instant noNodesDownSince) {
requireActive();
- return new Run(id, steps, versions, start, end, status, lastTestRecord, lastVespaLogTimestamp,
+ return new Run(id, steps, versions, isRedeployment, start, end, status, lastTestRecord, lastVespaLogTimestamp,
Optional.ofNullable(noNodesDownSince), convergenceSummary, testerCertificate);
}
public Run withSummary(ConvergenceSummary convergenceSummary) {
requireActive();
- return new Run(id, steps, versions, start, end, status, lastTestRecord, lastVespaLogTimestamp,
+ return new Run(id, steps, versions, isRedeployment, start, end, status, lastTestRecord, lastVespaLogTimestamp,
noNodesDownSince, Optional.ofNullable(convergenceSummary), testerCertificate);
}
public Run with(X509Certificate testerCertificate) {
requireActive();
- return new Run(id, steps, versions, start, end, status, lastTestRecord, lastVespaLogTimestamp,
+ return new Run(id, steps, versions, isRedeployment, start, end, status, lastTestRecord, lastVespaLogTimestamp,
noNodesDownSince, convergenceSummary, Optional.of(testerCertificate));
}
@@ -222,6 +224,11 @@ public class Run {
return testerCertificate;
}
+ /** Whether this is a automatic redeployment. */
+ public boolean isRedeployment() {
+ return isRedeployment;
+ }
+
@Override
public boolean equals(Object o) {
if (this == o) return true;
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Step.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Step.java
index baba4771370..ce34a021218 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Step.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Step.java
@@ -1,10 +1,12 @@
// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.hosted.controller.deployment;
+import java.util.Collection;
import java.util.List;
import java.util.stream.Collectors;
import java.util.stream.Stream;
+import static java.util.stream.Collectors.toList;
import static java.util.stream.Collectors.toUnmodifiableList;
/**
@@ -34,7 +36,7 @@ public enum Step {
installTester(false, deployTester),
/** Download and deploy the initial real application, for staging tests. */
- deployInitialReal(false, deployTester),
+ deployInitialReal(false),
/** See that the real application has had its nodes converge to the initial state. */
installInitialReal(false, deployInitialReal),
@@ -46,7 +48,7 @@ public enum Step {
endStagingSetup(false, startStagingSetup),
/** Download and deploy real application, restarting services if required. */
- deployReal(false, endStagingSetup, deployTester),
+ deployReal(false, endStagingSetup),
/** See that real application has had its nodes converge to the wanted version and generation. */
installReal(false, deployReal),
@@ -72,24 +74,24 @@ public enum Step {
private final boolean alwaysRun;
private final List<Step> prerequisites;
- private final List<Step> allPrerequisites;
Step(boolean alwaysRun, Step... prerequisites) {
this.alwaysRun = alwaysRun;
this.prerequisites = List.of(prerequisites);
- this.allPrerequisites = Stream.concat(Stream.of(prerequisites),
- Stream.of(prerequisites).flatMap(pre -> pre.allPrerequisites().stream()))
- .sorted()
- .distinct()
- .collect(toUnmodifiableList());
}
/** Returns whether this is a cleanup-step, and should always run, regardless of job outcome, when specified in a job. */
public boolean alwaysRun() { return alwaysRun; }
- /** Returns all prerequisite steps for this, recursively. */
- public List<Step> allPrerequisites() {
- return allPrerequisites;
+ /** Returns all prerequisite steps for this, including transient ones, in a job profile containing the given steps. */
+ public List<Step> allPrerequisites(Collection<Step> among) {
+ return prerequisites.stream()
+ .filter(among::contains)
+ .flatMap(pre -> Stream.concat(Stream.of(pre),
+ pre.allPrerequisites(among).stream()))
+ .sorted()
+ .distinct()
+ .collect(toList());
}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ApplicationOwnershipConfirmer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ApplicationOwnershipConfirmer.java
index 69e0eb26f16..de087737320 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ApplicationOwnershipConfirmer.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ApplicationOwnershipConfirmer.java
@@ -6,6 +6,7 @@ import com.yahoo.vespa.hosted.controller.Application;
import com.yahoo.vespa.hosted.controller.ApplicationController;
import com.yahoo.vespa.hosted.controller.Controller;
import com.yahoo.vespa.hosted.controller.Instance;
+import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId;
import com.yahoo.vespa.hosted.controller.api.integration.organization.ApplicationSummary;
import com.yahoo.vespa.hosted.controller.api.integration.organization.IssueId;
import com.yahoo.vespa.hosted.controller.api.integration.organization.OwnershipIssues;
@@ -17,7 +18,6 @@ import com.yahoo.yolean.Exceptions;
import java.time.Duration;
import java.util.HashMap;
-import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.logging.Level;
@@ -78,14 +78,15 @@ public class ApplicationOwnershipConfirmer extends ControllerMaintainer {
private ApplicationSummary summaryOf(TenantAndApplicationId application) {
var app = applications.requireApplication(application);
- var metrics = new HashMap<ZoneId, ApplicationSummary.Metric>();
+ var metrics = new HashMap<DeploymentId, ApplicationSummary.Metric>();
for (Instance instance : app.instances().values()) {
for (var kv : instance.deployments().entrySet()) {
var zone = kv.getKey();
var deploymentMetrics = kv.getValue().metrics();
- metrics.put(zone, new ApplicationSummary.Metric(deploymentMetrics.documentCount(),
- deploymentMetrics.queriesPerSecond(),
- deploymentMetrics.writesPerSecond()));
+ metrics.put(new DeploymentId(instance.id(), zone),
+ new ApplicationSummary.Metric(deploymentMetrics.documentCount(),
+ deploymentMetrics.queriesPerSecond(),
+ deploymentMetrics.writesPerSecond()));
}
}
return new ApplicationSummary(app.id().defaultInstance(), app.activity().lastQueried(), app.activity().lastWritten(),
@@ -112,20 +113,29 @@ public class ApplicationOwnershipConfirmer extends ControllerMaintainer {
}
private double updateConfirmedApplicationOwners() {
+ AtomicInteger attempts = new AtomicInteger(0);
+ AtomicInteger failures = new AtomicInteger(0);
applications()
- .withProjectId()
- .withProductionDeployment()
- .asList()
- .stream()
- .filter(application -> application.ownershipIssueId().isPresent())
- .forEach(application -> {
- IssueId ownershipIssueId = application.ownershipIssueId().get();
- ownershipIssues.getConfirmedOwner(ownershipIssueId).ifPresent(owner -> {
- controller().applications().lockApplicationIfPresent(application.id(), lockedApplication ->
- controller().applications().store(lockedApplication.withOwner(owner)));
- });
- });
- return 1.0;
+ .withProjectId()
+ .withProductionDeployment()
+ .asList()
+ .stream()
+ .filter(application -> application.ownershipIssueId().isPresent())
+ .forEach(application -> {
+ attempts.incrementAndGet();
+ IssueId issueId = application.ownershipIssueId().get();
+ try {
+ ownershipIssues.getConfirmedOwner(issueId).ifPresent(owner -> {
+ controller().applications().lockApplicationIfPresent(application.id(), lockedApplication ->
+ controller().applications().store(lockedApplication.withOwner(owner)));
+ });
+ }
+ catch (RuntimeException e) {
+ failures.incrementAndGet();
+ log.log(Level.INFO, "Exception caught when attempting to find confirmed owner of issue with id '" + issueId + "': " + Exceptions.toMessageString(e));
+ }
+ });
+ return asSuccessFactor(attempts.get(), failures.get());
}
private ApplicationList applications() {
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java
index 91dfed500e3..56bf870c7fc 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java
@@ -42,6 +42,7 @@ public class ControllerMaintenance extends AbstractComponent {
maintainers.add(upgrader);
maintainers.addAll(osUpgraders(controller, intervals.osUpgrader));
maintainers.add(new DeploymentExpirer(controller, intervals.defaultInterval));
+ maintainers.add(new DeploymentUpgrader(controller, intervals.defaultInterval));
maintainers.add(new DeploymentIssueReporter(controller, controller.serviceRegistry().deploymentIssues(), intervals.defaultInterval));
maintainers.add(new MetricsReporter(controller, metric));
maintainers.add(new OutstandingChangeDeployer(controller, intervals.outstandingChangeDeployer));
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentExpirer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentExpirer.java
index 9e3da506ca8..40191190eff 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentExpirer.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentExpirer.java
@@ -12,6 +12,7 @@ import com.yahoo.vespa.hosted.controller.deployment.Run;
import com.yahoo.yolean.Exceptions;
import java.time.Duration;
+import java.time.Instant;
import java.util.Optional;
import java.util.logging.Level;
@@ -62,9 +63,8 @@ public class DeploymentExpirer extends ControllerMaintainer {
.map(type -> new JobId(instance, type));
if (jobId.isEmpty()) return false;
- return controller().jobController().last(jobId.get())
- .flatMap(Run::end)
- .map(end -> end.plus(ttl.get()).isBefore(controller().clock().instant()))
+ return controller().jobController().jobStarts(jobId.get()).stream().findFirst()
+ .map(start -> start.plus(ttl.get()).isBefore(controller().clock().instant()))
.orElse(false);
}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentUpgrader.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentUpgrader.java
new file mode 100644
index 00000000000..a69af024b96
--- /dev/null
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentUpgrader.java
@@ -0,0 +1,92 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.maintenance;
+
+import com.yahoo.vespa.hosted.controller.Application;
+import com.yahoo.vespa.hosted.controller.Controller;
+import com.yahoo.vespa.hosted.controller.Instance;
+import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion;
+import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobId;
+import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType;
+import com.yahoo.vespa.hosted.controller.application.Deployment;
+import com.yahoo.vespa.hosted.controller.deployment.Versions;
+import com.yahoo.yolean.Exceptions;
+
+import java.time.Duration;
+import java.time.Instant;
+import java.util.Optional;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.logging.Level;
+
+/**
+ * Upgrades instances in manually deployed zones to the system version, at a convenient time.
+ *
+ * @author jonmv
+ */
+public class DeploymentUpgrader extends ControllerMaintainer {
+
+ public DeploymentUpgrader(Controller controller, Duration interval) {
+ super(controller, interval);
+ }
+
+ @Override
+ protected double maintain() {
+ AtomicInteger attempts = new AtomicInteger();
+ AtomicInteger failures = new AtomicInteger();
+ Versions target = new Versions(controller().readSystemVersion(), ApplicationVersion.unknown, Optional.empty(), Optional.empty());
+ for (Application application : controller().applications().readable())
+ for (Instance instance : application.instances().values())
+ for (Deployment deployment : instance.deployments().values())
+ try {
+ attempts.incrementAndGet();
+ JobId job = new JobId(instance.id(), JobType.from(controller().system(), deployment.zone()).get());
+ if ( ! deployment.zone().environment().isManuallyDeployed()) continue;
+ if ( ! deployment.version().isBefore(target.targetPlatform())) continue;
+ if ( controller().clock().instant().isBefore(controller().jobController().last(job).get().start().plus(Duration.ofDays(1)))) continue;
+ if ( ! isLikelyNightFor(job)) continue;
+
+ log.log(Level.FINE, "Upgrading deployment of " + instance.id() + " in " + deployment.zone());
+ controller().jobController().start(instance.id(), JobType.from(controller().system(), deployment.zone()).get(), target, true);
+ } catch (Exception e) {
+ failures.incrementAndGet();
+ log.log(Level.WARNING, "Failed upgrading " + deployment + " of " + instance +
+ ": " + Exceptions.toMessageString(e) + ". Retrying in " +
+ interval());
+ }
+ return asSuccessFactor(attempts.get(), failures.get());
+ }
+
+ private boolean isLikelyNightFor(JobId job) {
+ int hour = hourOf(controller().clock().instant());
+ int[] runStarts = controller().jobController().jobStarts(job).stream()
+ .mapToInt(DeploymentUpgrader::hourOf)
+ .toArray();
+ int localNight = mostLikelyWeeHour(runStarts);
+ return Math.abs(hour - localNight) <= 1;
+ }
+
+ static int mostLikelyWeeHour(int[] starts) {
+ double weight = 1;
+ double[] buckets = new double[24];
+ for (int start : starts)
+ buckets[start] += weight *= 0.8; // Weight more recent deployments higher.
+
+ int best = -1;
+ double min = Double.MAX_VALUE;
+ for (int i = 12; i < 36; i++) {
+ double sum = 0;
+ for (int j = -12; j < 12; j++)
+ sum += buckets[(i + j) % 24] / (Math.abs(j) + 1);
+
+ if (sum < min) {
+ min = sum;
+ best = i;
+ }
+ }
+ return (best + 2) % 24;
+ }
+
+ private static int hourOf(Instant instant) {
+ return (int) (instant.toEpochMilli() / 3_600_000 % 24);
+ }
+
+}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainer.java
index 85a69b0f338..cd7ce8c3fa6 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainer.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainer.java
@@ -86,14 +86,14 @@ public class EndpointCertificateMaintainer extends ControllerMaintainer {
}
/**
- * If it's been a week since the cert has been refreshed, re-trigger all prod deployment jobs.
+ * If it's been four days since the cert has been refreshed, re-trigger all prod deployment jobs.
*/
private void deployRefreshedCertificates() {
var now = clock.instant();
curator.readAllEndpointCertificateMetadata().forEach((applicationId, endpointCertificateMetadata) ->
endpointCertificateMetadata.lastRefreshed().ifPresent(lastRefreshTime -> {
Instant refreshTime = Instant.ofEpochSecond(lastRefreshTime);
- if (now.isAfter(refreshTime.plus(7, ChronoUnit.DAYS))) {
+ if (now.isAfter(refreshTime.plus(4, ChronoUnit.DAYS))) {
controller().applications().getInstance(applicationId)
.ifPresent(instance -> instance.productionDeployments().forEach((zone, deployment) -> {
if (deployment.at().isBefore(refreshTime)) {
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgradeScheduler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgradeScheduler.java
index 5942291f2b0..c2addbe9a67 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgradeScheduler.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgradeScheduler.java
@@ -47,10 +47,9 @@ public class OsUpgradeScheduler extends ControllerMaintainer {
if (currentTarget.isEmpty()) return;
if (upgradingToNewMajor(cloud)) return; // Skip further upgrades until major version upgrade is complete
- controller().upgradeOsIn(cloud,
- release.version(currentTarget.get(), now),
- release.upgradeBudget(),
- false);
+ Version version = release.version(currentTarget.get(), now);
+ if (!version.isAfter(currentTarget.get().osVersion().version())) return;
+ controller().upgradeOsIn(cloud, version, release.upgradeBudget(), false);
}
private boolean upgradingToNewMajor(CloudName cloud) {
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java
index 87527085237..8ffa4823ead 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java
@@ -83,6 +83,7 @@ class RunSerializer {
private static final String endField = "end";
private static final String statusField = "status";
private static final String versionsField = "versions";
+ private static final String isRedeploymentField = "isRedeployment";
private static final String platformVersionField = "platform";
private static final String repositoryField = "repository";
private static final String branchField = "branch";
@@ -131,6 +132,7 @@ class RunSerializer {
runObject.field(numberField).asLong()),
steps,
versionsFromSlime(runObject.field(versionsField)),
+ runObject.field(isRedeploymentField).asBool(),
SlimeUtils.instant(runObject.field(startField)),
SlimeUtils.optionalInstant(runObject.field(endField)),
runStatusOf(runObject.field(statusField).asString()),
@@ -177,7 +179,7 @@ class RunSerializer {
compileVersion, buildTime, sourceUrl, commit);
}
- // Don't change this — introduce a separate array instead.
+ // Don't change this — introduce a separate array instead.
private Optional<ConvergenceSummary> convergenceSummaryFrom(Inspector summaryArray) {
if ( ! summaryArray.valid()) return Optional.empty();
@@ -215,6 +217,7 @@ class RunSerializer {
private void toSlime(Run run, Cursor runObject) {
runObject.setString(applicationField, run.id().application().serializedForm());
runObject.setString(jobTypeField, run.id().type().jobName());
+ runObject.setBool(isRedeploymentField, run.isRedeployment());
runObject.setLong(numberField, run.id().number());
runObject.setLong(startField, run.start().toEpochMilli());
run.end().ifPresent(end -> runObject.setLong(endField, end.toEpochMilli()));
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java
index d022588c757..c1e6c362c83 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java
@@ -245,7 +245,7 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler {
if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/deploying")) return deploying(path.get("tenant"), path.get("application"), path.get("instance"), request);
if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/deploying/pin")) return deploying(path.get("tenant"), path.get("application"), path.get("instance"), request);
if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/job")) return JobControllerApiHandlerHelper.jobTypeResponse(controller, appIdFromPath(path), request.getUri());
- if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/job/{jobtype}")) return JobControllerApiHandlerHelper.runResponse(controller.jobController().runs(appIdFromPath(path), jobTypeFromPath(path)), request.getUri());
+ if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/job/{jobtype}")) return JobControllerApiHandlerHelper.runResponse(controller.jobController().runs(appIdFromPath(path), jobTypeFromPath(path)), Optional.ofNullable(request.getProperty("limit")), request.getUri());
if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/job/{jobtype}/package")) return devApplicationPackage(appIdFromPath(path), jobTypeFromPath(path));
if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/job/{jobtype}/test-config")) return testConfig(appIdFromPath(path), jobTypeFromPath(path));
if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/job/{jobtype}/run/{number}")) return JobControllerApiHandlerHelper.runDetailsResponse(controller.jobController(), runIdFromPath(path), request.getProperty("after"));
@@ -546,28 +546,32 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler {
if (controller.tenants().get(tenantName).isEmpty())
return ErrorResponse.notFoundError("Tenant '" + tenantName + "' does not exist");
+ List<Application> applications = applicationName.isEmpty() ?
+ controller.applications().asList(tenant) :
+ controller.applications().getApplication(TenantAndApplicationId.from(tenantName, applicationName.get()))
+ .map(List::of)
+ .orElseThrow(() -> new NotExistsException("Application '" + applicationName.get() + "' does not exist"));
+
Slime slime = new Slime();
Cursor applicationArray = slime.setArray();
- for (Application application : controller.applications().asList(tenant)) {
- if (applicationName.map(application.id().application().value()::equals).orElse(true)) {
- Cursor applicationObject = applicationArray.addObject();
- applicationObject.setString("tenant", application.id().tenant().value());
- applicationObject.setString("application", application.id().application().value());
- applicationObject.setString("url", withPath("/application/v4" +
- "/tenant/" + application.id().tenant().value() +
- "/application/" + application.id().application().value(),
- request.getUri()).toString());
- Cursor instanceArray = applicationObject.setArray("instances");
- for (InstanceName instance : showOnlyProductionInstances(request) ? application.productionInstances().keySet()
- : application.instances().keySet()) {
- Cursor instanceObject = instanceArray.addObject();
- instanceObject.setString("instance", instance.value());
- instanceObject.setString("url", withPath("/application/v4" +
- "/tenant/" + application.id().tenant().value() +
- "/application/" + application.id().application().value() +
- "/instance/" + instance.value(),
- request.getUri()).toString());
- }
+ for (Application application : applications) {
+ Cursor applicationObject = applicationArray.addObject();
+ applicationObject.setString("tenant", application.id().tenant().value());
+ applicationObject.setString("application", application.id().application().value());
+ applicationObject.setString("url", withPath("/application/v4" +
+ "/tenant/" + application.id().tenant().value() +
+ "/application/" + application.id().application().value(),
+ request.getUri()).toString());
+ Cursor instanceArray = applicationObject.setArray("instances");
+ for (InstanceName instance : showOnlyProductionInstances(request) ? application.productionInstances().keySet()
+ : application.instances().keySet()) {
+ Cursor instanceObject = instanceArray.addObject();
+ instanceObject.setString("instance", instance.value());
+ instanceObject.setString("url", withPath("/application/v4" +
+ "/tenant/" + application.id().tenant().value() +
+ "/application/" + application.id().application().value() +
+ "/instance/" + instance.value(),
+ request.getUri()).toString());
}
}
return new SlimeJsonResponse(slime);
@@ -1383,11 +1387,11 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler {
response.setString("yamasUrl", monitoringSystemUri(deploymentId).toString());
response.setString("version", deployment.version().toFullString());
response.setString("revision", deployment.applicationVersion().id());
- response.setLong("deployTimeEpochMs", deployment.at().toEpochMilli());
+ Instant lastDeploymentStart = lastDeploymentStart(deploymentId.applicationId(), deployment);
+ response.setLong("deployTimeEpochMs", lastDeploymentStart.toEpochMilli());
controller.zoneRegistry().getDeploymentTimeToLive(deploymentId.zoneId())
- .ifPresent(deploymentTimeToLive -> response.setLong("expiryTimeEpochMs", deployment.at().plus(deploymentTimeToLive).toEpochMilli()));
+ .ifPresent(deploymentTimeToLive -> response.setLong("expiryTimeEpochMs", lastDeploymentStart.plus(deploymentTimeToLive).toEpochMilli()));
- DeploymentStatus status = controller.jobController().deploymentStatus(application);
application.projectId().ifPresent(i -> response.setString("screwdriverId", String.valueOf(i)));
sourceRevisionToSlime(deployment.applicationVersion().source(), response);
@@ -1396,18 +1400,21 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler {
if (!instance.rotations().isEmpty() && deployment.zone().environment() == Environment.prod)
toSlime(instance.rotations(), instance.rotationStatus(), deployment, response);
- JobType.from(controller.system(), deployment.zone())
- .map(type -> new JobId(instance.id(), type))
- .map(status.jobSteps()::get)
- .ifPresent(stepStatus -> {
- JobControllerApiHandlerHelper.applicationVersionToSlime(
- response.setObject("applicationVersion"), deployment.applicationVersion());
- if (!status.jobsToRun().containsKey(stepStatus.job().get()))
- response.setString("status", "complete");
- else if (stepStatus.readyAt(instance.change()).map(controller.clock().instant()::isBefore).orElse(true))
- response.setString("status", "pending");
- else response.setString("status", "running");
- });
+ if (!deployment.zone().environment().isManuallyDeployed()) {
+ DeploymentStatus status = controller.jobController().deploymentStatus(application);
+ JobType.from(controller.system(), deployment.zone())
+ .map(type -> new JobId(instance.id(), type))
+ .map(status.jobSteps()::get)
+ .ifPresent(stepStatus -> {
+ JobControllerApiHandlerHelper.applicationVersionToSlime(
+ response.setObject("applicationVersion"), deployment.applicationVersion());
+ if (!status.jobsToRun().containsKey(stepStatus.job().get()))
+ response.setString("status", "complete");
+ else if (stepStatus.readyAt(instance.change()).map(controller.clock().instant()::isBefore).orElse(true))
+ response.setString("status", "pending");
+ else response.setString("status", "running");
+ });
+ }
}
response.setDouble("quota", deployment.quota().rate());
@@ -1435,6 +1442,11 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler {
metrics.instant().ifPresent(instant -> metricsObject.setLong("lastUpdated", instant.toEpochMilli()));
}
+ private Instant lastDeploymentStart(ApplicationId instanceId, Deployment deployment) {
+ return controller.jobController().jobStarts(new JobId(instanceId, JobType.from(controller.system(), deployment.zone()).get()))
+ .stream().findFirst().orElse(deployment.at());
+ }
+
private void toSlime(ApplicationVersion applicationVersion, Cursor object) {
if ( ! applicationVersion.isUnknown()) {
object.setLong("buildNumber", applicationVersion.buildNumber().getAsLong());
@@ -1666,9 +1678,9 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler {
private HttpResponse service(String tenantName, String applicationName, String instanceName, String environment, String region, String serviceName, String restPath, HttpRequest request) {
DeploymentId deploymentId = new DeploymentId(ApplicationId.from(tenantName, applicationName, instanceName), requireZone(environment, region));
- if ("container-clustercontroller".equals((serviceName)) && restPath.contains("/status/")) {
- String[] parts = restPath.split("/status/");
- String result = controller.serviceRegistry().configServer().getClusterControllerStatus(deploymentId, parts[0], parts[1]);
+ if (restPath.contains("/status/")) {
+ String[] parts = restPath.split("/status/", 2);
+ String result = controller.serviceRegistry().configServer().getServiceStatusPage(deploymentId, serviceName, parts[0], parts[1]);
return new HtmlResponse(result);
}
@@ -2079,12 +2091,16 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler {
controller.serviceRegistry().roleService().getTenantRole(tenant.name()),
cloudTenant.tenantSecretStores());
- var tenantQuota = controller.serviceRegistry().billingController().getQuota(tenant.name());
- var usedQuota = applications.stream()
- .map(Application::quotaUsage)
- .reduce(QuotaUsage.none, QuotaUsage::add);
+ try {
+ var tenantQuota = controller.serviceRegistry().billingController().getQuota(tenant.name());
+ var usedQuota = applications.stream()
+ .map(Application::quotaUsage)
+ .reduce(QuotaUsage.none, QuotaUsage::add);
- toSlime(tenantQuota, usedQuota, object.setObject("quota"));
+ toSlime(tenantQuota, usedQuota, object.setObject("quota"));
+ } catch (Exception e) {
+ log.warning(String.format("Failed to get quota for tenant %s: %s", tenant.name(), Exceptions.toMessageString(e)));
+ }
cloudTenant.archiveAccessRole().ifPresent(role -> object.setString("archiveAccessRole", role));
@@ -2187,9 +2203,9 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler {
private void tenantMetaDataToSlime(Tenant tenant, List<Application> applications, Cursor object) {
Optional<Instant> lastDev = applications.stream()
.flatMap(application -> application.instances().values().stream())
- .flatMap(instance -> instance.deployments().values().stream())
- .filter(deployment -> deployment.zone().environment() == Environment.dev)
- .map(Deployment::at)
+ .flatMap(instance -> instance.deployments().values().stream()
+ .filter(deployment -> deployment.zone().environment() == Environment.dev)
+ .map(deployment -> lastDeploymentStart(instance.id(), deployment)))
.max(Comparator.naturalOrder())
.or(() -> applications.stream()
.flatMap(application -> application.instances().values().stream())
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java
index d246b20f2fe..eb9b0e03f3d 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java
@@ -40,10 +40,12 @@ import java.time.Instant;
import java.time.format.TextStyle;
import java.util.Arrays;
import java.util.Collection;
+import java.util.Comparator;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Optional;
+import java.util.stream.Collectors;
import java.util.stream.Stream;
import static com.yahoo.config.application.api.DeploymentSpec.UpgradePolicy.canary;
@@ -84,7 +86,7 @@ class JobControllerApiHandlerHelper {
Cursor jobObject = jobsArray.addObject();
jobObject.setString("jobName", job.type().jobName());
- toSlime(jobObject.setArray("runs"), runs, baseUriForJobs);
+ toSlime(jobObject.setArray("runs"), runs, 10, baseUriForJobs);
});
return new SlimeJsonResponse(slime);
@@ -117,11 +119,19 @@ class JobControllerApiHandlerHelper {
}
/** Returns a response with the runs for the given job type. */
- static HttpResponse runResponse(Map<RunId, Run> runs, URI baseUriForJobType) {
+ static HttpResponse runResponse(Map<RunId, Run> runs, Optional<String> limitStr, URI baseUriForJobType) {
Slime slime = new Slime();
Cursor cursor = slime.setObject();
- runs.forEach((runid, run) -> runToSlime(cursor.setObject(Long.toString(runid.number())), run, baseUriForJobType));
+ // TODO (freva): Remove after console migrated to use new format
+ if (limitStr.isEmpty())
+ runs.forEach((runid, run) -> runToSlime(cursor.setObject(Long.toString(runid.number())), run, baseUriForJobType));
+ else {
+ int limit = limitStr.map(Integer::parseInt).orElse(Integer.MAX_VALUE);
+ toSlime(cursor.setArray("runs"), runs.values().stream()
+ .sorted(Comparator.comparing((Run run) -> run.id().number()).reversed())
+ .collect(Collectors.toUnmodifiableList()), limit, baseUriForJobType);
+ }
return new SlimeJsonResponse(slime);
}
@@ -377,7 +387,7 @@ class JobControllerApiHandlerHelper {
toSlime(runObject.setObject("versions"), versions);
}
- toSlime(stepObject.setArray("runs"), jobStatus.runs().descendingMap().values(), baseUriForJob);
+ toSlime(stepObject.setArray("runs"), jobStatus.runs().descendingMap().values(), 10, baseUriForJob);
});
}
@@ -428,8 +438,8 @@ class JobControllerApiHandlerHelper {
return Optional.of(versions.get(i));
}
- private static void toSlime(Cursor runsArray, Collection<Run> runs, URI baseUriForJob) {
- runs.stream().limit(10).forEach(run -> {
+ private static void toSlime(Cursor runsArray, Collection<Run> runs, int limit, URI baseUriForJob) {
+ runs.stream().limit(limit).forEach(run -> {
Cursor runObject = runsArray.addObject();
runObject.setLong("id", run.id().number());
runObject.setString("url", baseUriForJob.resolve(baseUriForJob.getPath() + "/run/" + run.id().number()).toString());
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/Badges.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/Badges.java
index 4951f2ba33b..c13a9b42382 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/Badges.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/Badges.java
@@ -113,7 +113,7 @@ public class Badges {
List<Run> runs = status.runs().descendingMap().values().stream()
.filter(Run::hasEnded)
.skip(1)
- .limit(length + (lastTriggered.hasEnded() ? 0 : 1))
+ .limit(length)
.collect(toList());
boolean isOk = status.lastCompleted().map(run -> run.status() == RunStatus.success).orElse(true);
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java
index 14244d7bdda..d8bddac4187 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java
@@ -33,7 +33,6 @@ import org.junit.Test;
import java.io.IOException;
import java.io.UncheckedIOException;
-import java.net.URI;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
@@ -58,6 +57,7 @@ import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.failed;
import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.succeeded;
import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.unfinished;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotEquals;
import static org.junit.Assert.assertTrue;
/**
@@ -383,28 +383,81 @@ public class InternalStepRunnerTest {
@Test
public void vespaLogIsCopied() {
+ // Tests fail. We should get logs. This fails too, on the first attempt.
RunId id = app.startSystemTestTests();
tester.cloud().set(TesterCloud.Status.ERROR);
- tester.configServer().setLogStream(vespaLog);
+ tester.configServer().setLogStream(() -> { throw new ConfigServerException(ConfigServerException.ErrorCode.NOT_FOUND, "404", "context"); });
long lastId = tester.jobs().details(id).get().lastId().getAsLong();
tester.runner().run();
assertEquals(failed, tester.jobs().run(id).get().stepStatuses().get(Step.endTests));
+ assertEquals(unfinished, tester.jobs().run(id).get().stepStatuses().get(Step.copyVespaLogs));
assertTestLogEntries(id, Step.copyVespaLogs,
- new LogEntry(lastId + 2, Instant.EPOCH.plus(3554970337935104L, ChronoUnit.MICROS), info,
+ new LogEntry(lastId + 2, tester.clock().instant(), info,
+ "Found no logs, but will retry"),
+ new LogEntry(lastId + 4, tester.clock().instant(), info,
+ "Found no logs, but will retry"));
+
+ // Config servers now provide the log, and we get it.
+ tester.configServer().setLogStream(() -> vespaLog);
+ tester.runner().run();
+ assertEquals(failed, tester.jobs().run(id).get().stepStatuses().get(Step.endTests));
+ assertTestLogEntries(id, Step.copyVespaLogs,
+ new LogEntry(lastId + 2, tester.clock().instant(), info,
+ "Found no logs, but will retry"),
+ new LogEntry(lastId + 4, tester.clock().instant(), info,
+ "Found no logs, but will retry"),
+ new LogEntry(lastId + 5, Instant.EPOCH.plus(3554970337935104L, ChronoUnit.MICROS), info,
"17491290-v6-1.ostk.bm2.prod.ne1.yahoo.com\tcontainer\tstdout\n" +
"ERROR: Bundle canary-application [71] Unable to get module class path. (java.lang.NullPointerException)"),
- new LogEntry(lastId + 3, Instant.EPOCH.plus(3554970337947777L, ChronoUnit.MICROS), info,
+ new LogEntry(lastId + 6, Instant.EPOCH.plus(3554970337947777L, ChronoUnit.MICROS), info,
"17491290-v6-1.ostk.bm2.prod.ne1.yahoo.com\tcontainer\tstdout\n" +
"ERROR: Bundle canary-application [71] Unable to get module class path. (java.lang.NullPointerException)"),
- new LogEntry(lastId + 4, Instant.EPOCH.plus(3554970337947820L, ChronoUnit.MICROS), info,
+ new LogEntry(lastId + 7, Instant.EPOCH.plus(3554970337947820L, ChronoUnit.MICROS), info,
"17491290-v6-1.ostk.bm2.prod.ne1.yahoo.com\tcontainer\tstdout\n" +
"ERROR: Bundle canary-application [71] Unable to get module class path. (java.lang.NullPointerException)"),
- new LogEntry(lastId + 5, Instant.EPOCH.plus(3554970337947845L, ChronoUnit.MICROS), warning,
+ new LogEntry(lastId + 8, Instant.EPOCH.plus(3554970337947845L, ChronoUnit.MICROS), warning,
"17491290-v6-1.ostk.bm2.prod.ne1.yahoo.com\tcontainer\tstderr\n" +
"java.lang.NullPointerException\n\tat org.apache.felix.framework.BundleRevisionImpl.calculateContentPath(BundleRevisionImpl.java:438)\n\tat org.apache.felix.framework.BundleRevisionImpl.initializeContentPath(BundleRevisionImpl.java:371)"));
}
@Test
+ public void realDeploymentRequiresForTesterCert() {
+ tester.controllerTester().zoneRegistry().setSystemName(SystemName.Public);
+ var zones = List.of(ZoneApiMock.fromId("test.aws-us-east-1c"),
+ ZoneApiMock.fromId("staging.aws-us-east-1c"),
+ ZoneApiMock.fromId("prod.aws-us-east-1c"));
+ tester.controllerTester().zoneRegistry()
+ .setZones(zones)
+ .setRoutingMethod(zones, RoutingMethod.exclusive);
+ tester.configServer().bootstrap(tester.controllerTester().zoneRegistry().zones().all().ids(), SystemApplication.values());
+ ZoneId testZone = JobType.systemTest.zone(tester.controller().system());
+ RunId id = app.newRun(JobType.systemTest);
+ tester.configServer().throwOnPrepare(instanceId -> {
+ if (instanceId.instance().isTester())
+ throw new ConfigServerException(ConfigServerException.ErrorCode.PARENT_HOST_NOT_READY, "provisioning", "deploy tester");
+ });
+ tester.runner().run();
+ assertEquals(unfinished, tester.jobs().run(id).get().stepStatuses().get(Step.deployTester));
+ assertEquals(unfinished, tester.jobs().run(id).get().stepStatuses().get(Step.deployReal));
+
+ List<X509Certificate> oldTrusted = new ArrayList<>(DeploymentContext.publicApplicationPackage().trustedCertificates());
+ X509Certificate oldCert = tester.jobs().run(id).get().testerCertificate().get();
+ oldTrusted.add(oldCert);
+ assertEquals(oldTrusted, tester.configServer().application(app.instanceId(), id.type().zone(system())).get().applicationPackage().trustedCertificates());
+
+ tester.configServer().throwOnNextPrepare(null);
+ tester.runner().run();
+ assertEquals(succeeded, tester.jobs().run(id).get().stepStatuses().get(Step.deployTester));
+ assertEquals(succeeded, tester.jobs().run(id).get().stepStatuses().get(Step.deployReal));
+
+ List<X509Certificate> newTrusted = new ArrayList<>(DeploymentContext.publicApplicationPackage().trustedCertificates());
+ X509Certificate newCert = tester.jobs().run(id).get().testerCertificate().get();
+ newTrusted.add(newCert);
+ assertEquals(newTrusted, tester.configServer().application(app.instanceId(), id.type().zone(system())).get().applicationPackage().trustedCertificates());
+ assertNotEquals(oldCert, newCert);
+ }
+
+ @Test
public void certificateTimeoutAbortsJob() {
tester.controllerTester().zoneRegistry().setSystemName(SystemName.Public);
var zones = List.of(ZoneApiMock.fromId("test.aws-us-east-1c"),
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java
index dca79c43afe..92c8cbc4889 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java
@@ -63,6 +63,11 @@ import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.UUID;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.function.BooleanSupplier;
+import java.util.function.Consumer;
+import java.util.function.Function;
+import java.util.function.Supplier;
import java.util.logging.Level;
import java.util.stream.Collectors;
@@ -93,9 +98,9 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer
private List<ProtonMetrics> protonMetrics;
private Version lastPrepareVersion = null;
- private RuntimeException prepareException = null;
+ private Consumer<ApplicationId> prepareException = null;
private ConfigChangeActions configChangeActions = null;
- private String log = "INFO - All good";
+ private Supplier<String> log = () -> "INFO - All good";
@Inject
public ConfigServerMock(ZoneRegistryMock zoneRegistry) {
@@ -205,9 +210,14 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer
return Optional.ofNullable(lastPrepareVersion);
}
+ /** Sets a function that may throw, determined by app id. */
+ public void throwOnPrepare(Consumer<ApplicationId> prepareThrower) {
+ this.prepareException = prepareThrower;
+ }
+
/** The exception to throw on the next prepare run, or null to continue normally */
public void throwOnNextPrepare(RuntimeException prepareException) {
- this.prepareException = prepareException;
+ this.prepareException = prepareException == null ? null : id -> { this.prepareException = null; throw prepareException; };
}
/** Set version for an application in a given zone */
@@ -366,11 +376,10 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer
@Override
public PreparedApplication deploy(DeploymentData deployment) {
lastPrepareVersion = deployment.platform();
- if (prepareException != null) {
- RuntimeException prepareException = this.prepareException;
- this.prepareException = null;
- throw prepareException;
- }
+ if (prepareException != null)
+ prepareException.accept(ApplicationId.from(deployment.instance().tenant(),
+ deployment.instance().application(),
+ deployment.instance().instance()));
DeploymentId id = new DeploymentId(deployment.instance(), deployment.zone());
applications.put(id, new Application(id.applicationId(), lastPrepareVersion, new ApplicationPackage(deployment.applicationPackage())));
@@ -523,7 +532,7 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer
}
@Override
- public String getClusterControllerStatus(DeploymentId deployment, String node, String subPath) {
+ public String getServiceStatusPage(DeploymentId deployment, String serviceName, String node, String subPath) {
return "<h1>OK</h1>";
}
@@ -554,7 +563,7 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer
@Override
public InputStream getLogs(DeploymentId deployment, Map<String, String> queryParameters) {
- return new ByteArrayInputStream(log.getBytes(StandardCharsets.UTF_8));
+ return new ByteArrayInputStream(log.get().getBytes(StandardCharsets.UTF_8));
}
@Override
@@ -562,7 +571,7 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer
return new ProxyResponse("{\"path\":\"" + path + "\"}", "application/json", 200);
}
- public void setLogStream(String log) {
+ public void setLogStream(Supplier<String> log) {
this.log = log;
}
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentExpirerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentExpirerTest.java
index ffc82f90ad4..31f8aaf9e2d 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentExpirerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentExpirerTest.java
@@ -70,7 +70,9 @@ public class DeploymentExpirerTest {
assertEquals(1, permanentDeployments(prodApp.instance()));
// Dev application expires when enough time has passed since most recent attempt
+ // Redeployments done by DeploymentUpgrader do not affect this
tester.clock().advance(Duration.ofDays(12).plus(Duration.ofSeconds(1)));
+ tester.jobs().start(devApp.instanceId(), JobType.devUsEast1, lastRun.versions(), true);
expirer.maintain();
assertEquals(0, permanentDeployments(devApp.instance()));
assertEquals(1, permanentDeployments(prodApp.instance()));
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentUpgraderTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentUpgraderTest.java
new file mode 100644
index 00000000000..7a8f775e8b1
--- /dev/null
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentUpgraderTest.java
@@ -0,0 +1,101 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.maintenance;
+
+import com.yahoo.component.Version;
+import com.yahoo.config.provision.Environment;
+import com.yahoo.config.provision.RegionName;
+import com.yahoo.config.provision.zone.ZoneId;
+import com.yahoo.vespa.hosted.controller.Instance;
+import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType;
+import com.yahoo.vespa.hosted.controller.application.ApplicationPackage;
+import com.yahoo.vespa.hosted.controller.application.Deployment;
+import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder;
+import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester;
+import com.yahoo.vespa.hosted.controller.deployment.Run;
+import com.yahoo.vespa.hosted.controller.deployment.RunStatus;
+import org.junit.Test;
+
+import java.time.Duration;
+import java.time.Instant;
+import java.time.temporal.ChronoUnit;
+import java.time.temporal.TemporalUnit;
+import java.util.Optional;
+
+import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.devUsEast1;
+import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.productionUsWest1;
+import static com.yahoo.vespa.hosted.controller.maintenance.DeploymentUpgrader.mostLikelyWeeHour;
+import static java.time.temporal.ChronoUnit.MILLIS;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertSame;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * @author jonmv
+ */
+public class DeploymentUpgraderTest {
+
+ private final DeploymentTester tester = new DeploymentTester();
+
+ @Test
+ public void testDeploymentUpgrading() {
+ ZoneId devZone = ZoneId.from(Environment.dev, RegionName.from("us-east-1"));
+ DeploymentUpgrader upgrader = new DeploymentUpgrader(tester.controller(), Duration.ofDays(1));
+ var devApp = tester.newDeploymentContext("tenant1", "app1", "default");
+ var prodApp = tester.newDeploymentContext("tenant2", "app2", "default");
+
+ ApplicationPackage appPackage = new ApplicationPackageBuilder().region("us-west-1").build();
+ Version systemVersion = tester.controller().readSystemVersion();
+ Instant start = tester.clock().instant().truncatedTo(MILLIS);
+
+ devApp.runJob(devUsEast1, appPackage);
+ prodApp.submit(appPackage).deploy();
+ assertEquals(systemVersion, tester.jobs().last(devApp.instanceId(), devUsEast1).get().versions().targetPlatform());
+ assertEquals(systemVersion, tester.jobs().last(prodApp.instanceId(), productionUsWest1).get().versions().targetPlatform());
+
+ // Not upgraded initially
+ upgrader.maintain();
+ assertEquals(start, tester.jobs().last(devApp.instanceId(), devUsEast1).get().start());
+ assertEquals(start, tester.jobs().last(prodApp.instanceId(), productionUsWest1).get().start());
+
+ // Not upgraded immediately after system upgrades
+ tester.controllerTester().upgradeSystem(new Version(7, 8, 9));
+ upgrader.maintain();
+ assertEquals(start, tester.jobs().last(devApp.instanceId(), devUsEast1).get().start());
+ assertEquals(start, tester.jobs().last(prodApp.instanceId(), productionUsWest1).get().start());
+
+ // 14 hours pass, but not upgraded before a day has passed since last deployment
+ tester.clock().advance(Duration.ofHours(14));
+ upgrader.maintain();
+ assertEquals(start, tester.jobs().last(devApp.instanceId(), devUsEast1).get().start());
+ assertEquals(start, tester.jobs().last(prodApp.instanceId(), productionUsWest1).get().start());
+
+ // 35 hours pass, but not upgraded since it's not likely in the middle of the night
+ tester.clock().advance(Duration.ofHours(21));
+ upgrader.maintain();
+ assertEquals(start, tester.jobs().last(devApp.instanceId(), devUsEast1).get().start());
+ assertEquals(start, tester.jobs().last(prodApp.instanceId(), productionUsWest1).get().start());
+
+ // 38 hours pass, and the dev deployment, only, is upgraded
+ tester.clock().advance(Duration.ofHours(3));
+ upgrader.maintain();
+ assertEquals(tester.clock().instant().truncatedTo(MILLIS), tester.jobs().last(devApp.instanceId(), devUsEast1).get().start());
+ assertTrue(tester.jobs().last(devApp.instanceId(), devUsEast1).get().isRedeployment());
+ assertEquals(start, tester.jobs().last(prodApp.instanceId(), productionUsWest1).get().start());
+ devApp.runJob(devUsEast1);
+
+ // After the upgrade, the dev app is mostly (re)deployed to at night, but this doesn't affect what is likely the night.
+ tester.controllerTester().upgradeSystem(new Version(7, 9, 11));
+ tester.clock().advance(Duration.ofHours(48));
+ upgrader.maintain();
+ assertEquals(tester.clock().instant().truncatedTo(MILLIS), tester.jobs().last(devApp.instanceId(), devUsEast1).get().start());
+ }
+
+ @Test
+ public void testNight() {
+ assertEquals(16, mostLikelyWeeHour(new int[]{ 0, 1, 2, 3, 4, 5, 6 }));
+ assertEquals(14, mostLikelyWeeHour(new int[]{ 22, 23, 0, 1, 2, 3, 4 }));
+ assertEquals(18, mostLikelyWeeHour(new int[]{ 6, 5, 4, 3, 2, 1, 0 }));
+ assertEquals(20, mostLikelyWeeHour(new int[]{ 0, 12, 0, 12, 0, 12, 0, 12, 0, 12, 0, 11 }));
+ }
+
+}
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainerTest.java
index ce219b8beed..3ed26b4fb6e 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainerTest.java
@@ -82,7 +82,7 @@ public class EndpointCertificateMaintainerTest {
}
@Test
- public void refreshed_certificate_is_deployed_after_one_week() {
+ public void refreshed_certificate_is_deployed_after_four_days() {
var appId = ApplicationId.from("tenant", "application", "default");
DeploymentTester deploymentTester = new DeploymentTester(tester);
@@ -107,7 +107,7 @@ public class EndpointCertificateMaintainerTest {
maintainer.maintain();
- tester.clock().advance(Duration.ofDays(8));
+ tester.clock().advance(Duration.ofDays(4));
deploymentContext.assertNotRunning(productionUsWest1);
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java
index f677cc079cc..023c5671b60 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java
@@ -131,8 +131,8 @@ public class JobRunnerTest {
Map<Step, Status> steps = run.get().stepStatuses();
runner.maintain();
assertEquals(steps, run.get().stepStatuses());
- assertEquals(List.of(deployTester), run.get().readySteps());
- assertStepsWithStartTime(run.get(), deployTester);
+ assertEquals(List.of(deployTester, deployReal), run.get().readySteps());
+ assertStepsWithStartTime(run.get(), deployTester, deployReal);
outcomes.put(deployTester, running);
runner.maintain();
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgradeSchedulerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgradeSchedulerTest.java
index d87dfcfa315..d80a8ce1152 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgradeSchedulerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgradeSchedulerTest.java
@@ -132,6 +132,16 @@ public class OsUpgradeSchedulerTest {
assertEquals("Target is unchanged as not enough time has passed", version1,
target1.osVersion().version());
assertEquals("Target is not re-scheduled", target0.scheduledAt(), target1.scheduledAt());
+
+ // A newer version is triggered manually
+ Version version3 = Version.fromString("8.3");
+ tester.controller().upgradeOsIn(cloud, version3, Duration.ZERO, false);
+
+ // Enough time passes for stable version to be promoted. Nothing happens as stable is now before the manually
+ // triggered version
+ tester.clock().advance(Duration.ofDays(14).plus(Duration.ofSeconds(1)));
+ scheduler.maintain();
+ assertEquals(version3, tester.controller().osVersionTarget(cloud).get().osVersion().version());
}
private static ZoneApi zone(String id, CloudName cloud) {
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializerTest.java
index 6e12373640d..03a050db74e 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializerTest.java
@@ -149,7 +149,7 @@ public class RunSerializerTest {
assertEquals(run.versions(), phoenix.versions());
assertEquals(run.steps(), phoenix.steps());
- Run initial = Run.initial(id, run.versions(), run.start(), JobProfile.production);
+ Run initial = Run.initial(id, run.versions(), run.isRedeployment(), run.start(), JobProfile.production);
assertEquals(initial, serializer.runFromSlime(serializer.toSlime(initial)));
}
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java
index 3fa593fcf64..a01097cfcb6 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java
@@ -225,6 +225,10 @@ public class ApplicationApiTest extends ControllerContainerTest {
// GET tenant applications
tester.assertResponse(request("/application/v4/tenant/tenant1/application/", GET).userIdentity(USER_ID),
new File("application-list.json"));
+ // GET tenant application instances for application that does not exist
+ tester.assertResponse(request("/application/v4/tenant/tenant1/application/fake-app/instance/", GET).userIdentity(USER_ID),
+ "{\"error-code\":\"NOT_FOUND\",\"message\":\"Application 'fake-app' does not exist\"}", 404);
+
// GET tenant applications (instances of "application1" only)
tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/", GET).userIdentity(USER_ID),
new File("application-list.json"));
@@ -801,7 +805,7 @@ public class ApplicationApiTest extends ControllerContainerTest {
// GET system test job overview.
tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/job/system-test", GET)
- .userIdentity(USER_ID),
+ .userIdentity(USER_ID).properties(Map.of("limit", "100")),
new File("system-test-job.json"));
// GET system test run 1 details.
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelperTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelperTest.java
index 1be7f16e85f..72295497c03 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelperTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelperTest.java
@@ -131,14 +131,14 @@ public class JobControllerApiHandlerHelperTest {
// Only us-east-3 is verified, on revision1.
// staging-test has 5 runs: one success without sources on revision1, one success from revision1 to revision2,
// one success from revision2 to revision3 and two failures from revision1 to revision3.
- assertResponse(JobControllerApiHandlerHelper.runResponse(tester.jobs().runs(app.instanceId(), stagingTest), URI.create("https://some.url:43/root")), "staging-runs.json");
+ assertResponse(JobControllerApiHandlerHelper.runResponse(tester.jobs().runs(app.instanceId(), stagingTest), Optional.empty(), URI.create("https://some.url:43/root")), "staging-runs.json");
assertResponse(JobControllerApiHandlerHelper.runDetailsResponse(tester.jobs(), tester.jobs().last(app.instanceId(), stagingTest).get().id(), "0"), "staging-test-log.json");
assertResponse(JobControllerApiHandlerHelper.runDetailsResponse(tester.jobs(), tester.jobs().last(app.instanceId(), productionUsEast3).get().id(), "0"), "us-east-3-log-without-first.json");
assertResponse(JobControllerApiHandlerHelper.jobTypeResponse(tester.controller(), app.instanceId(), URI.create("https://some.url:43/root/")), "overview.json");
var userApp = tester.newDeploymentContext(app.instanceId().tenant().value(), app.instanceId().application().value(), "user");
userApp.runJob(devAwsUsEast2a, applicationPackage);
- assertResponse(JobControllerApiHandlerHelper.runResponse(tester.jobs().runs(userApp.instanceId(), devAwsUsEast2a), URI.create("https://some.url:43/root")), "dev-aws-us-east-2a-runs.json");
+ assertResponse(JobControllerApiHandlerHelper.runResponse(tester.jobs().runs(userApp.instanceId(), devAwsUsEast2a), Optional.empty(), URI.create("https://some.url:43/root")), "dev-aws-us-east-2a-runs.json");
assertResponse(JobControllerApiHandlerHelper.jobTypeResponse(tester.controller(), userApp.instanceId(), URI.create("https://some.url:43/root/")), "overview-user-instance.json");
assertResponse(JobControllerApiHandlerHelper.overviewResponse(tester.controller(), app.application().id(), URI.create("https://some.url:43/root/")), "deployment-overview-2.json");
}
@@ -151,10 +151,10 @@ public class JobControllerApiHandlerHelperTest {
ZoneId zone = JobType.devUsEast1.zone(tester.controller().system());
tester.jobs().deploy(app.instanceId(), JobType.devUsEast1, Optional.empty(), applicationPackage());
- tester.configServer().setLogStream("1554970337.935104\t17491290-v6-1.ostk.bm2.prod.ne1.yahoo.com\t5480\tcontainer\tstdout\tinfo\tERROR: Bundle canary-application [71] Unable to get module class path. (java.lang.NullPointerException)\n");
+ tester.configServer().setLogStream(() -> "1554970337.935104\t17491290-v6-1.ostk.bm2.prod.ne1.yahoo.com\t5480\tcontainer\tstdout\tinfo\tERROR: Bundle canary-application [71] Unable to get module class path. (java.lang.NullPointerException)\n");
assertResponse(JobControllerApiHandlerHelper.runDetailsResponse(tester.jobs(), tester.jobs().last(app.instanceId(), devUsEast1).get().id(), null), "dev-us-east-1-log-first-part.json");
- tester.configServer().setLogStream("Nope, this won't be logged");
+ tester.configServer().setLogStream(() -> "Nope, this won't be logged");
tester.configServer().convergeServices(app.instanceId(), zone);
tester.runner().run();
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/system-test-job.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/system-test-job.json
index 3234333c092..c0988e8c301 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/system-test-job.json
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/system-test-job.json
@@ -1,69 +1,119 @@
{
- "1": {
- "id": 1,
- "status": "success",
- "start": "(ignore)",
- "end": "(ignore)",
- "wantedPlatform": "6.1",
- "wantedApplication": {
- "hash": "1.0.1-commit1",
- "build": 1,
- "source": {
- "gitRepository": "repository1",
- "gitBranch": "master",
- "gitCommit": "commit1"
+ "runs": [
+ {
+ "id": 2,
+ "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/job/system-test/run/2",
+ "start": "(ignore)",
+ "status": "running",
+ "versions": {
+ "targetPlatform": "6.1.0",
+ "targetApplication": {
+ "build": 4,
+ "compileVersion": "6.1.0",
+ "sourceUrl": "repository1/tree/commit1",
+ "commit": "commit1"
+ }
},
- "sourceUrl": "repository1/tree/commit1",
- "commit": "commit1"
+ "steps": [
+ {
+ "name": "deployTester",
+ "status": "unfinished"
+ },
+ {
+ "name": "installTester",
+ "status": "unfinished"
+ },
+ {
+ "name": "deployReal",
+ "status": "unfinished"
+ },
+ {
+ "name": "installReal",
+ "status": "unfinished"
+ },
+ {
+ "name": "startTests",
+ "status": "unfinished"
+ },
+ {
+ "name": "endTests",
+ "status": "unfinished"
+ },
+ {
+ "name": "copyVespaLogs",
+ "status": "unfinished"
+ },
+ {
+ "name": "deactivateReal",
+ "status": "unfinished"
+ },
+ {
+ "name": "deactivateTester",
+ "status": "unfinished"
+ },
+ {
+ "name": "report",
+ "status": "unfinished"
+ }
+ ]
},
- "steps": {
- "deployTester": "succeeded",
- "installTester": "succeeded",
- "deployReal": "succeeded",
- "installReal": "succeeded",
- "startTests": "succeeded",
- "endTests": "succeeded",
- "copyVespaLogs": "succeeded",
- "deactivateReal": "succeeded",
- "deactivateTester": "succeeded",
- "report": "succeeded"
- },
- "tasks": {
- "deploy": "succeeded",
- "install": "succeeded",
- "test": "succeeded"
- },
- "log": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/job/system-test/run/1"
- },
- "2": {
- "id": 2,
- "status": "running",
- "start": "(ignore)",
- "wantedPlatform": "6.1",
- "wantedApplication": {
- "hash": "1.0.4-commit1",
- "build": 4,
- "source": {
- "gitRepository": "repository1",
- "gitBranch": "master",
- "gitCommit": "commit1"
+ {
+ "id": 1,
+ "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/job/system-test/run/1",
+ "start": "(ignore)",
+ "end": "(ignore)",
+ "status": "success",
+ "versions": {
+ "targetPlatform": "6.1.0",
+ "targetApplication": {
+ "build": 1,
+ "compileVersion": "6.1.0",
+ "sourceUrl": "repository1/tree/commit1",
+ "commit": "commit1"
+ }
},
- "sourceUrl": "repository1/tree/commit1",
- "commit": "commit1"
- },
- "steps": {
- "deployTester": "unfinished",
- "installTester": "unfinished",
- "deployReal": "unfinished",
- "installReal": "unfinished",
- "startTests": "unfinished",
- "endTests": "unfinished",
- "copyVespaLogs": "unfinished",
- "deactivateReal": "unfinished",
- "deactivateTester": "unfinished",
- "report": "unfinished"
- },
- "tasks": {},
- "log": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/job/system-test/run/2"
- }
+ "steps": [
+ {
+ "name": "deployTester",
+ "status": "succeeded"
+ },
+ {
+ "name": "installTester",
+ "status": "succeeded"
+ },
+ {
+ "name": "deployReal",
+ "status": "succeeded"
+ },
+ {
+ "name": "installReal",
+ "status": "succeeded"
+ },
+ {
+ "name": "startTests",
+ "status": "succeeded"
+ },
+ {
+ "name": "endTests",
+ "status": "succeeded"
+ },
+ {
+ "name": "copyVespaLogs",
+ "status": "succeeded"
+ },
+ {
+ "name": "deactivateReal",
+ "status": "succeeded"
+ },
+ {
+ "name": "deactivateTester",
+ "status": "succeeded"
+ },
+ {
+ "name": "report",
+ "status": "succeeded"
+ }
+ ]
+ }
+ ]
}
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json
index e906df94023..668baa50cc1 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json
@@ -43,6 +43,9 @@
"name": "DeploymentMetricsMaintainer"
},
{
+ "name": "DeploymentUpgrader"
+ },
+ {
"name": "EndpointCertificateMaintainer"
},
{
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/BadgeApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/BadgeApiTest.java
index 766a84ca33c..63474ebb7c9 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/BadgeApiTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/BadgeApiTest.java
@@ -53,6 +53,8 @@ public class BadgeApiTest extends ControllerContainerTest {
tester.assertResponse(authenticatedRequest("http://localhost:8080/badge/v1/tenant/application/default"),
Files.readString(Paths.get(responseFiles + "overview.svg")), 200);
+ tester.assertResponse(authenticatedRequest("http://localhost:8080/badge/v1/tenant/application/default/production-us-west-1?historyLength=0"),
+ Files.readString(Paths.get(responseFiles + "single-running.svg")), 200);
tester.assertResponse(authenticatedRequest("http://localhost:8080/badge/v1/tenant/application/default/production-us-west-1?historyLength=32"),
Files.readString(Paths.get(responseFiles + "history.svg")), 200);
@@ -66,6 +68,9 @@ public class BadgeApiTest extends ControllerContainerTest {
tester.serviceRegistry().clock().advance(Duration.ofSeconds(1));
tester.assertResponse(authenticatedRequest("http://localhost:8080/badge/v1/tenant/application/default/production-us-west-1?historyLength=32"),
Files.readString(Paths.get(responseFiles + "history2.svg")), 200);
+
+ tester.assertResponse(authenticatedRequest("http://localhost:8080/badge/v1/tenant/application/default/production-us-west-1?historyLength=0"),
+ Files.readString(Paths.get(responseFiles + "single-done.svg")), 200);
}
}
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/responses/single-done.svg b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/responses/single-done.svg
new file mode 100644
index 00000000000..3bcbed97499
--- /dev/null
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/responses/single-done.svg
@@ -0,0 +1,81 @@
+<svg xmlns='http://www.w3.org/2000/svg' width='294.82606' height='20' role='img' aria-label='Deployment Status'>
+ <title>Deployment Status</title>
+ <linearGradient id='light' x2='0' y2='100%'>
+ <stop offset='0' stop-color='#fff' stop-opacity='.5'/>
+ <stop offset='.1' stop-color='#fff' stop-opacity='.15'/>
+ <stop offset='.9' stop-color='#000' stop-opacity='.15'/>
+ <stop offset='1' stop-color='#000' stop-opacity='.5'/>
+ </linearGradient>
+ <linearGradient id='left-light' x2='100%' y2='0'>
+ <stop offset='0' stop-color='#fff' stop-opacity='.3'/>
+ <stop offset='.5' stop-color='#fff' stop-opacity='.1'/>
+ <stop offset='1' stop-color='#fff' stop-opacity='.0'/>
+ </linearGradient>
+ <linearGradient id='right-shadow' x2='100%' y2='0'>
+ <stop offset='0' stop-color='#000' stop-opacity='.0'/>
+ <stop offset='.5' stop-color='#000' stop-opacity='.1'/>
+ <stop offset='1' stop-color='#000' stop-opacity='.3'/>
+ </linearGradient>
+ <linearGradient id='shadow' x2='100%' y2='0'>
+ <stop offset='0' stop-color='#222' stop-opacity='.3'/>
+ <stop offset='.625' stop-color='#555' stop-opacity='.3'/>
+ <stop offset='.9' stop-color='#555' stop-opacity='.05'/>
+ <stop offset='1' stop-color='#555' stop-opacity='.0'/>
+ </linearGradient>
+ <linearGradient id='shade' x2='100%' y2='0'>
+ <stop offset='0' stop-color='#000' stop-opacity='.20'/>
+ <stop offset='0.05' stop-color='#000' stop-opacity='.10'/>
+ <stop offset='1' stop-color='#000' stop-opacity='.0'/>
+ </linearGradient>
+ <linearGradient id='run-on-failure' x1='40%' x2='80%' y2='0%'>
+ <stop offset='0' stop-color='#ab83ff' />
+ <stop offset='1' stop-color='#bf103c' />
+ <animate attributeName='x1' values='-110%;150%;20%;-110%' dur='6s' repeatCount='indefinite' />
+ <animate attributeName='x2' values='-10%;250%;120%;-10%' dur='6s' repeatCount='indefinite' />
+ </linearGradient>
+ <linearGradient id='run-on-success' x1='40%' x2='80%' y2='0%'>
+ <stop offset='0' stop-color='#ab83ff' />
+ <stop offset='1' stop-color='#00f244' />
+ <animate attributeName='x1' values='-110%;150%;20%;-110%' dur='6s' repeatCount='indefinite' />
+ <animate attributeName='x2' values='-10%;250%;120%;-10%' dur='6s' repeatCount='indefinite' />
+ </linearGradient>
+ <clipPath id='rounded'>
+ <rect width='294.82606' height='20' rx='3' fill='#fff'/>
+ </clipPath>
+ <g clip-path='url(#rounded)'>
+ <rect x='288.82606' rx='3' width='8' height='20' fill='url(#shadow)'/>
+ <rect x='163.18729000000002' rx='3' width='131.63876999999997' height='20' fill='#00f244'/>
+ <rect x='163.18729000000002' rx='3' width='131.63876999999997' height='20' fill='url(#shade)'/>
+ <rect width='169.18729000000002' height='20' fill='#404040'/>
+ <rect x='-6.0' rx='3' width='175.18729000000002' height='20' fill='url(#shade)'/>
+ <rect width='2' height='20' fill='url(#left-light)'/>
+ <rect x='292.82606' width='2' height='20' fill='url(#right-shadow)'/>
+ <rect width='294.82606' height='20' fill='url(#light)'/>
+ </g>
+ <g fill='#fff' text-anchor='middle' font-family='Verdana,Geneva,DejaVu Sans,sans-serif' text-rendering='geometricPrecision' font-size='11'>
+ <svg x='6.5' y='3.0' width='16.0' height='16.0' viewBox='0 0 150 150'>
+ <polygon fill='#402a14' fill-opacity='0.5' points='84.84 10 34.1 44.46 34.1 103.78 84.84 68.02 135.57 103.78 135.57 44.46 84.84 10'/>
+ <polygon fill='#402a14' fill-opacity='0.5' points='84.84 68.02 84.84 10 135.57 44.46 135.57 103.78 84.84 68.02'/>
+ <polygon fill='#061a29' fill-opacity='0.5' points='65.07 81.99 14.34 46.22 14.34 105.54 65.07 140 115.81 105.54 115.81 46.22 65.07 81.99'/>
+ <polygon fill='#061a29' fill-opacity='0.5' points='65.07 81.99 65.07 140 14.34 105.54 14.34 46.22 65.07 81.99'/>
+ </svg>
+ <svg x='6.0' y='2.0' width='16.0' height='16.0' viewBox='0 0 150 150'>
+ <linearGradient id='yellow-shaded' x1='91.17' y1='44.83' x2='136.24' y2='73.4' gradientUnits='userSpaceOnUse'>
+ <stop offset='0.01' stop-color='#c6783e'/>
+ <stop offset='0.54' stop-color='#ff9750'/>
+ </linearGradient>
+ <linearGradient id='blue-shaded' x1='60.71' y1='104.56' x2='-15.54' y2='63' gradientUnits='userSpaceOnUse'>
+ <stop offset='0' stop-color='#005a8e'/>
+ <stop offset='0.54' stop-color='#1a7db6'/>
+ </linearGradient>
+ <polygon fill='#ff9d4b' points='84.84 10 34.1 44.46 34.1 103.78 84.84 68.02 135.57 103.78 135.57 44.46 84.84 10'/>
+ <polygon fill='url(#yellow-shaded)' points='84.84 68.02 84.84 10 135.57 44.46 135.57 103.78 84.84 68.02'/>
+ <polygon fill='#1a7db6' points='65.07 81.99 14.34 46.22 14.34 105.54 65.07 140 115.81 105.54 115.81 46.22 65.07 81.99'/>
+ <polygon fill='url(#blue-shaded)' points='65.07 81.99 65.07 140 14.34 105.54 14.34 46.22 65.07 81.99'/>
+ </svg>
+ <text font-size='11' x='96.09364500000001' y='15' fill='#000' fill-opacity='.4' textLength='135.18729000000002'>tenant.application.default</text>
+ <text font-size='11' x='95.59364500000001' y='14' fill='#fff' textLength='135.18729000000002'>tenant.application.default</text>
+ <text font-size='11' x='232.506675' y='15' fill='#000' fill-opacity='.4' textLength='113.63876999999998'>production-us-west-1</text>
+ <text font-size='11' x='232.006675' y='14' fill='#fff' textLength='113.63876999999998'>production-us-west-1</text>
+ </g>
+</svg>
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/responses/single-running.svg b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/responses/single-running.svg
new file mode 100644
index 00000000000..27e967f8e46
--- /dev/null
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/responses/single-running.svg
@@ -0,0 +1,81 @@
+<svg xmlns='http://www.w3.org/2000/svg' width='294.82606' height='20' role='img' aria-label='Deployment Status'>
+ <title>Deployment Status</title>
+ <linearGradient id='light' x2='0' y2='100%'>
+ <stop offset='0' stop-color='#fff' stop-opacity='.5'/>
+ <stop offset='.1' stop-color='#fff' stop-opacity='.15'/>
+ <stop offset='.9' stop-color='#000' stop-opacity='.15'/>
+ <stop offset='1' stop-color='#000' stop-opacity='.5'/>
+ </linearGradient>
+ <linearGradient id='left-light' x2='100%' y2='0'>
+ <stop offset='0' stop-color='#fff' stop-opacity='.3'/>
+ <stop offset='.5' stop-color='#fff' stop-opacity='.1'/>
+ <stop offset='1' stop-color='#fff' stop-opacity='.0'/>
+ </linearGradient>
+ <linearGradient id='right-shadow' x2='100%' y2='0'>
+ <stop offset='0' stop-color='#000' stop-opacity='.0'/>
+ <stop offset='.5' stop-color='#000' stop-opacity='.1'/>
+ <stop offset='1' stop-color='#000' stop-opacity='.3'/>
+ </linearGradient>
+ <linearGradient id='shadow' x2='100%' y2='0'>
+ <stop offset='0' stop-color='#222' stop-opacity='.3'/>
+ <stop offset='.625' stop-color='#555' stop-opacity='.3'/>
+ <stop offset='.9' stop-color='#555' stop-opacity='.05'/>
+ <stop offset='1' stop-color='#555' stop-opacity='.0'/>
+ </linearGradient>
+ <linearGradient id='shade' x2='100%' y2='0'>
+ <stop offset='0' stop-color='#000' stop-opacity='.20'/>
+ <stop offset='0.05' stop-color='#000' stop-opacity='.10'/>
+ <stop offset='1' stop-color='#000' stop-opacity='.0'/>
+ </linearGradient>
+ <linearGradient id='run-on-failure' x1='40%' x2='80%' y2='0%'>
+ <stop offset='0' stop-color='#ab83ff' />
+ <stop offset='1' stop-color='#bf103c' />
+ <animate attributeName='x1' values='-110%;150%;20%;-110%' dur='6s' repeatCount='indefinite' />
+ <animate attributeName='x2' values='-10%;250%;120%;-10%' dur='6s' repeatCount='indefinite' />
+ </linearGradient>
+ <linearGradient id='run-on-success' x1='40%' x2='80%' y2='0%'>
+ <stop offset='0' stop-color='#ab83ff' />
+ <stop offset='1' stop-color='#00f244' />
+ <animate attributeName='x1' values='-110%;150%;20%;-110%' dur='6s' repeatCount='indefinite' />
+ <animate attributeName='x2' values='-10%;250%;120%;-10%' dur='6s' repeatCount='indefinite' />
+ </linearGradient>
+ <clipPath id='rounded'>
+ <rect width='294.82606' height='20' rx='3' fill='#fff'/>
+ </clipPath>
+ <g clip-path='url(#rounded)'>
+ <rect x='288.82606' rx='3' width='8' height='20' fill='url(#shadow)'/>
+ <rect x='163.18729000000002' rx='3' width='131.63876999999997' height='20' fill='url(#run-on-failure)'/>
+ <rect x='163.18729000000002' rx='3' width='131.63876999999997' height='20' fill='url(#shade)'/>
+ <rect width='169.18729000000002' height='20' fill='#404040'/>
+ <rect x='-6.0' rx='3' width='175.18729000000002' height='20' fill='url(#shade)'/>
+ <rect width='2' height='20' fill='url(#left-light)'/>
+ <rect x='292.82606' width='2' height='20' fill='url(#right-shadow)'/>
+ <rect width='294.82606' height='20' fill='url(#light)'/>
+ </g>
+ <g fill='#fff' text-anchor='middle' font-family='Verdana,Geneva,DejaVu Sans,sans-serif' text-rendering='geometricPrecision' font-size='11'>
+ <svg x='6.5' y='3.0' width='16.0' height='16.0' viewBox='0 0 150 150'>
+ <polygon fill='#402a14' fill-opacity='0.5' points='84.84 10 34.1 44.46 34.1 103.78 84.84 68.02 135.57 103.78 135.57 44.46 84.84 10'/>
+ <polygon fill='#402a14' fill-opacity='0.5' points='84.84 68.02 84.84 10 135.57 44.46 135.57 103.78 84.84 68.02'/>
+ <polygon fill='#061a29' fill-opacity='0.5' points='65.07 81.99 14.34 46.22 14.34 105.54 65.07 140 115.81 105.54 115.81 46.22 65.07 81.99'/>
+ <polygon fill='#061a29' fill-opacity='0.5' points='65.07 81.99 65.07 140 14.34 105.54 14.34 46.22 65.07 81.99'/>
+ </svg>
+ <svg x='6.0' y='2.0' width='16.0' height='16.0' viewBox='0 0 150 150'>
+ <linearGradient id='yellow-shaded' x1='91.17' y1='44.83' x2='136.24' y2='73.4' gradientUnits='userSpaceOnUse'>
+ <stop offset='0.01' stop-color='#c6783e'/>
+ <stop offset='0.54' stop-color='#ff9750'/>
+ </linearGradient>
+ <linearGradient id='blue-shaded' x1='60.71' y1='104.56' x2='-15.54' y2='63' gradientUnits='userSpaceOnUse'>
+ <stop offset='0' stop-color='#005a8e'/>
+ <stop offset='0.54' stop-color='#1a7db6'/>
+ </linearGradient>
+ <polygon fill='#ff9d4b' points='84.84 10 34.1 44.46 34.1 103.78 84.84 68.02 135.57 103.78 135.57 44.46 84.84 10'/>
+ <polygon fill='url(#yellow-shaded)' points='84.84 68.02 84.84 10 135.57 44.46 135.57 103.78 84.84 68.02'/>
+ <polygon fill='#1a7db6' points='65.07 81.99 14.34 46.22 14.34 105.54 65.07 140 115.81 105.54 115.81 46.22 65.07 81.99'/>
+ <polygon fill='url(#blue-shaded)' points='65.07 81.99 65.07 140 14.34 105.54 14.34 46.22 65.07 81.99'/>
+ </svg>
+ <text font-size='11' x='96.09364500000001' y='15' fill='#000' fill-opacity='.4' textLength='135.18729000000002'>tenant.application.default</text>
+ <text font-size='11' x='95.59364500000001' y='14' fill='#fff' textLength='135.18729000000002'>tenant.application.default</text>
+ <text font-size='11' x='232.506675' y='15' fill='#000' fill-opacity='.4' textLength='113.63876999999998'>production-us-west-1</text>
+ <text font-size='11' x='232.006675' y='14' fill='#fff' textLength='113.63876999999998'>production-us-west-1</text>
+ </g>
+</svg>
diff --git a/default_build_settings.cmake b/default_build_settings.cmake
index f034c9260e4..152a0d3eef3 100644
--- a/default_build_settings.cmake
+++ b/default_build_settings.cmake
@@ -31,7 +31,11 @@ endfunction()
function(setup_vespa_default_build_settings_centos_8)
message("-- Setting up default build settings for centos 8")
set(DEFAULT_EXTRA_INCLUDE_DIRECTORY "${VESPA_DEPS}/include" "/usr/include/openblas" PARENT_SCOPE)
- set(DEFAULT_VESPA_LLVM_VERSION "11" PARENT_SCOPE)
+ if (VESPA_OS_DISTRO_NAME STREQUAL "CentOS Stream")
+ set(DEFAULT_VESPA_LLVM_VERSION "12" PARENT_SCOPE)
+ else()
+ set(DEFAULT_VESPA_LLVM_VERSION "11" PARENT_SCOPE)
+ endif()
endfunction()
function(setup_vespa_default_build_settings_rocky_8_4)
diff --git a/dist/vespa.spec b/dist/vespa.spec
index 7bee7e2a8d0..d8e3653af39 100644
--- a/dist/vespa.spec
+++ b/dist/vespa.spec
@@ -102,7 +102,12 @@ BuildRequires: cmake >= 3.11.4-3
BuildRequires: libarchive
%endif
%define _command_cmake cmake
+%global _centos_stream %(grep -qs '^NAME="CentOS Stream"' /etc/os-release && echo 1 || echo 0)
+%if 0%{?_centos_stream}
+BuildRequires: (llvm-devel >= 12.0.0 and llvm-devel < 13)
+%else
BuildRequires: (llvm-devel >= 11.0.0 and llvm-devel < 12)
+%endif
%else
BuildRequires: (llvm-devel >= 10.0.1 and llvm-devel < 11)
%endif
@@ -226,7 +231,11 @@ Requires: vespa-valgrind >= 3.17.0-1
%endif
%if 0%{?el8}
%if 0%{?centos} || 0%{?rocky}
+%if 0%{?_centos_stream}
+%define _vespa_llvm_version 12
+%else
%define _vespa_llvm_version 11
+%endif
%else
%define _vespa_llvm_version 10
%endif
@@ -358,7 +367,11 @@ Requires: openssl-libs
%endif
%if 0%{?el8}
%if 0%{?centos} || 0%{?rocky}
+%if 0%{?_centos_stream}
+Requires: (llvm-libs >= 12.0.0 and llvm-libs < 13)
+%else
Requires: (llvm-libs >= 11.0.0 and llvm-libs < 12)
+%endif
%else
Requires: (llvm-libs >= 10.0.1 and llvm-libs < 11)
%endif
@@ -781,7 +794,6 @@ fi
%{_prefix}/lib/jars/config-provisioning-jar-with-dependencies.jar
%{_prefix}/lib/jars/container-apache-http-client-bundle-jar-with-dependencies.jar
%{_prefix}/lib/jars/container-disc-jar-with-dependencies.jar
-%{_prefix}/lib/jars/container-jersey2-jar-with-dependencies.jar
%{_prefix}/lib/jars/container-search-and-docproc-jar-with-dependencies.jar
%{_prefix}/lib/jars/container-search-gui-jar-with-dependencies.jar
%{_prefix}/lib/jars/defaults-jar-with-dependencies.jar
diff --git a/docproc/src/test/java/com/yahoo/docproc/jdisc/DocumentProcessingHandlerAllMessageTypesTestCase.java b/docproc/src/test/java/com/yahoo/docproc/jdisc/DocumentProcessingHandlerAllMessageTypesTestCase.java
index 268a40f04c9..c508e7da61d 100644
--- a/docproc/src/test/java/com/yahoo/docproc/jdisc/DocumentProcessingHandlerAllMessageTypesTestCase.java
+++ b/docproc/src/test/java/com/yahoo/docproc/jdisc/DocumentProcessingHandlerAllMessageTypesTestCase.java
@@ -36,7 +36,7 @@ import static org.junit.Assert.assertTrue;
import static org.junit.Assert.assertNotNull;
/**
- * @author Einar M R Rosenvinge
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
*/
public class DocumentProcessingHandlerAllMessageTypesTestCase extends DocumentProcessingHandlerTestBase {
diff --git a/docproc/src/test/java/com/yahoo/docproc/jdisc/DocumentProcessingHandlerTestBase.java b/docproc/src/test/java/com/yahoo/docproc/jdisc/DocumentProcessingHandlerTestBase.java
index c437261224d..05009f484c8 100644
--- a/docproc/src/test/java/com/yahoo/docproc/jdisc/DocumentProcessingHandlerTestBase.java
+++ b/docproc/src/test/java/com/yahoo/docproc/jdisc/DocumentProcessingHandlerTestBase.java
@@ -5,7 +5,6 @@ import com.yahoo.collections.Pair;
import com.yahoo.component.ComponentId;
import com.yahoo.component.provider.ComponentRegistry;
import com.yahoo.container.core.document.ContainerDocumentConfig;
-import com.yahoo.container.jdisc.ContainerMbusConfig;
import com.yahoo.container.jdisc.messagebus.MbusServerProvider;
import com.yahoo.container.jdisc.messagebus.SessionCache;
import com.yahoo.docproc.CallStack;
@@ -14,15 +13,12 @@ import com.yahoo.docproc.jdisc.messagebus.MbusRequestContext;
import com.yahoo.document.DocumentType;
import com.yahoo.document.DocumentTypeManager;
-import com.yahoo.document.config.DocumentmanagerConfig;
import com.yahoo.documentapi.messagebus.loadtypes.LoadType;
import com.yahoo.documentapi.messagebus.protocol.DocumentMessage;
import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol;
-import com.yahoo.documentapi.messagebus.protocol.DocumentProtocolPoliciesConfig;
import com.yahoo.jdisc.AbstractResource;
import com.yahoo.jdisc.ReferencedResource;
import com.yahoo.jdisc.application.ContainerBuilder;
-import com.yahoo.messagebus.MessagebusConfig;
import com.yahoo.messagebus.Protocol;
import com.yahoo.messagebus.SourceSessionParams;
import com.yahoo.messagebus.jdisc.MbusClient;
@@ -30,8 +26,6 @@ import com.yahoo.messagebus.jdisc.test.RemoteServer;
import com.yahoo.messagebus.jdisc.test.ServerTestDriver;
import com.yahoo.messagebus.routing.Route;
import com.yahoo.messagebus.shared.SharedSourceSession;
-import com.yahoo.vespa.config.content.DistributionConfig;
-import com.yahoo.vespa.config.content.LoadTypeConfig;
import org.junit.After;
import org.junit.Before;
@@ -58,11 +52,8 @@ public abstract class DocumentProcessingHandlerTestBase {
driver = ServerTestDriver.newInactiveInstanceWithProtocol(protocol, true);
- sessionCache = new SessionCache(new ContainerMbusConfig.Builder().build(),
- driver.client().slobroksConfig(),
- new MessagebusConfig.Builder().build(),
- "test",
- protocol);
+ sessionCache =
+ new SessionCache("raw:", driver.client().slobrokId(), "test", "raw:", null, "raw:", documentTypeManager);
ContainerBuilder builder = driver.parent().newContainerBuilder();
ComponentRegistry<DocprocService> registry = new ComponentRegistry<>();
@@ -111,7 +102,7 @@ public abstract class DocumentProcessingHandlerTestBase {
resource.release();
}
- remoteServer = RemoteServer.newInstance("foobar", protocol);
+ remoteServer = RemoteServer.newInstance(driver.client().slobrokId(), "foobar", protocol);
}
@After
diff --git a/filedistribution/pom.xml b/filedistribution/pom.xml
index 8bfa85d8b16..e922d878dd7 100644
--- a/filedistribution/pom.xml
+++ b/filedistribution/pom.xml
@@ -20,6 +20,12 @@
<dependencies>
<dependency>
<groupId>com.yahoo.vespa</groupId>
+ <artifactId>container-apache-http-client-bundle</artifactId>
+ <version>${project.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>com.yahoo.vespa</groupId>
<artifactId>vespajlib</artifactId>
<version>${project.version}</version>
</dependency>
@@ -58,10 +64,6 @@
<artifactId>airline</artifactId>
</dependency>
<dependency>
- <groupId>org.apache.httpcomponents</groupId>
- <artifactId>httpclient</artifactId>
- </dependency>
- <dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
</dependency>
diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java
index 9f78056d15b..a3e6fff410e 100644
--- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java
+++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java
@@ -137,12 +137,6 @@ public class Flags {
"Takes effect on the next suspension request to the Orchestrator.",
APPLICATION_ID);
- public static final UnboundBooleanFlag ENCRYPT_DISK = defineFeatureFlag(
- "encrypt-disk", false,
- List.of("hakonhall"), "2021-05-05", "2021-08-05",
- "Allow migrating an unencrypted data partition to being encrypted.",
- "Takes effect on next host-admin tick.");
-
public static final UnboundBooleanFlag ENCRYPT_DIRTY_DISK = defineFeatureFlag(
"encrypt-dirty-disk", false,
List.of("hakonhall"), "2021-05-14", "2021-08-05",
@@ -292,6 +286,12 @@ public class Flags {
"List of applications where encryption of their host should be deferred",
"Takes effect on next run of HostEncrypter");
+ public static final UnboundBooleanFlag PODMAN3 = defineFeatureFlag(
+ "podman3", false,
+ List.of("mpolden"), "2021-07-05", "2021-09-01",
+ "Whether to use Podman 3 on supported hosts",
+ "Takes effect on host-admin restart");
+
/** WARNING: public for testing: All flags should be defined in {@link Flags}. */
public static UnboundBooleanFlag defineFeatureFlag(String flagId, boolean defaultValue, List<String> owners,
String createdAt, String expiresAt, String description,
diff --git a/http-utils/pom.xml b/http-utils/pom.xml
index 1f85658430f..2a8ec1b9bb9 100644
--- a/http-utils/pom.xml
+++ b/http-utils/pom.xml
@@ -31,22 +31,21 @@
<artifactId>slf4j-api</artifactId>
<scope>provided</scope>
</dependency>
-
- <!-- compile scope -->
+ <!-- Apache client artifacts are provided by the jdisc container and are therefore scoped as such -->
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpclient</artifactId>
- <scope>compile</scope>
+ <scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpcore</artifactId>
- <scope>compile</scope>
+ <scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.httpcomponents.client5</groupId>
<artifactId>httpclient5</artifactId>
- <scope>compile</scope>
+ <scope>provided</scope>
<exclusions>
<exclusion>
<groupId>org.slf4j</groupId>
diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/NormalizeExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/NormalizeExpression.java
index 896799d4492..9c6e698a154 100644
--- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/NormalizeExpression.java
+++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/NormalizeExpression.java
@@ -45,14 +45,10 @@ public final class NormalizeExpression extends Expression {
}
@Override
- public boolean equals(Object obj) {
- if (!(obj instanceof NormalizeExpression)) {
- return false;
- }
- NormalizeExpression rhs = (NormalizeExpression)obj;
- if (linguistics != rhs.linguistics) {
- return false;
- }
+ public boolean equals(Object o) {
+ if (!(o instanceof NormalizeExpression)) return false;
+ NormalizeExpression other = (NormalizeExpression)o;
+ if (linguistics != other.linguistics) return false;
return true;
}
@@ -60,4 +56,5 @@ public final class NormalizeExpression extends Expression {
public int hashCode() {
return getClass().hashCode();
}
+
}
diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/parser/ExpressionTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/parser/ExpressionTestCase.java
index e9a0d0253e1..223c8191186 100644
--- a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/parser/ExpressionTestCase.java
+++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/parser/ExpressionTestCase.java
@@ -90,4 +90,5 @@ public class ExpressionTestCase {
assertEquals(foo.hashCode(), bar.hashCode());
assertEquals(foo, bar);
}
+
}
diff --git a/jdisc-cloud-aws/pom.xml b/jdisc-cloud-aws/pom.xml
index 9f4572736c8..0c89872aa46 100644
--- a/jdisc-cloud-aws/pom.xml
+++ b/jdisc-cloud-aws/pom.xml
@@ -24,8 +24,20 @@
<scope>provided</scope>
</dependency>
<dependency>
+ <groupId>com.yahoo.vespa</groupId>
+ <artifactId>container-apache-http-client-bundle</artifactId>
+ <version>${project.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
<groupId>com.amazonaws</groupId>
<artifactId>aws-java-sdk-core</artifactId>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpclient</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
<dependency>
<groupId>com.amazonaws</groupId>
diff --git a/messagebus/src/tests/targetpool/targetpool.cpp b/messagebus/src/tests/targetpool/targetpool.cpp
index 9259f992d6c..9ff352abcf5 100644
--- a/messagebus/src/tests/targetpool/targetpool.cpp
+++ b/messagebus/src/tests/targetpool/targetpool.cpp
@@ -37,7 +37,7 @@ TEST("targetpool_test") {
FRT_Supervisor & orb = server.supervisor();
std::unique_ptr<PoolTimer> ptr(new PoolTimer());
PoolTimer &timer = *ptr;
- RPCTargetPool pool(std::move(ptr), 0.666);
+ RPCTargetPool pool(std::move(ptr), 0.666, 1 );
// Assert that all connections expire.
RPCTarget::SP target;
diff --git a/messagebus/src/vespa/messagebus/network/rpcnetwork.cpp b/messagebus/src/vespa/messagebus/network/rpcnetwork.cpp
index fbe00275626..41298c28a88 100644
--- a/messagebus/src/vespa/messagebus/network/rpcnetwork.cpp
+++ b/messagebus/src/vespa/messagebus/network/rpcnetwork.cpp
@@ -133,7 +133,7 @@ RPCNetwork::RPCNetwork(const RPCNetworkParams &params) :
_mirror(std::make_unique<slobrok::api::MirrorAPI>(*_orb, *_slobrokCfgFactory)),
_regAPI(std::make_unique<slobrok::api::RegisterAPI>(*_orb, *_slobrokCfgFactory)),
_requestedPort(params.getListenPort()),
- _targetPool(std::make_unique<RPCTargetPool>(params.getConnectionExpireSecs())),
+ _targetPool(std::make_unique<RPCTargetPool>(params.getConnectionExpireSecs(), params.getNumRpcTargets())),
_targetPoolTask(std::make_unique<TargetPoolTask>(_scheduler, *_targetPool)),
_servicePool(std::make_unique<RPCServicePool>(*_mirror, 4_Ki)),
_executor(std::make_unique<vespalib::ThreadStackExecutor>(params.getNumThreads(), 64_Ki)),
diff --git a/messagebus/src/vespa/messagebus/network/rpctargetpool.cpp b/messagebus/src/vespa/messagebus/network/rpctargetpool.cpp
index b42ac47e54d..cfb1e120a2c 100644
--- a/messagebus/src/vespa/messagebus/network/rpctargetpool.cpp
+++ b/messagebus/src/vespa/messagebus/network/rpctargetpool.cpp
@@ -4,23 +4,45 @@
namespace mbus {
-RPCTargetPool::Entry::Entry(RPCTarget::SP target, uint64_t lastUse) :
- _target(target),
- _lastUse(lastUse)
+RPCTargetPool::Entry::Entry(std::vector<RPCTarget::SP> targets, uint64_t lastUse)
+ : _targets(std::move(targets)),
+ _lastUse(lastUse),
+ _next(0)
{ }
-RPCTargetPool::RPCTargetPool(double expireSecs) :
- _lock(),
- _targets(),
- _timer(new SteadyTimer()),
- _expireMillis(static_cast<uint64_t>(expireSecs * 1000))
+RPCTarget::SP
+RPCTargetPool::Entry::getTarget(const LockGuard &, uint64_t now) {
+ if (_next >= _targets.size()) {
+ _next = 0;
+ }
+ RPCTarget::SP target = _targets[_next++];
+ if ( ! target->isValid()) {
+ return RPCTarget::SP();
+ }
+ _lastUse = now;
+ return target;
+}
+
+bool
+RPCTargetPool::Entry::inUse(const LockGuard &) const {
+ for (const auto & target : _targets) {
+ if (target.use_count() > 1) {
+ return true;
+ }
+ }
+ return false;
+}
+
+RPCTargetPool::RPCTargetPool(double expireSecs, size_t numTargetsPerSpec)
+ : RPCTargetPool(std::make_unique<SteadyTimer>(), expireSecs, numTargetsPerSpec)
{ }
-RPCTargetPool::RPCTargetPool(ITimer::UP timer, double expireSecs) :
+RPCTargetPool::RPCTargetPool(ITimer::UP timer, double expireSecs, size_t numTargetsPerSpec) :
_lock(),
_targets(),
_timer(std::move(timer)),
- _expireMillis(static_cast<uint64_t>(expireSecs * 1000))
+ _expireMillis(static_cast<uint64_t>(expireSecs * 1000)),
+ _numTargetsPerSpec(numTargetsPerSpec)
{ }
RPCTargetPool::~RPCTargetPool()
@@ -35,21 +57,12 @@ RPCTargetPool::flushTargets(bool force)
LockGuard guard(_lock);
TargetMap::iterator it = _targets.begin();
while (it != _targets.end()) {
- Entry &entry = it->second;
- if (entry._target.get() != nullptr) {
- if (entry._target.use_count() > 1) {
- entry._lastUse = currentTime;
- ++it;
- continue; // someone is using this
- }
- if (!force) {
- if (entry._lastUse + _expireMillis > currentTime) {
- ++it;
- continue; // not sufficiently idle
- }
- }
+ const Entry &entry = it->second;
+ if ( ! entry.inUse(guard) && (force || ((entry.lastUse() + _expireMillis) < currentTime))) {
+ _targets.erase(it++); // postfix increment to move the iterator
+ } else {
+ ++it;
}
- _targets.erase(it++); // postfix increment to move the iterator
}
}
@@ -68,16 +81,19 @@ RPCTargetPool::getTarget(FRT_Supervisor &orb, const RPCServiceAddress &address)
LockGuard guard(_lock);
auto it = _targets.find(spec);
if (it != _targets.end()) {
- Entry &entry = it->second;
- if (entry._target->isValid()) {
- entry._lastUse = currentTime;
- return entry._target;
+ RPCTarget::SP target = it->second.getTarget(guard, currentTime);
+ if (target) {
+ return target;
}
_targets.erase(it);
}
- auto ret = std::make_shared<RPCTarget>(spec, orb);
- _targets.insert(TargetMap::value_type(spec, Entry(ret, currentTime)));
- return ret;
+ std::vector<RPCTarget::SP> targets;
+ targets.reserve(_numTargetsPerSpec);
+ for (size_t i(0); i < _numTargetsPerSpec; i++) {
+ targets.push_back(std::make_shared<RPCTarget>(spec, orb));
+ }
+ _targets.insert(TargetMap::value_type(spec, Entry(std::move(targets), currentTime)));
+ return _targets.find(spec)->second.getTarget(guard, currentTime);
}
} // namespace mbus
diff --git a/messagebus/src/vespa/messagebus/network/rpctargetpool.h b/messagebus/src/vespa/messagebus/network/rpctargetpool.h
index bc9e1a4b19f..2a6fb75d511 100644
--- a/messagebus/src/vespa/messagebus/network/rpctargetpool.h
+++ b/messagebus/src/vespa/messagebus/network/rpctargetpool.h
@@ -16,24 +16,30 @@ namespace mbus {
*/
class RPCTargetPool {
private:
+ using LockGuard = std::lock_guard<std::mutex>;
/**
* Implements a helper class holds the necessary reference and token counter
* for a JRT target to keep connections open as long as they get used from
* time to time.
*/
- struct Entry {
- RPCTarget::SP _target;
- uint64_t _lastUse;
-
- Entry(RPCTarget::SP target, uint64_t lastUse);
+ class Entry {
+ public:
+ Entry(std::vector<RPCTarget::SP> targets, uint64_t lastUse);
+ RPCTarget::SP getTarget(const LockGuard & guard, uint64_t now);
+ uint64_t lastUse() const { return _lastUse; }
+ bool inUse(const LockGuard & guard) const;
+ private:
+ std::vector<RPCTarget::SP> _targets;
+ uint64_t _lastUse;
+ size_t _next;
};
using TargetMap = std::map<string, Entry>;
- using LockGuard = std::lock_guard<std::mutex>;
std::mutex _lock;
TargetMap _targets;
ITimer::UP _timer;
uint64_t _expireMillis;
+ size_t _numTargetsPerSpec;
public:
RPCTargetPool(const RPCTargetPool &) = delete;
@@ -46,7 +52,7 @@ public:
* @param expireSecs The number of seconds until an idle connection is
* closed.
*/
- RPCTargetPool(double expireSecs);
+ RPCTargetPool(double expireSecs, size_t numTargetsPerSpec);
/**
* Constructs a new instance of this class, using the given {@link Timer}
@@ -57,7 +63,7 @@ public:
* @param expireSecs The number of seconds until an idle connection is
* closed.
*/
- RPCTargetPool(ITimer::UP timer, double expireSecs);
+ RPCTargetPool(ITimer::UP timer, double expireSecs, size_t numTargetsPerSpec);
/**
* Destructor. Frees any allocated resources.
diff --git a/metrics-proxy/pom.xml b/metrics-proxy/pom.xml
index a7579aeb2ea..19b545df616 100644
--- a/metrics-proxy/pom.xml
+++ b/metrics-proxy/pom.xml
@@ -60,6 +60,12 @@
</dependency>
<dependency>
<groupId>com.yahoo.vespa</groupId>
+ <artifactId>container-apache-http-client-bundle</artifactId>
+ <version>${project.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>com.yahoo.vespa</groupId>
<artifactId>container-core</artifactId>
<version>${project.version}</version>
<scope>provided</scope>
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/ContainerEngine.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/ContainerEngine.java
index 58ff1af4681..75069fa815e 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/ContainerEngine.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/ContainerEngine.java
@@ -25,7 +25,7 @@ public interface ContainerEngine {
/** Start a created container */
void startContainer(NodeAgentContext context);
- /** Update an existing container */
+ /** Update an existing container with new resources */
void updateContainer(NodeAgentContext context, ContainerId containerId, ContainerResources containerResources);
/** Remove given container. The container will be stopped if necessary */
@@ -34,25 +34,25 @@ public interface ContainerEngine {
/** Get container for given context */
Optional<Container> getContainer(NodeAgentContext context);
- /** List containers managed by this */
+ /** Returns all containers known by this */
List<PartialContainer> listContainers(TaskContext context);
/** Returns the network interface used by container in given context */
String networkInterface(NodeAgentContext context);
- /** Executes a command in inside container as root user, throws on non-zero exit code */
+ /** Execute command inside container as root. Ignores non-zero exit code */
CommandResult executeAsRoot(NodeAgentContext context, Duration timeout, String... command);
- /** Executes a command in inside containers network namespace, throws on non-zero exit code */
+ /** Execute command inside the container's network namespace. Throws on non-zero exit code */
CommandResult executeInNetworkNamespace(NodeAgentContext context, String... command);
- /** Download giving image */
+ /** Download given image */
void pullImage(TaskContext context, DockerImage image, RegistryCredentials registryCredentials);
/** Returns whether given image is already downloaded */
boolean hasImage(TaskContext context, DockerImage image);
- /** Remove given image */
+ /** Remove image by id */
void removeImage(TaskContext context, String id);
/** Returns images available in this */
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/ContainerOperations.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/ContainerOperations.java
index 444e92e793f..49216ddfdaf 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/ContainerOperations.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/ContainerOperations.java
@@ -70,17 +70,16 @@ public class ContainerOperations {
return executeCommandInContainerAsRoot(context, CommandLine.DEFAULT_TIMEOUT.toSeconds(), command);
}
- /** Executes a command inside container identified by given context. Does NOT throw on non-zero exit code */
+ /** Execute command inside container identified by given context. Does NOT throw on non-zero exit code */
public CommandResult executeCommandInContainerAsRoot(NodeAgentContext context, Long timeoutSeconds, String... command) {
return containerEngine.executeAsRoot(context, Duration.ofSeconds(timeoutSeconds), command);
}
- /** Executes a command in inside containers network namespace, throws on non-zero exit code */
+ /** Execute command in inside containers network namespace, identified by given context. Throws on non-zero exit code */
public CommandResult executeCommandInNetworkNamespace(NodeAgentContext context, String... command) {
return containerEngine.executeInNetworkNamespace(context, command);
}
-
/** Resume node. Resuming a node means that it is ready to receive traffic */
public String resumeNode(NodeAgentContext context) {
return executeNodeCtlInContainer(context, "resume");
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/PartialContainer.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/PartialContainer.java
index c3be670c009..f960201b2a9 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/PartialContainer.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/PartialContainer.java
@@ -104,8 +104,8 @@ public class PartialContainer {
stopped,
paused,
exited,
- removing;
-
+ removing,
+ stopping;
public boolean isRunning() {
return this == running;
@@ -121,6 +121,7 @@ public class PartialContainer {
case "paused": return paused;
case "exited": return exited;
case "removing": return removing;
+ case "stopping": return stopping;
}
throw new IllegalArgumentException("Invalid state '" + state + "'");
}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/image/ContainerImagePruner.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/image/ContainerImagePruner.java
index bb33dfee663..37601055d00 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/image/ContainerImagePruner.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/image/ContainerImagePruner.java
@@ -40,11 +40,10 @@ import java.util.stream.Stream;
* <li>Deleting a tag of an image with multiple tags will only remove the tag, the image with the
* remaining tags will remain</li>
* <li>Deleting the last tag of an image will delete the entire image.</li>
- * <li>Image cannot be deleted if:</li>
- * <ol>
- * <li>It has 1 or more children</li>
- * <li>A container uses it</li>
- * </ol>
+ * <li>Image cannot be deleted if:
+ * <p>- It has 1 or more children
+ * <p>- A container uses it
+ * </li>
* </ol>
*
* @author freva
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumCommand.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumCommand.java
index 2a01a5ebcb4..ba9ba80ccb7 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumCommand.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumCommand.java
@@ -1,7 +1,6 @@
// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.hosted.node.admin.task.util.yum;
-import com.yahoo.component.Version;
import com.yahoo.vespa.hosted.node.admin.component.TaskContext;
import com.yahoo.vespa.hosted.node.admin.task.util.process.CommandLine;
import com.yahoo.vespa.hosted.node.admin.task.util.process.CommandResult;
@@ -57,16 +56,6 @@ public abstract class YumCommand<T extends YumCommand<T>> {
public abstract boolean converge(TaskContext context);
- /** Returns the version of Yum itself */
- protected final Version version(TaskContext context) {
- return terminal.newCommandLine(context).add("yum", "--version")
- .executeSilently()
- .getOutputLinesStream()
- .findFirst()
- .map(Version::fromString).orElseThrow(() -> new IllegalStateException("Failed to detect Yum version"));
- }
-
-
public static class GenericYumCommand extends YumCommand<GenericYumCommand> {
private static final Pattern UNKNOWN_PACKAGE_PATTERN = Pattern.compile("(?dm)^No package ([^ ]+) available\\.$");
@@ -111,11 +100,10 @@ public abstract class YumCommand<T extends YumCommand<T>> {
if (yumCommand == CommandType.remove)
if (packages.stream().noneMatch(pkg -> isInstalled(context, pkg))) return false;
- Version yumVersion = version(context);
CommandLine commandLine = terminal.newCommandLine(context);
commandLine.add("yum", yumCommand.name());
addParametersToCommandLine(commandLine);
- commandLine.add(packages.stream().map(pkg -> pkg.toName(yumVersion)).collect(Collectors.toList()));
+ commandLine.add(packages.stream().map(pkg -> pkg.toName()).collect(Collectors.toList()));
// There's no way to figure out whether a yum command would have been a no-op.
// Therefore, run the command and parse the output to decide.
@@ -168,15 +156,10 @@ public abstract class YumCommand<T extends YumCommand<T>> {
@Override
public boolean converge(TaskContext context) {
- Version yumVersion = version(context);
- String targetVersionLockName = yumPackage.toVersionLockName(yumVersion);
+ String targetVersionLockName = yumPackage.toVersionLockName();
List<String> command = new ArrayList<>(4);
command.add("yum");
- // Using --quiet on Yum 4 always results in an empty list, even if locks exist...
- if (yumVersion.getMajor() < 4) {
- command.add("--quiet");
- }
command.add("versionlock");
command.add("list");
@@ -193,7 +176,7 @@ public abstract class YumCommand<T extends YumCommand<T>> {
if (packageName.getName().equals(yumPackage.getName())) {
// If existing lock doesn't exactly match the full package name,
// it means it's locked to another version and we must remove that lock.
- String versionLockName = packageName.toVersionLockName(yumVersion);
+ String versionLockName = packageName.toVersionLockName();
if (versionLockName.equals(targetVersionLockName)) {
return true;
} else {
@@ -231,7 +214,7 @@ public abstract class YumCommand<T extends YumCommand<T>> {
var installCommand = terminal.newCommandLine(context).add("yum", "install");
addParametersToCommandLine(installCommand);
- installCommand.add(yumPackage.toName(yumVersion));
+ installCommand.add(yumPackage.toName());
String output = installCommand.executeSilently().getUntrimmedOutput();
@@ -240,7 +223,7 @@ public abstract class YumCommand<T extends YumCommand<T>> {
// case 3.
var upgradeCommand = terminal.newCommandLine(context).add("yum", "downgrade");
addParametersToCommandLine(upgradeCommand);
- upgradeCommand.add(yumPackage.toName(yumVersion)).execute();
+ upgradeCommand.add(yumPackage.toName()).execute();
modified = true;
} else {
// case 2.
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumPackageName.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumPackageName.java
index abe5dc1cbb3..5b32f50ad49 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumPackageName.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumPackageName.java
@@ -2,7 +2,6 @@
package com.yahoo.vespa.hosted.node.admin.task.util.yum;
import com.google.common.base.Strings;
-import com.yahoo.component.Version;
import java.util.Arrays;
import java.util.Objects;
@@ -12,7 +11,7 @@ import java.util.regex.Pattern;
import java.util.stream.Collectors;
/**
- * YUM package name.
+ * A YUM/DNF package name.
*
* <p>From yum(8): YUM package names are used with install, update, remove, list, info etc
* with any of the following as well as globs of any of the following, with any of the
@@ -42,9 +41,10 @@ public class YumPackageName {
Arrays.stream(Architecture.values()).map(Architecture::name).collect(Collectors.joining("|"));
private static final Pattern ARCHITECTURE_PATTERN = Pattern.compile("\\.(" + ARCHITECTURES_OR + "|\\*)$");
private static final Pattern EPOCH_PATTERN = Pattern.compile("^((.+)-)?([0-9]+)$");
- private static final Pattern NAME_VER_REL_PATTERN = Pattern.compile("^((.+)-)?" +
- "([+()a-z0-9._]*[0-9][a-z0-9._]*)-" + // ver contains at least one digit
- "([+()a-z0-9._]*[0-9][a-z0-9._]*)$"); // rel contains at least one digit
+ private static final Pattern NAME_VER_REL_PATTERN =
+ Pattern.compile("^(.+?)?" + // name
+ "-([+()a-z0-9._]*[0-9]+\\.[a-z0-9._]*)" + // ver: contains at least one digit and dot
+ "(?:-([+()a-z0-9._]*[0-9][a-z0-9._]*))?$"); // rel: contains at least one digit
private static final Pattern NAME_PATTERN = Pattern.compile("^[+()a-zA-Z0-9._-]+$");
private final Optional<String> epoch;
@@ -74,15 +74,6 @@ public class YumPackageName {
architecture = packageName.architecture;
}
- /**
- * Set the epoch of the YUM package.
- *
- * <p>WARNING: Should only be invoked if the YUM package actually has an epoch. Typically
- * YUM packages doesn't have one explicitly set, and in case "0" will be used with
- * {@link #toVersionLockName(Version)} (otherwise it fails), but it will be absent from an
- * install with {@link #toName(Version)} (otherwise it fails). This typically means that
- * you should set this only if the epoch is != "0".</p>
- */
public Builder setEpoch(String epoch) { this.epoch = Optional.of(epoch); return this; }
public Builder setName(String name) { this.name = name; return this; }
public Builder setVersion(String version) { this.version = Optional.of(version); return this; }
@@ -100,7 +91,6 @@ public class YumPackageName {
Optional<String> architecture) {
if (Strings.isNullOrEmpty(name))
throw new IllegalArgumentException("name cannot be null or empty");
-
this.epoch = epoch;
this.name = name;
this.version = version;
@@ -116,6 +106,7 @@ public class YumPackageName {
* <ol>
* <li>name
* <li>name.arch
+ * <li>name-ver
* <li>name-ver-rel
* <li>name-ver-rel.arch
* <li>name-epoch:ver-rel.arch
@@ -125,19 +116,12 @@ public class YumPackageName {
* @throws IllegalArgumentException if spec does not specify a package name.
* @see #parseString(String)
*/
- public static YumPackageName fromString(final String packageSpec) {
+ public static YumPackageName fromString(String packageSpec) {
String spec = packageSpec;
Optional<String> epoch = Optional.empty();
String name = null;
- // packageSpec spec
- // name name
- // name.arch name.arch
- // name-ver-rel name-ver-rel
- // name-ver-rel.arch name-ver-rel.arch
- // name-epoch:ver-rel.arch name-epoch:ver-rel.arch
- // epoch:name-ver-rel.arch epoch:name-ver-rel.arch
-
+ // Parse epoch and remove it from spec
int epochColon = spec.indexOf(':');
if (epochColon >= 0) {
Matcher epochMatcher = EPOCH_PATTERN.matcher(spec.substring(0, epochColon));
@@ -148,17 +132,12 @@ public class YumPackageName {
name = epochMatcher.group(2);
epoch = Optional.of(epochMatcher.group(3));
- spec = spec.substring(epochColon + 1);
+ spec = epochColon == 0
+ ? spec.substring(epochColon + 1)
+ : spec.substring(0, epochColon - 1) + spec.substring(epochColon + 1);
}
- // packageSpec spec
- // name name
- // name.arch name.arch
- // name-ver-rel name-ver-rel
- // name-ver-rel.arch name-ver-rel.arch
- // name-epoch:ver-rel.arch ver-rel.arch (non-null name)
- // epoch:name-ver-rel.arch name-ver-rel.arch
-
+ // Parse architecture and remove it from spec
Optional<String> architecture = Optional.empty();
Matcher architectureMatcher = ARCHITECTURE_PATTERN.matcher(spec);
if (architectureMatcher.find()) {
@@ -166,44 +145,27 @@ public class YumPackageName {
spec = spec.substring(0, architectureMatcher.start());
}
- // packageSpec spec
- // name name
- // name.arch name
- // name-ver-rel name-ver-rel
- // name-ver-rel.arch name-ver-rel
- // name-epoch:ver-rel.arch ver-rel (non-null name)
- // epoch:name-ver-rel.arch name-ver-rel
-
+ // Parse name, version and release and remove the latter two from spec
Optional<String> version = Optional.empty();
Optional<String> release = Optional.empty();
Matcher matcher = NAME_VER_REL_PATTERN.matcher(spec);
if (matcher.find()) {
- // spec format one of:
- // 1. name-ver-rel
- // 2. ver-rel
-
- spec = matcher.group(2);
+ spec = matcher.group(1);
if (spec == null) {
if (name == null) {
throw new IllegalArgumentException("No package name was found: " + packageSpec);
}
spec = name; // makes spec hold the package name in all cases below.
- } else if (name != null) {
- throw new IllegalArgumentException("Ambiguous package names were found for " +
- packageSpec + ": '" + name + "' and '" + spec + "'");
}
- version = Optional.of(matcher.group(3));
- release = Optional.of(matcher.group(4));
+ version = Optional.of(matcher.group(2));
+ release = Optional.ofNullable(matcher.group(3));
}
- // packageSpec spec
- // name name
- // name.arch name
- // name-ver-rel name
- // name-ver-rel.arch name
- // name-epoch:ver-rel.arch name
- // epoch:name-ver-rel.arch name
+ // Set default epoch if we have a version
+ if (version.isPresent() && epoch.isEmpty()) {
+ epoch = Optional.of("0");
+ }
if (!NAME_PATTERN.matcher(spec).find()) {
throw new IllegalArgumentException("Bad package name in " + packageSpec + ": '" + spec + "'");
@@ -229,13 +191,12 @@ public class YumPackageName {
public Optional<String> getArchitecture() { return architecture; }
/** Return package name, omitting components that are not specified. */
- public String toName(Version yumVersion) {
+ public String toName() {
StringBuilder builder = new StringBuilder();
- boolean isBare = version.isEmpty() && release.isEmpty() && architecture.isEmpty();
char nextDelimiter;
builder.append(name);
// Fully versioned package names must always include epoch in Yum 4
- epoch.or(() -> Optional.of("0").filter(v -> !isBare))
+ epoch.or(() -> Optional.of("0").filter(v -> version.isPresent()))
.ifPresent(ep -> builder.append('-').append(ep));
nextDelimiter = ':';
version.ifPresent(s -> builder.append(nextDelimiter).append(s));
@@ -249,7 +210,7 @@ public class YumPackageName {
*
* @throws IllegalStateException if any field required for the version lock spec is missing
*/
- public String toVersionLockName(Version yumVersion) {
+ public String toVersionLockName() {
Builder b = new Builder(this).setArchitecture("*");
if (epoch.isEmpty()) {
b.setEpoch("0");
@@ -257,7 +218,7 @@ public class YumPackageName {
YumPackageName lockSpec = b.build();
if (lockSpec.getVersion().isEmpty()) throw new IllegalStateException("Version is missing for YUM package " + name);
if (lockSpec.getRelease().isEmpty()) throw new IllegalStateException("Release is missing for YUM package " + name);
- return lockSpec.toName(yumVersion);
+ return lockSpec.toName();
}
public boolean isSubsetOf(YumPackageName other) {
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumTester.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumTester.java
index a73f5d04704..e48a0fa2683 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumTester.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumTester.java
@@ -1,7 +1,6 @@
// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.hosted.node.admin.task.util.yum;
-import com.yahoo.component.Version;
import com.yahoo.vespa.hosted.node.admin.task.util.process.TestChildProcess2;
import com.yahoo.vespa.hosted.node.admin.task.util.process.TestTerminal;
@@ -17,20 +16,10 @@ import java.util.stream.Stream;
public class YumTester extends Yum {
private final TestTerminal terminal;
- private final Version yumVersion;
public YumTester(TestTerminal terminal) {
- this(terminal, YumVersion.rhel8);
- }
-
- public YumTester(TestTerminal terminal, YumVersion yumVersion) {
super(terminal);
this.terminal = terminal;
- this.yumVersion = yumVersion.asVersion();
- }
-
- public Version yumVersion() {
- return yumVersion;
}
public GenericYumCommandExpectation expectInstall(String... packages) {
@@ -81,10 +70,6 @@ public class YumTester extends Yum {
}
}
- protected void expectYumVersion() {
- terminal.expectCommand("yum --version 2>&1", 0, yumVersion.toFullString() + "\ntrailing garbage\n");
- }
-
private YumTester execute(String output) {
if (commandType == CommandType.install)
terminal.interceptCommand("rpm query", cmd -> new TestChildProcess2(1, "Not installed"));
@@ -106,7 +91,7 @@ public class YumTester extends Yum {
if (commandType == CommandType.upgrade && packages.size() > 1)
cmd.append(" --setopt skip_missing_names_on_update=False");
packages.forEach(pkg -> {
- String name = pkg.toName(yumVersion);
+ String name = pkg.toName();
if (name.contains("(") || name.contains(")")) { // Ugly hack to handle implicit quoting done in com.yahoo.vespa.hosted.node.admin.task.util.process.CommandLine
name = "\"" + name + "\"";
}
@@ -114,26 +99,20 @@ public class YumTester extends Yum {
});
cmd.append(" 2>&1");
- expectYumVersion();
terminal.expectCommand(cmd.toString(), 0, output);
return YumTester.this;
}
}
public class InstallFixedCommandExpectation extends GenericYumCommandExpectation {
+
private InstallFixedCommandExpectation(String yumPackage) {
super(CommandType.installFixed, yumPackage);
}
@Override
- protected void expectYumVersion() {}
-
- @Override
public YumTester andReturn(boolean value) {
- // Pretend package is already correctly version-locked to simplify expectations
- terminal.expectCommand("yum --version 2>&1", 0, yumVersion.toFullString() + "\ntrailing garbage\n");
-
- terminal.expectCommand("yum versionlock list 2>&1", 0, packages.get(0).toVersionLockName(yumVersion));
+ terminal.expectCommand("yum versionlock list 2>&1", 0, packages.get(0).toVersionLockName());
return super.andReturn(value);
}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumVersion.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumVersion.java
deleted file mode 100644
index 8eba4561805..00000000000
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumVersion.java
+++ /dev/null
@@ -1,25 +0,0 @@
-// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.hosted.node.admin.task.util.yum;
-
-import com.yahoo.component.Version;
-
-/**
- * Red Hat versions and their associated Yum/DNF major version.
- *
- * @author mpolden
- */
-public enum YumVersion {
-
- rhel8(4);
-
- private final Version version;
-
- YumVersion(int yumMajor) {
- this.version = new Version(yumMajor, 0, 0);
- }
-
- public Version asVersion() {
- return version;
- }
-
-}
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumPackageNameTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumPackageNameTest.java
index 0341558d538..a4dc3ac359a 100644
--- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumPackageNameTest.java
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumPackageNameTest.java
@@ -1,7 +1,6 @@
// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.hosted.node.admin.task.util.yum;
-import com.yahoo.component.Version;
import org.junit.Test;
import java.util.Optional;
@@ -26,7 +25,7 @@ public class YumPackageNameTest {
.setRelease("71.git3e8e77d.el7.centos.1")
.setArchitecture("x86_64")
.build();
- assertEquals("docker-2:1.12.6-71.git3e8e77d.el7.centos.1.x86_64", yumPackage.toName(Version.fromString("4")));
+ assertEquals("docker-2:1.12.6-71.git3e8e77d.el7.centos.1.x86_64", yumPackage.toName());
}
@Test
@@ -61,11 +60,21 @@ public class YumPackageNameTest {
null,
null,
"x86_64",
- "docker-engine-selinux-0.x86_64",
+ "docker-engine-selinux.x86_64",
null);
+ // name-ver
+ verifyPackageName("docker-engine-selinux-1.12.6",
+ "0",
+ "docker-engine-selinux",
+ "1.12.6",
+ null,
+ null,
+ "docker-engine-selinux-0:1.12.6",
+ null);
+
// name-ver-rel
- verifyPackageName("docker-engine-selinux-0:1.12.6-1.el7",
+ verifyPackageName("docker-engine-selinux-1.12.6-1.el7",
"0",
"docker-engine-selinux",
"1.12.6",
@@ -75,7 +84,7 @@ public class YumPackageNameTest {
"docker-engine-selinux-0:1.12.6-1.el7.*");
// name-ver-rel.arch
- verifyPackageName("docker-engine-selinux-0:1.12.6-1.el7.x86_64",
+ verifyPackageName("docker-engine-selinux-1.12.6-1.el7.x86_64",
"0",
"docker-engine-selinux",
"1.12.6",
@@ -97,7 +106,7 @@ public class YumPackageNameTest {
// epoch:name-ver-rel.arch
verifyPackageName(
- "docker-2:1.12.6-71.git3e8e77d.el7.centos.1.x86_64",
+ "2:docker-1.12.6-71.git3e8e77d.el7.centos.1.x86_64",
"2",
"docker",
"1.12.6",
@@ -107,7 +116,7 @@ public class YumPackageNameTest {
"docker-2:1.12.6-71.git3e8e77d.el7.centos.1.*");
}
- private void verifyPackageName(String packageName,
+ private void verifyPackageName(String input,
String epoch,
String name,
String version,
@@ -115,32 +124,33 @@ public class YumPackageNameTest {
String architecture,
String toName,
String toVersionName) {
- YumVersion yumVersion = YumVersion.rhel8;
- YumPackageName yumPackageName = YumPackageName.fromString(packageName);
- verifyValue(epoch, yumPackageName.getEpoch());
- verifyValue(name, Optional.of(yumPackageName.getName()));
- verifyValue(version, yumPackageName.getVersion());
- verifyValue(release, yumPackageName.getRelease());
- verifyValue(architecture, yumPackageName.getArchitecture());
- verifyValue(toName, Optional.of(yumPackageName.toName(yumVersion.asVersion())));
+ YumPackageName yumPackageName = YumPackageName.fromString(input);
+ assertPackageField("epoch", epoch, yumPackageName.getEpoch());
+ assertPackageField("name", name, Optional.of(yumPackageName.getName()));
+ assertPackageField("version", version, yumPackageName.getVersion());
+ assertPackageField("release", release, yumPackageName.getRelease());
+ assertPackageField("architecture", architecture, yumPackageName.getArchitecture());
+ assertPackageField("toName()", toName, Optional.of(yumPackageName.toName()));
if (toVersionName == null) {
try {
- yumPackageName.toVersionLockName(yumVersion.asVersion());
+ yumPackageName.toVersionLockName();
fail();
} catch (IllegalStateException e) {
- assertThat(e.getMessage(), containsStringIgnoringCase("Version is missing "));
+ assertTrue("Exception message contains expected substring: " + e.getMessage(),
+ e.getMessage().contains("Version is missing ") ||
+ e.getMessage().contains("Release is missing "));
}
} else {
- assertEquals(toVersionName, yumPackageName.toVersionLockName(yumVersion.asVersion()));
+ assertEquals(toVersionName, yumPackageName.toVersionLockName());
}
}
- private void verifyValue(String value, Optional<String> actual) {
- if (value == null) {
- assertFalse(actual.isPresent());
+ private void assertPackageField(String field, String expected, Optional<String> actual) {
+ if (expected == null) {
+ assertFalse(field + " is not present", actual.isPresent());
} else {
- assertEquals(value, actual.get());
+ assertEquals(field + " has expected value", expected, actual.get());
}
}
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumTest.java
index 2699135b5b9..62fd7410e95 100644
--- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumTest.java
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumTest.java
@@ -79,20 +79,6 @@ public class YumTest {
@Test
public void testAlreadyInstalled() {
mockRpmQuery("package-1", null);
- mockYumVersion();
- terminal.expectCommand(
- "yum install --assumeyes --enablerepo=repo1 --enablerepo=repo2 --setopt skip_missing_names_on_install=False package-1 package-2 2>&1",
- 0,
- "foobar\nNothing to do\n");
-
- assertFalse(yum
- .install("package-1", "package-2")
- .enableRepo("repo1", "repo2")
- .converge(taskContext));
-
- // RHEL 8
- mockRpmQuery("package-1", null);
- mockYumVersion();
terminal.expectCommand(
"yum install --assumeyes --enablerepo=repo1 --enablerepo=repo2 --setopt skip_missing_names_on_install=False package-1 package-2 2>&1",
0,
@@ -104,18 +90,6 @@ public class YumTest {
@Test
public void testAlreadyUpgraded() {
- mockYumVersion();
- terminal.expectCommand(
- "yum upgrade --assumeyes --setopt skip_missing_names_on_update=False package-1 package-2 2>&1",
- 0,
- "foobar\nNo packages marked for update\n");
-
- assertFalse(yum
- .upgrade("package-1", "package-2")
- .converge(taskContext));
-
- // RHEL 8
- mockYumVersion();
terminal.expectCommand(
"yum upgrade --assumeyes --setopt skip_missing_names_on_update=False package-1 package-2 2>&1",
0,
@@ -128,19 +102,6 @@ public class YumTest {
@Test
public void testAlreadyRemoved() {
mockRpmQuery("package-1", YumPackageName.fromString("package-1-1.2.3-1"));
- mockYumVersion();
- terminal.expectCommand(
- "yum remove --assumeyes package-1 package-2 2>&1",
- 0,
- "foobar\nNo Packages marked for removal\n");
-
- assertFalse(yum
- .remove("package-1", "package-2")
- .converge(taskContext));
-
- // RHEL 8
- mockRpmQuery("package-1", YumPackageName.fromString("package-1-1.2.3-1"));
- mockYumVersion();
terminal.expectCommand(
"yum remove --assumeyes package-1 package-2 2>&1",
0,
@@ -160,7 +121,6 @@ public class YumTest {
@Test
public void testInstall() {
mockRpmQuery("package-1", null);
- mockYumVersion();
terminal.expectCommand(
"yum install --assumeyes --setopt skip_missing_names_on_install=False package-1 package-2 2>&1",
0,
@@ -181,7 +141,6 @@ public class YumTest {
@Test
public void testInstallWithEnablerepo() {
mockRpmQuery("package-1", null);
- mockYumVersion();
terminal.expectCommand(
"yum install --assumeyes --enablerepo=repo-name --setopt skip_missing_names_on_install=False package-1 package-2 2>&1",
0,
@@ -195,23 +154,6 @@ public class YumTest {
@Test
public void testWithVersionLock() {
- mockYumVersion();
- terminal.expectCommand("yum versionlock list 2>&1",
- 0,
- "Repository chef_rpms-release is listed more than once in the configuration\n" +
- "0:chef-12.21.1-1.el7.*\n");
- terminal.expectCommand("yum versionlock add --assumeyes \"package-0:0.10-654.el7.*\" 2>&1");
- terminal.expectCommand(
- "yum install --assumeyes package-0:0.10-654.el7.x86_64 2>&1",
- 0,
- "installing");
-
- assertTrue(yum.installFixedVersion(YumPackageName.fromString("package-0:0.10-654.el7.x86_64")).converge(taskContext));
- }
-
- @Test
- public void testWithVersionLockYum4() {
- mockYumVersion();
terminal.expectCommand("yum versionlock list 2>&1",
0,
"Last metadata expiration check: 0:51:26 ago on Thu 14 Jan 2021 09:39:24 AM UTC.\n");
@@ -232,7 +174,6 @@ public class YumTest {
@Test
public void testWithDifferentVersionLock() {
- mockYumVersion();
terminal.expectCommand("yum versionlock list 2>&1",
0,
"Repository chef_rpms-release is listed more than once in the configuration\n" +
@@ -257,7 +198,6 @@ public class YumTest {
@Test
public void testWithExistingVersionLock() {
- mockYumVersion();
terminal.expectCommand("yum versionlock list 2>&1",
0,
"Repository chef_rpms-release is listed more than once in the configuration\n" +
@@ -273,7 +213,6 @@ public class YumTest {
@Test
public void testWithDowngrade() {
- mockYumVersion();
terminal.expectCommand("yum versionlock list 2>&1",
0,
"Repository chef_rpms-release is listed more than once in the configuration\n" +
@@ -294,7 +233,6 @@ public class YumTest {
@Test(expected = ChildProcessFailureException.class)
public void testFailedInstall() {
mockRpmQuery("package-1", null);
- mockYumVersion();
terminal.expectCommand(
"yum install --assumeyes --enablerepo=repo-name --setopt skip_missing_names_on_install=False package-1 package-2 2>&1",
1,
@@ -310,7 +248,6 @@ public class YumTest {
@Test
public void testUnknownPackages() {
mockRpmQuery("package-1", null);
- mockYumVersion();
terminal.expectCommand(
"yum install --assumeyes --setopt skip_missing_names_on_install=False package-1 package-2 package-3 2>&1",
0,
@@ -337,15 +274,10 @@ public class YumTest {
@Test
public void allowToCallUpgradeWithNoPackages() {
- mockYumVersion();
terminal.expectCommand("yum upgrade --assumeyes 2>&1", 0, "OK");
yum.upgrade().converge(taskContext);
}
- private void mockYumVersion() {
- terminal.expectCommand("yum --version 2>&1", 0, YumVersion.rhel8.asVersion().toFullString() + "\ntrailing garbage\n");
- }
-
private void mockRpmQuery(String packageName, YumPackageName installedOrNull) {
new YumTester(terminal).expectQueryInstalled(packageName).andReturn(installedOrNull);
}
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumTesterTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumTesterTest.java
index bb4bb2686a3..93fffffb2fa 100644
--- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumTesterTest.java
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumTesterTest.java
@@ -37,7 +37,7 @@ public class YumTesterTest {
assertYumMethod(yum -> yum.expectRemove(packages).withEnableRepo(repos),
yum -> yum.remove(List.of(packages)).enableRepo(repos).converge(context));
- assertYumMethod(yum -> yum.expectInstallFixedVersion(minimalPackage.toName(yum.yumVersion())).withEnableRepo(repos),
+ assertYumMethod(yum -> yum.expectInstallFixedVersion(minimalPackage.toName()).withEnableRepo(repos),
yum -> yum.installFixedVersion(minimalPackage).enableRepo(repos).converge(context));
}
diff --git a/node-repository/pom.xml b/node-repository/pom.xml
index 30aa76658fd..e7fdc560bc5 100644
--- a/node-repository/pom.xml
+++ b/node-repository/pom.xml
@@ -26,6 +26,12 @@
</dependency>
<dependency>
<groupId>com.yahoo.vespa</groupId>
+ <artifactId>container-apache-http-client-bundle</artifactId>
+ <version>${project.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>com.yahoo.vespa</groupId>
<artifactId>config-provisioning</artifactId>
<version>${project.version}</version>
<scope>provided</scope>
@@ -79,10 +85,6 @@
<scope>compile</scope>
</dependency>
<dependency>
- <groupId>org.apache.httpcomponents</groupId>
- <artifactId>httpclient</artifactId>
- </dependency>
- <dependency>
<groupId>com.yahoo.vespa</groupId>
<artifactId>http-utils</artifactId>
<version>${project.version}</version>
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/AllocatableClusterResources.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/AllocatableClusterResources.java
index b465917e9c9..b799e0056f3 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/AllocatableClusterResources.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/AllocatableClusterResources.java
@@ -107,8 +107,13 @@ public class AllocatableClusterResources {
}
public boolean preferableTo(AllocatableClusterResources other) {
- if (this.fulfilment < 1 || other.fulfilment < 1)
- return this.fulfilment > other.fulfilment; // we always want to fulfil as much as possible
+ if (this.fulfilment < 1 || other.fulfilment < 1) // always fulfil as much as possible
+ return this.fulfilment > other.fulfilment;
+
+ if (clusterSpec.type().isContent() // always prefer local storage on content nodes
+ && this.realResources.storageType() != other.realResources().nodeResources().storageType())
+ return this.realResources.storageType() == NodeResources.StorageType.local;
+
return this.cost() < other.cost(); // otherwise, prefer lower cost
}
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeResourceLimits.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeResourceLimits.java
index 59556418fb5..2b64cc86c9a 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeResourceLimits.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeResourceLimits.java
@@ -7,6 +7,7 @@ import com.yahoo.config.provision.NodeResources;
import com.yahoo.config.provision.NodeType;
import com.yahoo.config.provision.SystemName;
import com.yahoo.config.provision.Zone;
+import com.yahoo.vespa.hosted.provision.Node;
import com.yahoo.vespa.hosted.provision.NodeRepository;
import java.util.Locale;
@@ -40,7 +41,10 @@ public class NodeResourceLimits {
/** Returns whether the real resources we'll end up with on a given tenant node are within limits */
public boolean isWithinRealLimits(NodeCandidate candidateNode, ClusterSpec cluster) {
if (candidateNode.type() != NodeType.tenant) return true; // Resource limits only apply to tenant nodes
- return isWithinRealLimits(nodeRepository.resourcesCalculator().realResourcesOf(candidateNode, nodeRepository, cluster.isExclusive()),
+ // This node is allocated exclusively if that has been explicitly requested, or if the host of the node was
+ // provisioned exclusively
+ boolean exclusive = cluster.isExclusive() || candidateNode.parent.flatMap(Node::exclusiveTo).isPresent();
+ return isWithinRealLimits(nodeRepository.resourcesCalculator().realResourcesOf(candidateNode, nodeRepository, exclusive),
cluster.type());
}
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicAllocationTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicAllocationTest.java
index 761afed23cc..5c452989f56 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicAllocationTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicAllocationTest.java
@@ -1,7 +1,6 @@
// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.hosted.provision.provisioning;
-import com.google.common.collect.ImmutableSet;
import com.yahoo.config.provision.ApplicationId;
import com.yahoo.config.provision.Capacity;
import com.yahoo.config.provision.ClusterMembership;
@@ -24,7 +23,6 @@ import com.yahoo.vespa.hosted.provision.Node.State;
import com.yahoo.vespa.hosted.provision.NodeList;
import com.yahoo.vespa.hosted.provision.node.Agent;
import com.yahoo.vespa.hosted.provision.node.IP;
-import org.junit.Ignore;
import org.junit.Test;
import java.time.Instant;
@@ -150,7 +148,7 @@ public class DynamicAllocationTest {
}
@Test
- public void test_allocation_balancing() {
+ public void allocation_balancing() {
// Here we test balancing between cpu and memory and ignore disk
ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))).flavorsConfig(flavorsConfig()).build();
@@ -234,14 +232,12 @@ public class DynamicAllocationTest {
fail("Two groups have been allocated to the same parent host");
}
- @Ignore // TODO: Re-enable if we reintroduce spare capacity requirement
@Test
public void spare_capacity_used_only_when_replacement() {
- // Use spare capacity only when replacement (i.e one node is failed)
- // Test should allocate as much capacity as possible, verify that it is not possible to allocate one more unit
- // Verify that there is still capacity (available spare)
- // Fail one node and redeploy, Verify that one less node is empty.
- ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))).flavorsConfig(flavorsConfig()).build();
+ ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east")))
+ .flavorsConfig(flavorsConfig())
+ .spareCount(2)
+ .build();
// Setup test
ApplicationId application1 = ProvisioningTester.applicationId();
@@ -252,7 +248,7 @@ public class DynamicAllocationTest {
// Deploy initial state (can max deploy 3 nodes due to redundancy requirements)
ClusterSpec clusterSpec = clusterSpec("myContent.t1.a1");
List<HostSpec> hosts = tester.prepare(application1, clusterSpec, 3, 1, flavor);
- tester.activate(application1, ImmutableSet.copyOf(hosts));
+ tester.activate(application1, Set.copyOf(hosts));
List<Node> initialSpareCapacity = findSpareCapacity(tester);
assertEquals(2, initialSpareCapacity.size());
@@ -264,10 +260,9 @@ public class DynamicAllocationTest {
tester.fail(hosts.get(0));
hosts = tester.prepare(application1, clusterSpec, 3, 1, flavor);
- tester.activate(application1, ImmutableSet.copyOf(hosts));
+ tester.activate(application1, Set.copyOf(hosts));
List<Node> finalSpareCapacity = findSpareCapacity(tester);
-
assertEquals(1, finalSpareCapacity.size());
}
@@ -278,7 +273,7 @@ public class DynamicAllocationTest {
tester.activateTenantHosts();
ApplicationId application1 = ProvisioningTester.applicationId();
List<HostSpec> hosts = tester.prepare(application1, clusterSpec("myContent.t1.a1"), 3, 1, new NodeResources(1, 4, 100, 1));
- tester.activate(application1, ImmutableSet.copyOf(hosts));
+ tester.activate(application1, Set.copyOf(hosts));
List<Node> initialSpareCapacity = findSpareCapacity(tester);
assertEquals(0, initialSpareCapacity.size());
@@ -291,7 +286,7 @@ public class DynamicAllocationTest {
tester.activateTenantHosts();
ApplicationId application1 = ProvisioningTester.applicationId();
List<HostSpec> hosts = tester.prepare(application1, clusterSpec("myContent.t1.a1"), 3, 1, new NodeResources(1, 4, 100, 1));
- tester.activate(application1, ImmutableSet.copyOf(hosts));
+ tester.activate(application1, Set.copyOf(hosts));
}
@Test(expected = OutOfCapacityException.class)
@@ -315,8 +310,8 @@ public class DynamicAllocationTest {
tester.activate(application, hosts);
NodeList activeNodes = tester.nodeRepository().nodes().list().owner(application);
- assertEquals(ImmutableSet.of("127.0.127.13", "::13"), activeNodes.asList().get(0).ipConfig().primary());
- assertEquals(ImmutableSet.of("127.0.127.2", "::2"), activeNodes.asList().get(1).ipConfig().primary());
+ assertEquals(Set.of("127.0.127.13", "::13"), activeNodes.asList().get(0).ipConfig().primary());
+ assertEquals(Set.of("127.0.127.2", "::2"), activeNodes.asList().get(1).ipConfig().primary());
}
@Test
@@ -374,7 +369,7 @@ public class DynamicAllocationTest {
}
@Test
- public void nodeResourcesAreRelaxedInDev() {
+ public void node_resources_are_relaxed_in_dev() {
ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.dev, RegionName.from("us-east"))).flavorsConfig(flavorsConfig()).build();
tester.makeReadyNodes(2, new Flavor(new NodeResources(1, 8, 120, 1, NodeResources.DiskSpeed.fast)), NodeType.host, 10, true);
tester.makeReadyNodes(2, new Flavor(new NodeResources(1, 8, 120, 1, NodeResources.DiskSpeed.slow)), NodeType.host, 10, true);
@@ -394,7 +389,7 @@ public class DynamicAllocationTest {
}
@Test
- public void testSwitchingFromLegacyFlavorSyntaxToResourcesDoesNotCauseReallocation() {
+ public void switching_from_legacy_flavor_syntax_to_resources_does_not_cause_reallocation() {
ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))).flavorsConfig(flavorsConfig()).build();
tester.makeReadyNodes(2, new Flavor(new NodeResources(5, 20, 1400, 3)), NodeType.host, 10, true);
tester.activateTenantHosts();
@@ -413,7 +408,7 @@ public class DynamicAllocationTest {
}
@Test
- public void testPreferExclusiveNetworkSwitch() {
+ public void prefer_exclusive_network_switch() {
// Hosts are provisioned, without switch information
ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))).flavorsConfig(flavorsConfig()).build();
NodeResources hostResources = new NodeResources(32, 128, 2000, 10);
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicProvisioningTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicProvisioningTest.java
index 6f1e2630434..0b7b9f2fa13 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicProvisioningTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicProvisioningTest.java
@@ -364,7 +364,7 @@ public class DynamicProvisioningTest {
}
@Test
- public void test_any_disk_prefers_remote() {
+ public void test_any_disk_prefers_remote_for_container() {
int memoryTax = 3;
int localDiskTax = 55;
// Disk tax is not included in flavor resources but memory tax is
@@ -383,7 +383,7 @@ public class DynamicProvisioningTest {
tester.activateTenantHosts();
ApplicationId app1 = ProvisioningTester.applicationId("app1");
- ClusterSpec cluster1 = ClusterSpec.request(ClusterSpec.Type.content, new ClusterSpec.Id("cluster1")).vespaVersion("7").build();
+ ClusterSpec cluster1 = ClusterSpec.request(ClusterSpec.Type.container, new ClusterSpec.Id("cluster1")).vespaVersion("7").build();
tester.activate(app1, cluster1, Capacity.from(resources(4, 2, 2, 10, 200, fast, StorageType.any),
resources(6, 3, 3, 25, 400, fast, StorageType.any)));
@@ -392,6 +392,35 @@ public class DynamicProvisioningTest {
app1, cluster1);
}
+ @Test
+ public void test_any_disk_prefers_local_for_content() {
+ int memoryTax = 3;
+ int localDiskTax = 55;
+ // Disk tax is not included in flavor resources but memory tax is
+ List<Flavor> flavors = List.of(new Flavor("2x", new NodeResources(2, 20 - memoryTax, 200, 0.1, fast, local)),
+ new Flavor("4x", new NodeResources(4, 40 - memoryTax, 400, 0.1, fast, local)),
+ new Flavor("2xl", new NodeResources(2, 20 - memoryTax, 200, 0.1, fast, remote)),
+ new Flavor("4xl", new NodeResources(4, 40 - memoryTax, 400, 0.1, fast, remote)));
+
+ ProvisioningTester tester = new ProvisioningTester.Builder().zone(zone)
+ .flavors(flavors)
+ .hostProvisioner(new MockHostProvisioner(flavors, memoryTax))
+ .nameResolver(nameResolver)
+ .resourcesCalculator(memoryTax, localDiskTax)
+ .build();
+
+ tester.activateTenantHosts();
+
+ ApplicationId app1 = ProvisioningTester.applicationId("app1");
+ ClusterSpec cluster1 = ClusterSpec.request(ClusterSpec.Type.content, new ClusterSpec.Id("cluster1")).vespaVersion("7").build();
+
+ tester.activate(app1, cluster1, Capacity.from(resources(4, 2, 2, 10, 200, fast, StorageType.any),
+ resources(6, 3, 3, 25, 400, fast, StorageType.any)));
+ tester.assertNodes("'any' selects a flavor with local storage",
+ 6, 2, 2, 20, 200, fast, local,
+ app1, cluster1);
+ }
+
private void prepareAndActivate(ApplicationId application, ClusterSpec clusterSpec, int nodes, int groups, NodeResources resources) {
List<HostSpec> prepared = tester.prepare(application, clusterSpec, nodes, groups, resources);
NodeList provisionedHosts = tester.nodeRepository().nodes().list(Node.State.provisioned).nodeType(NodeType.host);
diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/ZkStatusService.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/ZkStatusService.java
index daafc81e0f3..8902bf1641c 100644
--- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/ZkStatusService.java
+++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/ZkStatusService.java
@@ -3,7 +3,6 @@ package com.yahoo.vespa.orchestrator.status;
import com.google.common.util.concurrent.UncheckedTimeoutException;
import com.yahoo.config.provision.ApplicationId;
-import com.yahoo.container.jaxrs.annotation.Component;
import com.yahoo.jdisc.Metric;
import com.yahoo.jdisc.Timer;
import com.yahoo.path.Path;
@@ -54,16 +53,10 @@ public class ZkStatusService implements StatusService {
private final ConcurrentHashMap<Map<String, String>, Metric.Context> cachedContexts = new ConcurrentHashMap<>();
@Inject
- public ZkStatusService(
- @Component Curator curator,
- @Component Metric metric,
- @Component Timer timer,
- @Component AntiServiceMonitor antiServiceMonitor) {
- this(curator,
- metric,
- timer,
- new HostInfosCache(curator, new HostInfosServiceImpl(curator, timer)),
- antiServiceMonitor);
+ public ZkStatusService(Curator curator, Metric metric, Timer timer, AntiServiceMonitor antiServiceMonitor) {
+ this(curator, metric, timer,
+ new HostInfosCache(curator, new HostInfosServiceImpl(curator, timer)),
+ antiServiceMonitor);
}
/** Non-private for testing only. */
diff --git a/parent/pom.xml b/parent/pom.xml
index 908491835ea..2a8527f80ea 100644
--- a/parent/pom.xml
+++ b/parent/pom.xml
@@ -863,6 +863,7 @@
-->
<curator.version>5.1.0</curator.version>
<jna.version>4.5.2</jna.version>
+ <commons.codec.version>1.15</commons.codec.version>
<commons.math3.version>3.6.1</commons.math3.version>
<junit.version>5.7.0</junit.version>
<maven-assembly-plugin.version>3.1.1</maven-assembly-plugin.version>
diff --git a/searchcore/CMakeLists.txt b/searchcore/CMakeLists.txt
index 55b3b66a75c..dfcc7542e09 100644
--- a/searchcore/CMakeLists.txt
+++ b/searchcore/CMakeLists.txt
@@ -143,6 +143,7 @@ vespa_define_module(
src/tests/proton/reprocessing/reprocessing_runner
src/tests/proton/server
src/tests/proton/server/disk_mem_usage_filter
+ src/tests/proton/server/disk_mem_usage_metrics
src/tests/proton/server/disk_mem_usage_sampler
src/tests/proton/server/health_adapter
src/tests/proton/server/memory_flush_config_updater
diff --git a/searchcore/src/tests/proton/server/disk_mem_usage_metrics/CMakeLists.txt b/searchcore/src/tests/proton/server/disk_mem_usage_metrics/CMakeLists.txt
new file mode 100644
index 00000000000..fbd557991c8
--- /dev/null
+++ b/searchcore/src/tests/proton/server/disk_mem_usage_metrics/CMakeLists.txt
@@ -0,0 +1,9 @@
+# Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_executable(searchcore_disk_mem_usage_metrics_test_app TEST
+ SOURCES
+ disk_mem_usage_metrics_test.cpp
+ DEPENDS
+ searchcore_server
+ GTest::GTest
+)
+vespa_add_test(NAME searchcore_disk_mem_usage_metrics_test_app COMMAND searchcore_disk_mem_usage_metrics_test_app)
diff --git a/searchcore/src/tests/proton/server/disk_mem_usage_metrics/disk_mem_usage_metrics_test.cpp b/searchcore/src/tests/proton/server/disk_mem_usage_metrics/disk_mem_usage_metrics_test.cpp
new file mode 100644
index 00000000000..569633c518b
--- /dev/null
+++ b/searchcore/src/tests/proton/server/disk_mem_usage_metrics/disk_mem_usage_metrics_test.cpp
@@ -0,0 +1,39 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/searchcore/proton/server/disk_mem_usage_metrics.h>
+#include <vespa/searchcore/proton/server/disk_mem_usage_state.h>
+#include <vespa/vespalib/gtest/gtest.h>
+
+
+using proton::DiskMemUsageMetrics;
+using proton::DiskMemUsageState;
+using proton::ResourceUsageState;
+
+bool
+expect_metrics(double disk_usage, double disk_utilization, double memory_usage, double memory_utilization, const DiskMemUsageMetrics &dm_metrics)
+{
+ bool result = true;
+ EXPECT_DOUBLE_EQ(disk_usage, dm_metrics.get_disk_usage()) << (result = false, "");
+ EXPECT_DOUBLE_EQ(disk_utilization, dm_metrics.get_disk_utilization()) << (result = false, "");
+ EXPECT_DOUBLE_EQ(memory_usage, dm_metrics.get_memory_usage()) << (result = false, "");
+ EXPECT_DOUBLE_EQ(memory_utilization, dm_metrics.get_memory_utilization()) << (result = false, "");
+ return result;
+}
+
+TEST(DiskMemUsageMetricsTest, default_value_is_zero)
+{
+ DiskMemUsageMetrics dm_metrics;
+ EXPECT_TRUE(expect_metrics(0.0, 0.0, 0.0, 0.0, dm_metrics));
+}
+
+TEST(DiskMemUsageMetricsTest, merging_uses_max)
+{
+ DiskMemUsageMetrics dm_metrics({ResourceUsageState(0.5, 0.4), ResourceUsageState(0.5, 0.3)});
+ EXPECT_TRUE(expect_metrics(0.4, 0.8, 0.3, 0.6, dm_metrics));
+ dm_metrics.merge({ResourceUsageState(0.4, 0.4), ResourceUsageState(0.5, 0.4)});
+ EXPECT_TRUE(expect_metrics(0.4, 1.0, 0.4, 0.8, dm_metrics));
+ dm_metrics.merge({ResourceUsageState(0.5, 0.4), ResourceUsageState(0.5, 0.3)});
+ EXPECT_TRUE(expect_metrics(0.4, 1.0, 0.4, 0.8, dm_metrics));
+}
+
+GTEST_MAIN_RUN_ALL_TESTS()
diff --git a/searchcore/src/vespa/searchcore/proton/docsummary/docsumcontext.cpp b/searchcore/src/vespa/searchcore/proton/docsummary/docsumcontext.cpp
index c0b5bcab791..b8d1d2f17e6 100644
--- a/searchcore/src/vespa/searchcore/proton/docsummary/docsumcontext.cpp
+++ b/searchcore/src/vespa/searchcore/proton/docsummary/docsumcontext.cpp
@@ -71,6 +71,7 @@ DocsumContext::createReply()
reply->docsums.resize(_docsumState._docsumcnt);
SymbolTable::UP symbols = std::make_unique<SymbolTable>();
IDocsumWriter::ResolveClassInfo rci = _docsumWriter.resolveClassInfo(_docsumState._args.getResultClassName(), _docsumStore.getSummaryClassId());
+ _docsumState._omit_summary_features = rci.outputClass->omit_summary_features();
for (uint32_t i = 0; i < _docsumState._docsumcnt; ++i) {
buf.reset();
uint32_t docId = _docsumState._docsumbuf[i];
@@ -114,6 +115,7 @@ DocsumContext::createSlimeReply()
const Symbol docsumSym = response->insert(DOCSUM);
IDocsumWriter::ResolveClassInfo rci = _docsumWriter.resolveClassInfo(_docsumState._args.getResultClassName(),
_docsumStore.getSummaryClassId());
+ _docsumState._omit_summary_features = rci.outputClass->omit_summary_features();
uint32_t i(0);
for (i = 0; (i < _docsumState._docsumcnt) && !_request.expired(); ++i) {
uint32_t docId = _docsumState._docsumbuf[i];
diff --git a/searchcore/src/vespa/searchcore/proton/server/CMakeLists.txt b/searchcore/src/vespa/searchcore/proton/server/CMakeLists.txt
index d2ebb7a58a2..285e259e1a8 100644
--- a/searchcore/src/vespa/searchcore/proton/server/CMakeLists.txt
+++ b/searchcore/src/vespa/searchcore/proton/server/CMakeLists.txt
@@ -11,6 +11,7 @@ vespa_add_library(searchcore_server STATIC
ddbstate.cpp
disk_mem_usage_filter.cpp
disk_mem_usage_forwarder.cpp
+ disk_mem_usage_metrics.cpp
disk_mem_usage_sampler.cpp
docstorevalidator.cpp
document_db_config_owner.cpp
diff --git a/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_filter.cpp b/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_filter.cpp
index ef77081132b..cbd378da9a0 100644
--- a/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_filter.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_filter.cpp
@@ -154,13 +154,14 @@ DiskMemUsageFilter::getDiskUsedRatio(const Guard &guard) const
DiskMemUsageFilter::DiskMemUsageFilter(const HwInfo &hwInfo)
: _lock(),
_hwInfo(hwInfo),
+ _acceptWrite(true),
_memoryStats(),
_diskUsedSizeBytes(),
_transient_memory_usage(0u),
_config(),
_state(),
- _acceptWrite(true),
_dmstate(),
+ _disk_mem_usage_metrics(),
_listeners()
{ }
@@ -254,6 +255,15 @@ DiskMemUsageFilter::usageState() const
return _dmstate;
}
+DiskMemUsageMetrics
+DiskMemUsageFilter::get_metrics() const
+{
+ Guard guard(_lock);
+ DiskMemUsageMetrics result(_disk_mem_usage_metrics);
+ _disk_mem_usage_metrics = DiskMemUsageMetrics(_dmstate);
+ return result;
+}
+
bool
DiskMemUsageFilter::acceptWriteOperation() const
{
@@ -295,6 +305,7 @@ DiskMemUsageFilter::notifyDiskMemUsage(const Guard &guard, DiskMemUsageState sta
return;
}
_dmstate = state;
+ _disk_mem_usage_metrics.merge(state);
for (const auto &listener : _listeners) {
listener->notifyDiskMemUsage(_dmstate);
}
diff --git a/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_filter.h b/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_filter.h
index cdb9fc5f4cb..5bec34e51d9 100644
--- a/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_filter.h
+++ b/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_filter.h
@@ -4,6 +4,7 @@
#include "i_disk_mem_usage_notifier.h"
#include "disk_mem_usage_state.h"
+#include "disk_mem_usage_metrics.h"
#include <vespa/searchcore/proton/common/hw_info.h>
#include <vespa/searchcore/proton/persistenceengine/i_resource_write_filter.h>
#include <vespa/vespalib/util/process_memory_stats.h>
@@ -40,16 +41,18 @@ public:
};
private:
- mutable Mutex _lock; // protect _memoryStats, _usedDiskSizeBytes, _config, _state
+ mutable Mutex _lock;
HwInfo _hwInfo;
+ std::atomic<bool> _acceptWrite;
+ // Following member variables are protected by _lock
vespalib::ProcessMemoryStats _memoryStats;
uint64_t _diskUsedSizeBytes;
size_t _transient_memory_usage;
size_t _transient_disk_usage;
Config _config;
State _state;
- std::atomic<bool> _acceptWrite;
DiskMemUsageState _dmstate;
+ mutable DiskMemUsageMetrics _disk_mem_usage_metrics;
std::vector<IDiskMemUsageListener *> _listeners;
void recalcState(const Guard &guard); // called with _lock held
@@ -73,6 +76,7 @@ public:
Config getConfig() const;
const HwInfo &getHwInfo() const { return _hwInfo; }
DiskMemUsageState usageState() const;
+ DiskMemUsageMetrics get_metrics() const;
bool acceptWriteOperation() const override;
State getAcceptState() const override;
void addDiskMemUsageListener(IDiskMemUsageListener *listener) override;
diff --git a/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_metrics.cpp b/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_metrics.cpp
new file mode 100644
index 00000000000..2a97c56e63f
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_metrics.cpp
@@ -0,0 +1,31 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include "disk_mem_usage_metrics.h"
+#include "disk_mem_usage_state.h"
+#include <algorithm>
+
+namespace proton {
+
+DiskMemUsageMetrics::DiskMemUsageMetrics() noexcept
+ : DiskMemUsageMetrics(DiskMemUsageState())
+{
+}
+
+DiskMemUsageMetrics::DiskMemUsageMetrics(const DiskMemUsageState &usage_state) noexcept
+ : _disk_usage(usage_state.diskState().usage()),
+ _disk_utilization(usage_state.diskState().utilization()),
+ _memory_usage(usage_state.memoryState().usage()),
+ _memory_utilization(usage_state.memoryState().utilization())
+{
+}
+
+void
+DiskMemUsageMetrics::merge(const DiskMemUsageState &usage_state) noexcept
+{
+ _disk_usage = std::max(_disk_usage, usage_state.diskState().usage());
+ _disk_utilization = std::max(_disk_utilization, usage_state.diskState().utilization());
+ _memory_usage = std::max(_memory_usage, usage_state.memoryState().usage());
+ _memory_utilization = std::max(_memory_utilization, usage_state.memoryState().utilization());
+}
+
+}
diff --git a/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_metrics.h b/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_metrics.h
new file mode 100644
index 00000000000..34d3bb36b40
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_metrics.h
@@ -0,0 +1,30 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+namespace proton {
+
+class DiskMemUsageState;
+
+/**
+ * Class containing disk and memory usage in a form suitable for
+ * metrics reporting.
+ */
+class DiskMemUsageMetrics
+{
+ double _disk_usage;
+ double _disk_utilization;
+ double _memory_usage;
+ double _memory_utilization;
+
+public:
+ DiskMemUsageMetrics() noexcept;
+ DiskMemUsageMetrics(const DiskMemUsageState &usage_state) noexcept;
+ void merge(const DiskMemUsageState &usage_state) noexcept;
+ double get_disk_usage() const noexcept { return _disk_usage; }
+ double get_disk_utilization() const noexcept { return _disk_utilization; }
+ double get_memory_usage() const noexcept { return _memory_usage; }
+ double get_memory_utilization() const noexcept { return _memory_utilization; }
+};
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/server/proton.cpp b/searchcore/src/vespa/searchcore/proton/server/proton.cpp
index adae323c1d9..ef72a52eb02 100644
--- a/searchcore/src/vespa/searchcore/proton/server/proton.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/proton.cpp
@@ -756,11 +756,11 @@ Proton::updateMetrics(const metrics::MetricLockGuard &)
}
const DiskMemUsageFilter &usageFilter = _diskMemUsageSampler->writeFilter();
- DiskMemUsageState usageState = usageFilter.usageState();
- metrics.resourceUsage.disk.set(usageState.diskState().usage());
- metrics.resourceUsage.diskUtilization.set(usageState.diskState().utilization());
- metrics.resourceUsage.memory.set(usageState.memoryState().usage());
- metrics.resourceUsage.memoryUtilization.set(usageState.memoryState().utilization());
+ auto dm_metrics = usageFilter.get_metrics();
+ metrics.resourceUsage.disk.set(dm_metrics.get_disk_usage());
+ metrics.resourceUsage.diskUtilization.set(dm_metrics.get_disk_utilization());
+ metrics.resourceUsage.memory.set(dm_metrics.get_memory_usage());
+ metrics.resourceUsage.memoryUtilization.set(dm_metrics.get_memory_utilization());
metrics.resourceUsage.transient_memory.set(usageFilter.get_relative_transient_memory_usage());
metrics.resourceUsage.transient_disk.set(usageFilter.get_relative_transient_disk_usage());
metrics.resourceUsage.memoryMappings.set(usageFilter.getMemoryStats().getMappingsCount());
diff --git a/searchcore/src/vespa/searchcore/proton/server/resource_usage_state.h b/searchcore/src/vespa/searchcore/proton/server/resource_usage_state.h
index 29f9959aea9..0b6f34ba055 100644
--- a/searchcore/src/vespa/searchcore/proton/server/resource_usage_state.h
+++ b/searchcore/src/vespa/searchcore/proton/server/resource_usage_state.h
@@ -18,7 +18,7 @@ private:
public:
ResourceUsageState()
- : _limit(0),
+ : _limit(1.0),
_usage(0)
{
}
diff --git a/searchsummary/src/vespa/searchsummary/docsummary/docsumstate.cpp b/searchsummary/src/vespa/searchsummary/docsummary/docsumstate.cpp
index 99a372d9676..6f042ee3907 100644
--- a/searchsummary/src/vespa/searchsummary/docsummary/docsumstate.cpp
+++ b/searchsummary/src/vespa/searchsummary/docsummary/docsumstate.cpp
@@ -36,6 +36,7 @@ GetDocsumsState::GetDocsumsState(GetDocsumsStateCallback &callback)
_parsedLocations(),
_summaryFeatures(nullptr),
_summaryFeaturesCached(false),
+ _omit_summary_features(false),
_rankFeatures(nullptr),
_matching_elements(),
_jsonStringer()
diff --git a/searchsummary/src/vespa/searchsummary/docsummary/docsumstate.h b/searchsummary/src/vespa/searchsummary/docsummary/docsumstate.h
index db0f8e6e8ad..88a95d0446c 100644
--- a/searchsummary/src/vespa/searchsummary/docsummary/docsumstate.h
+++ b/searchsummary/src/vespa/searchsummary/docsummary/docsumstate.h
@@ -84,6 +84,7 @@ public:
// used by SummaryFeaturesDFW
FeatureSet::SP _summaryFeatures;
bool _summaryFeaturesCached;
+ bool _omit_summary_features;
// used by RankFeaturesDFW
FeatureSet::SP _rankFeatures;
diff --git a/searchsummary/src/vespa/searchsummary/docsummary/resultclass.cpp b/searchsummary/src/vespa/searchsummary/docsummary/resultclass.cpp
index 8066a5e65db..f52cc62af92 100644
--- a/searchsummary/src/vespa/searchsummary/docsummary/resultclass.cpp
+++ b/searchsummary/src/vespa/searchsummary/docsummary/resultclass.cpp
@@ -15,7 +15,8 @@ ResultClass::ResultClass(const char *name, uint32_t id, util::StringEnum & field
_nameMap(),
_fieldEnum(fieldEnum),
_enumMap(),
- _dynInfo(NULL)
+ _dynInfo(NULL),
+ _omit_summary_features(false)
{ }
diff --git a/searchsummary/src/vespa/searchsummary/docsummary/resultclass.h b/searchsummary/src/vespa/searchsummary/docsummary/resultclass.h
index 52e331cd365..88c3552387b 100644
--- a/searchsummary/src/vespa/searchsummary/docsummary/resultclass.h
+++ b/searchsummary/src/vespa/searchsummary/docsummary/resultclass.h
@@ -141,6 +141,9 @@ private:
util::StringEnum &_fieldEnum; // fieldname -> f.n. enum value [SHARED]
std::vector<int> _enumMap; // fieldname enum value -> entry index
DynamicInfo *_dynInfo; // fields overridden and generated
+ // Whether or not summary features should be omitted when filling this summary class.
+ // As default, summary features are always included.
+ bool _omit_summary_features;
public:
typedef std::unique_ptr<ResultClass> UP;
@@ -278,6 +281,14 @@ public:
{
return (offset < _entries.size()) ? &_entries[offset] : NULL;
}
+
+ void set_omit_summary_features(bool value) {
+ _omit_summary_features = value;
+ }
+
+ bool omit_summary_features() const {
+ return _omit_summary_features;
+ }
};
}
diff --git a/searchsummary/src/vespa/searchsummary/docsummary/resultconfig.cpp b/searchsummary/src/vespa/searchsummary/docsummary/resultconfig.cpp
index ead3a4a2f9d..579779efb73 100644
--- a/searchsummary/src/vespa/searchsummary/docsummary/resultconfig.cpp
+++ b/searchsummary/src/vespa/searchsummary/docsummary/resultconfig.cpp
@@ -127,25 +127,27 @@ ResultConfig::ReadConfig(const vespa::config::search::SummaryConfig &cfg, const
int maxclassID = 0x7fffffff; // avoid negative classids
_defaultSummaryId = cfg.defaultsummaryid;
for (uint32_t i = 0; rc && i < cfg.classes.size(); i++) {
- if (cfg.classes[i].name.empty()) {
+ const auto& cfg_class = cfg.classes[i];
+ if (cfg_class.name.empty()) {
LOG(warning, "%s classes[%d]: empty name", configId, i);
}
- int classID = cfg.classes[i].id;
+ int classID = cfg_class.id;
if (classID < 0 || classID > maxclassID) {
LOG(error, "%s classes[%d]: bad id %d", configId, i, classID);
rc = false;
break;
}
- ResultClass *resClass = AddResultClass(cfg.classes[i].name.c_str(), classID);
+ ResultClass *resClass = AddResultClass(cfg_class.name.c_str(), classID);
if (resClass == nullptr) {
- LOG(error,"%s: unable to add classes[%d] name %s", configId, i, cfg.classes[i].name.c_str());
+ LOG(error,"%s: unable to add classes[%d] name %s", configId, i, cfg_class.name.c_str());
rc = false;
break;
}
- for (unsigned int j = 0; rc && (j < cfg.classes[i].fields.size()); j++) {
- const char *fieldtype = cfg.classes[i].fields[j].type.c_str();
- const char *fieldname = cfg.classes[i].fields[j].name.c_str();
- LOG(debug, "Reconfiguring class '%s' field '%s' of type '%s'", cfg.classes[i].name.c_str(), fieldname, fieldtype);
+ resClass->set_omit_summary_features(cfg_class.omitsummaryfeatures);
+ for (unsigned int j = 0; rc && (j < cfg_class.fields.size()); j++) {
+ const char *fieldtype = cfg_class.fields[j].type.c_str();
+ const char *fieldname = cfg_class.fields[j].name.c_str();
+ LOG(debug, "Reconfiguring class '%s' field '%s' of type '%s'", cfg_class.name.c_str(), fieldname, fieldtype);
if (strcmp(fieldtype, "integer") == 0) {
rc = resClass->AddConfigEntry(fieldname, RES_INT);
} else if (strcmp(fieldtype, "short") == 0) {
@@ -179,12 +181,12 @@ ResultConfig::ReadConfig(const vespa::config::search::SummaryConfig &cfg, const
} else if (strcmp(fieldtype, "featuredata") == 0) {
rc = resClass->AddConfigEntry(fieldname, RES_FEATUREDATA);
} else {
- LOG(error, "%s %s.fields[%d]: unknown type '%s'", configId, cfg.classes[i].name.c_str(), j, fieldtype);
+ LOG(error, "%s %s.fields[%d]: unknown type '%s'", configId, cfg_class.name.c_str(), j, fieldtype);
rc = false;
break;
}
if (!rc) {
- LOG(error, "%s %s.fields[%d]: duplicate name '%s'", configId, cfg.classes[i].name.c_str(), j, fieldname);
+ LOG(error, "%s %s.fields[%d]: duplicate name '%s'", configId, cfg_class.name.c_str(), j, fieldname);
break;
}
}
diff --git a/searchsummary/src/vespa/searchsummary/docsummary/summaryfeaturesdfw.cpp b/searchsummary/src/vespa/searchsummary/docsummary/summaryfeaturesdfw.cpp
index 425faff6a67..4d81f4505a7 100644
--- a/searchsummary/src/vespa/searchsummary/docsummary/summaryfeaturesdfw.cpp
+++ b/searchsummary/src/vespa/searchsummary/docsummary/summaryfeaturesdfw.cpp
@@ -30,6 +30,9 @@ static vespalib::Memory _M_cached("vespa.summaryFeatures.cached");
void
SummaryFeaturesDFW::insertField(uint32_t docid, GetDocsumsState *state, ResType type, vespalib::slime::Inserter &target)
{
+ if (state->_omit_summary_features) {
+ return;
+ }
if ( ! state->_summaryFeatures) {
state->_callback.FillSummaryFeatures(state, _env);
if ( !state->_summaryFeatures) { // still no summary features to write
diff --git a/service-monitor/pom.xml b/service-monitor/pom.xml
index 578fcc83006..df28737f352 100644
--- a/service-monitor/pom.xml
+++ b/service-monitor/pom.xml
@@ -16,14 +16,6 @@
<description>Service monitor component for hosted vespa.</description>
<dependencies>
- <!-- compile scope -->
- <dependency>
- <groupId>org.apache.httpcomponents</groupId>
- <artifactId>httpclient</artifactId>
- <version>4.5</version>
- <!-- This is necessary to get 4.4's HostnameVerifier API of SSLConnectionSocketFactory::new -->
- <scope>compile</scope>
- </dependency>
<dependency>
<groupId>com.yahoo.vespa</groupId>
<artifactId>http-utils</artifactId>
@@ -34,6 +26,12 @@
<!-- provided scope -->
<dependency>
<groupId>com.yahoo.vespa</groupId>
+ <artifactId>container-apache-http-client-bundle</artifactId>
+ <version>${project.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>com.yahoo.vespa</groupId>
<artifactId>config</artifactId>
<version>${project.version}</version>
<scope>provided</scope>
diff --git a/standalone-container/pom.xml b/standalone-container/pom.xml
index 11d0406fcc5..4aa66e9b1a4 100644
--- a/standalone-container/pom.xml
+++ b/standalone-container/pom.xml
@@ -90,6 +90,8 @@
config-model-api-jar-with-dependencies.jar,
config-model-jar-with-dependencies.jar,
container-disc-jar-with-dependencies.jar,
+ model-evaluation-jar-with-dependencies.jar,
+ model-integration-jar-with-dependencies.jar,
vespajlib.jar
</discPreInstallBundle>
</configuration>
diff --git a/standalone-container/src/main/sh/standalone-container.sh b/standalone-container/src/main/sh/standalone-container.sh
index 3f3937658d2..b30a7ad4cd0 100755
--- a/standalone-container/src/main/sh/standalone-container.sh
+++ b/standalone-container/src/main/sh/standalone-container.sh
@@ -167,6 +167,7 @@ StartCommand() {
--add-opens=java.base/java.io=ALL-UNNAMED \
--add-opens=java.base/java.lang=ALL-UNNAMED \
--add-opens=java.base/java.net=ALL-UNNAMED \
+ --add-opens=java.base/java.nio=ALL-UNNAMED \
--add-opens=java.base/jdk.internal.loader=ALL-UNNAMED \
--add-opens=java.base/sun.security.ssl=ALL-UNNAMED \
-Djava.library.path="$VESPA_HOME/lib64" \
diff --git a/vespa-athenz/pom.xml b/vespa-athenz/pom.xml
index 90ab2a81e0c..1b03891f313 100644
--- a/vespa-athenz/pom.xml
+++ b/vespa-athenz/pom.xml
@@ -25,6 +25,12 @@
</dependency>
<dependency>
<groupId>com.yahoo.vespa</groupId>
+ <artifactId>container-apache-http-client-bundle</artifactId>
+ <version>${project.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>com.yahoo.vespa</groupId>
<artifactId>security-utils</artifactId>
<version>${project.version}</version>
<scope>provided</scope>
@@ -131,14 +137,6 @@
</exclusions>
</dependency>
<dependency>
- <groupId>org.apache.httpcomponents</groupId>
- <artifactId>httpcore</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.httpcomponents</groupId>
- <artifactId>httpclient</artifactId>
- </dependency>
- <dependency>
<groupId>com.amazonaws</groupId>
<artifactId>aws-java-sdk-core</artifactId>
<exclusions>
@@ -154,6 +152,10 @@
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-annotations</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpclient</artifactId>
+ </exclusion>
</exclusions>
</dependency>
<dependency>
@@ -170,7 +172,7 @@
<!-- required by java-jwt -->
<groupId>commons-codec</groupId>
<artifactId>commons-codec</artifactId>
- <version>1.15</version>
+ <version>${commons.codec.version}</version>
<scope>compile</scope>
</dependency>
</dependencies>
diff --git a/vespa-hadoop/src/main/java/com/yahoo/vespa/hadoop/mapreduce/VespaOutputFormat.java b/vespa-hadoop/src/main/java/com/yahoo/vespa/hadoop/mapreduce/VespaOutputFormat.java
index 97bc7dc838e..f80c28424f2 100644
--- a/vespa-hadoop/src/main/java/com/yahoo/vespa/hadoop/mapreduce/VespaOutputFormat.java
+++ b/vespa-hadoop/src/main/java/com/yahoo/vespa/hadoop/mapreduce/VespaOutputFormat.java
@@ -35,7 +35,7 @@ public class VespaOutputFormat extends OutputFormat {
public RecordWriter getRecordWriter(TaskAttemptContext context) throws IOException, InterruptedException {
VespaCounters counters = VespaCounters.get(context);
VespaConfiguration configuration = VespaConfiguration.get(context.getConfiguration(), configOverride);
- return configuration.useLegacyClient()
+ return configuration.useLegacyClient().orElse(true)
? new LegacyVespaRecordWriter(configuration, counters)
: new VespaRecordWriter(configuration, counters);
}
diff --git a/vespa-hadoop/src/main/java/com/yahoo/vespa/hadoop/mapreduce/util/VespaConfiguration.java b/vespa-hadoop/src/main/java/com/yahoo/vespa/hadoop/mapreduce/util/VespaConfiguration.java
index 7219e621486..1be794f8e11 100644
--- a/vespa-hadoop/src/main/java/com/yahoo/vespa/hadoop/mapreduce/util/VespaConfiguration.java
+++ b/vespa-hadoop/src/main/java/com/yahoo/vespa/hadoop/mapreduce/util/VespaConfiguration.java
@@ -6,6 +6,7 @@ import org.apache.hadoop.conf.Configuration;
import java.io.IOException;
import java.io.StringReader;
+import java.util.Optional;
import java.util.Properties;
public class VespaConfiguration {
@@ -131,7 +132,11 @@ public class VespaConfiguration {
return getInt(PROGRESS_REPORT, 1000);
}
- public boolean useLegacyClient() { return getBoolean(USE_LEGACY_CLIENT, true); }
+ public Optional<Boolean> useLegacyClient() {
+ String raw = getString(USE_LEGACY_CLIENT);
+ if (raw == null || raw.trim().isEmpty()) return Optional.empty();
+ return Optional.of(Boolean.parseBoolean(raw));
+ }
public String getString(String name) {
if (override != null && override.containsKey(name)) {
@@ -191,6 +196,7 @@ public class VespaConfiguration {
sb.append(MAX_IN_FLIGHT_REQUESTS + ": " + maxInFlightRequests() +"\n");
sb.append(RANDOM_STARTUP_SLEEP + ": " + randomStartupSleepMs() +"\n");
sb.append(NUM_RETRIES + ": " + numRetries() +"\n");
+ sb.append(USE_LEGACY_CLIENT + ": " + useLegacyClient().map(Object::toString).orElse("<empty>") +"\n");
return sb.toString();
}
diff --git a/vespa-maven-plugin/src/main/java/ai/vespa/hosted/plugin/GenerateTestDescriptorMojo.java b/vespa-maven-plugin/src/main/java/ai/vespa/hosted/plugin/GenerateTestDescriptorMojo.java
index 259ae2602c4..69c0e343872 100644
--- a/vespa-maven-plugin/src/main/java/ai/vespa/hosted/plugin/GenerateTestDescriptorMojo.java
+++ b/vespa-maven-plugin/src/main/java/ai/vespa/hosted/plugin/GenerateTestDescriptorMojo.java
@@ -39,6 +39,8 @@ public class GenerateTestDescriptorMojo extends AbstractMojo {
}
private void analyzeTestClasses(TestAnnotationAnalyzer analyzer) throws MojoExecutionException {
+ if (! Files.exists(testClassesDirectory())) return;
+
try (Stream<Path> files = Files.walk(testClassesDirectory())) {
files
.filter(f -> f.toString().endsWith(".class"))
diff --git a/vespaclient-container-plugin/pom.xml b/vespaclient-container-plugin/pom.xml
index 834c3d7c988..17443b11b6a 100644
--- a/vespaclient-container-plugin/pom.xml
+++ b/vespaclient-container-plugin/pom.xml
@@ -38,11 +38,6 @@
<scope>provided</scope>
</dependency>
<dependency>
- <groupId>org.apache.httpcomponents</groupId>
- <artifactId>httpclient</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
<groupId>com.yahoo.vespa</groupId>
<artifactId>vespa-http-client</artifactId>
<version>${project.version}</version>
diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/DocumentV1ApiHandler.java b/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/DocumentV1ApiHandler.java
index 9db296e33cd..770aa9bdb81 100644
--- a/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/DocumentV1ApiHandler.java
+++ b/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/DocumentV1ApiHandler.java
@@ -1133,7 +1133,7 @@ public class DocumentV1ApiHandler extends AbstractRequestHandler {
default:
response.writeMessage(error.get() != null ? error.get() : message != null ? message : "Visiting failed");
if (getVisitorStatistics() != null)
- response.writeDocumentCount(getVisitorStatistics().getDocumentsReturned());
+ response.writeDocumentCount(getVisitorStatistics().getDocumentsVisited());
response.respond(Response.Status.BAD_GATEWAY);
}
diff --git a/zkfacade/pom.xml b/zkfacade/pom.xml
index 25cc678fe66..7140c490460 100644
--- a/zkfacade/pom.xml
+++ b/zkfacade/pom.xml
@@ -50,6 +50,12 @@
<dependency>
<groupId>org.apache.curator</groupId>
<artifactId>curator-recipes</artifactId>
+ <exclusions>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
<dependency>
<groupId>com.google.guava</groupId>
@@ -57,19 +63,27 @@
<scope>provided</scope>
</dependency>
<dependency>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-jdk14</artifactId>
- </dependency>
- <dependency>
- <!-- Needed to have the same version as slf4j-api -->
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-log4j12</artifactId>
- <version>${slf4j.version}</version>
- </dependency>
- <dependency>
<groupId>org.apache.zookeeper</groupId>
<artifactId>zookeeper</artifactId>
<version>${zookeeper.client.version}</version>
+ <exclusions>
+ <!--
+ Container provides wiring for all common log libraries
+ Duplicate embedding results in various warnings being printed to stderr
+ -->
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>log4j</groupId>
+ <artifactId>log4j</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
<!-- snappy-java and metrics-core are included here
to be able to work with ZooKeeper 3.6.2 due to
@@ -79,6 +93,12 @@
<artifactId>metrics-core</artifactId>
<scope>compile</scope>
<version>3.2.5</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
<dependency>
<groupId>org.xerial.snappy</groupId>
diff --git a/zookeeper-server/zookeeper-server-3.6.2/pom.xml b/zookeeper-server/zookeeper-server-3.6.2/pom.xml
index 2f0029169ea..e40ba8fa161 100644
--- a/zookeeper-server/zookeeper-server-3.6.2/pom.xml
+++ b/zookeeper-server/zookeeper-server-3.6.2/pom.xml
@@ -33,15 +33,24 @@
<groupId>org.apache.zookeeper</groupId>
<artifactId>zookeeper</artifactId>
<version>3.6.2</version>
- </dependency>
- <dependency>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-jdk14</artifactId>
- </dependency>
- <dependency>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-log4j12</artifactId>
- <version>${slf4j.version}</version>
+ <exclusions>
+ <!--
+ Container provides wiring for all common log libraries
+ Duplicate embedding results in various warnings being printed to stderr
+ -->
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>log4j</groupId>
+ <artifactId>log4j</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
<!-- snappy-java and metrics-core are included here
to be able to work with ZooKeeper 3.6.2 due to
@@ -51,6 +60,12 @@
<artifactId>metrics-core</artifactId>
<scope>compile</scope>
<version>3.2.5</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
<dependency>
<groupId>org.xerial.snappy</groupId>
diff --git a/zookeeper-server/zookeeper-server-3.6.3/pom.xml b/zookeeper-server/zookeeper-server-3.6.3/pom.xml
index a4568d3585f..3074f3d2117 100644
--- a/zookeeper-server/zookeeper-server-3.6.3/pom.xml
+++ b/zookeeper-server/zookeeper-server-3.6.3/pom.xml
@@ -33,15 +33,24 @@
<groupId>org.apache.zookeeper</groupId>
<artifactId>zookeeper</artifactId>
<version>3.6.3</version>
- </dependency>
- <dependency>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-jdk14</artifactId>
- </dependency>
- <dependency>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-log4j12</artifactId>
- <version>${slf4j.version}</version>
+ <exclusions>
+ <!--
+ Container provides wiring for all common log libraries
+ Duplicate embedding results in various warnings being printed to stderr
+ -->
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>log4j</groupId>
+ <artifactId>log4j</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
<!-- snappy-java and metrics-core are included here
to be able to work with ZooKeeper 3.6.3 due to
@@ -51,6 +60,12 @@
<artifactId>metrics-core</artifactId>
<scope>compile</scope>
<version>3.2.5</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
<dependency>
<groupId>org.xerial.snappy</groupId>