diff options
176 files changed, 2411 insertions, 855 deletions
diff --git a/CMakeLists.txt b/CMakeLists.txt index 413d232c00d..f7752d5915e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -16,7 +16,6 @@ find_package(JNI REQUIRED) include(functions.cmake) include(build_settings.cmake) -include(install_java.cmake) # Enable CTest unit testing enable_testing() @@ -24,12 +23,36 @@ enable_testing() # Include vespa config definitions in every target include_directories(BEFORE ${CMAKE_BINARY_DIR}/configdefinitions/src) +add_subdirectory(application-preprocessor) +add_subdirectory(chain) +add_subdirectory(component) +add_subdirectory(config-bundle) +add_subdirectory(config-model) +add_subdirectory(config-model-api) +add_subdirectory(config-provisioning) +add_subdirectory(config-proxy) add_subdirectory(config) +add_subdirectory(config-model-fat) add_subdirectory(configd) add_subdirectory(configdefinitions) add_subdirectory(configserver) add_subdirectory(configutil) +add_subdirectory(container-accesslogging) +add_subdirectory(container-core) +add_subdirectory(container-di) +add_subdirectory(container-disc) +add_subdirectory(container-jersey2) +add_subdirectory(container-messagebus) +add_subdirectory(container-search) +add_subdirectory(container-search-and-docproc) +add_subdirectory(clustercontroller-apps) +add_subdirectory(clustercontroller-apputil) +add_subdirectory(clustercontroller-utils) +add_subdirectory(clustercontroller-core) add_subdirectory(defaults) +add_subdirectory(docker-api) +add_subdirectory(docproc) +add_subdirectory(docprocs) add_subdirectory(document) add_subdirectory(documentapi) add_subdirectory(eval) @@ -41,14 +64,20 @@ add_subdirectory(filedistribution) add_subdirectory(fnet) add_subdirectory(frtstream) add_subdirectory(fsa) +add_subdirectory(jdisc_core) +add_subdirectory(jdisc_http_service) +add_subdirectory(jdisc_jetty) add_subdirectory(jrt_test) add_subdirectory(juniper) add_subdirectory(logd) +add_subdirectory(logserver) add_subdirectory(lowercasing_test) add_subdirectory(memfilepersistence) add_subdirectory(messagebus) add_subdirectory(messagebus_test) add_subdirectory(metrics) +add_subdirectory(node-repository) +add_subdirectory(orchestrator) add_subdirectory(persistence) add_subdirectory(persistencetypes) add_subdirectory(searchcommon) @@ -56,25 +85,32 @@ add_subdirectory(searchcore) add_subdirectory(searchcorespi) add_subdirectory(searchlib) add_subdirectory(searchsummary) +add_subdirectory(simplemetrics) add_subdirectory(slobrok) add_subdirectory(staging_vespalib) +add_subdirectory(standalone-container) add_subdirectory(storage) add_subdirectory(storageapi) add_subdirectory(storageframework) add_subdirectory(storageserver) +add_subdirectory(statistics) add_subdirectory(streamingvisitors) add_subdirectory(vbench) add_subdirectory(vdslib) add_subdirectory(vdstestlib) +add_subdirectory(vespa-http-client) +add_subdirectory(vespa_jersey2) add_subdirectory(vespabase) add_subdirectory(vespaclient) +add_subdirectory(vespaclient-core) +add_subdirectory(vespaclient-container-plugin) +add_subdirectory(vespaclient-java) +add_subdirectory(vespajlib) add_subdirectory(vespalib) add_subdirectory(vespalog) add_subdirectory(vespamalloc) add_subdirectory(vsm) -# Note: Change when cmake gets proper post-install support. -# Post installation steps are run from dist subdirectory which needs to be the last add_subdirectory(...) call in this file. -add_subdirectory(dist) +add_subdirectory(zkfacade) # Create module targets with name ${MODULE}+module depending on every target defined within that module __create_module_targets(TARGETS "module") diff --git a/application-preprocessor/CMakeLists.txt b/application-preprocessor/CMakeLists.txt new file mode 100644 index 00000000000..e40fd4a6736 --- /dev/null +++ b/application-preprocessor/CMakeLists.txt @@ -0,0 +1,4 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +install_fat_java_artifact(application-preprocessor) +vespa_install_script(src/main/sh/vespa-preprocess-application bin) + diff --git a/build_settings.cmake b/build_settings.cmake index 5894eb9082b..425a2eddda7 100644 --- a/build_settings.cmake +++ b/build_settings.cmake @@ -23,14 +23,8 @@ set(C_WARN_OPTS "-Winline -Wuninitialized -Werror -Wall -W -Wchar-subscripts -Wc # Note: this is not a union of C_WARN_OPTS, since CMAKE_CXX_FLAGS already includes CMAKE_C_FLAGS, which in turn includes C_WARN_OPTS transitively set(CXX_SPECIFIC_WARN_OPTS "-Wsuggest-override -Wnon-virtual-dtor") -# Select C++ ABI -if(DEFINED VESPA_CXX_ABI_FLAGS) -else() - set (VESPA_CXX_ABI_FLAGS "-D_GLIBCXX_USE_CXX11_ABI=0") -endif() - # C and C++ compiler flags -set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -g -O3 -fno-omit-frame-pointer ${C_WARN_OPTS} -fPIC ${VESPA_CXX_ABI_FLAGS} -DBOOST_DISABLE_ASSERTS -march=westmere -mtune=intel ${EXTRA_C_FLAGS}") +set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -g -O3 -fno-omit-frame-pointer ${C_WARN_OPTS} -fPIC ${VESPA_CXX_ABI_FLAGS} -DBOOST_DISABLE_ASSERTS ${VESPA_CPU_ARCH_FLAGS} -mtune=intel ${EXTRA_C_FLAGS}") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${CMAKE_C_FLAGS} ${CXX_SPECIFIC_WARN_OPTS} -std=c++1z -fvisibility-inlines-hidden -fdiagnostics-color=auto ${EXTRA_CXX_FLAGS}") # Linker flags diff --git a/dist/CMakeLists.txt b/chain/CMakeLists.txt index d06074074df..3b5b5fd2c99 100644 --- a/dist/CMakeLists.txt +++ b/chain/CMakeLists.txt @@ -1,2 +1,2 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -install(CODE "execute_process(COMMAND ${CMAKE_CURRENT_LIST_DIR}/post_install.sh ${CMAKE_INSTALL_PREFIX})") +install_config_definition(src/main/resources/configdefinitions/chains.def container.core.chains.def) diff --git a/clustercontroller-apps/CMakeLists.txt b/clustercontroller-apps/CMakeLists.txt new file mode 100644 index 00000000000..f59ffbfa7bf --- /dev/null +++ b/clustercontroller-apps/CMakeLists.txt @@ -0,0 +1,2 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +install_fat_java_artifact(clustercontroller-apps) diff --git a/clustercontroller-apputil/CMakeLists.txt b/clustercontroller-apputil/CMakeLists.txt new file mode 100644 index 00000000000..bdfb3ab3ed7 --- /dev/null +++ b/clustercontroller-apputil/CMakeLists.txt @@ -0,0 +1,2 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +install_fat_java_artifact(clustercontroller-apputil) diff --git a/clustercontroller-core/CMakeLists.txt b/clustercontroller-core/CMakeLists.txt new file mode 100644 index 00000000000..6754e893009 --- /dev/null +++ b/clustercontroller-core/CMakeLists.txt @@ -0,0 +1,2 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +install_fat_java_artifact(clustercontroller-core) diff --git a/clustercontroller-utils/CMakeLists.txt b/clustercontroller-utils/CMakeLists.txt new file mode 100644 index 00000000000..250a8e7e693 --- /dev/null +++ b/clustercontroller-utils/CMakeLists.txt @@ -0,0 +1,2 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +install_fat_java_artifact(clustercontroller-utils) diff --git a/component/CMakeLists.txt b/component/CMakeLists.txt new file mode 100644 index 00000000000..87d0a4989ba --- /dev/null +++ b/component/CMakeLists.txt @@ -0,0 +1,2 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +install_fat_java_artifact(component) diff --git a/config-bundle/CMakeLists.txt b/config-bundle/CMakeLists.txt new file mode 100644 index 00000000000..8d4878920a4 --- /dev/null +++ b/config-bundle/CMakeLists.txt @@ -0,0 +1,2 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +install_fat_java_artifact(config-bundle) diff --git a/config-model-api/CMakeLists.txt b/config-model-api/CMakeLists.txt new file mode 100644 index 00000000000..f69e5242e2e --- /dev/null +++ b/config-model-api/CMakeLists.txt @@ -0,0 +1,2 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +install_fat_java_artifact(config-model-api) diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/FileDistribution.java b/config-model-api/src/main/java/com/yahoo/config/model/api/FileDistribution.java index d53b6735064..a5d54fb84b3 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/FileDistribution.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/FileDistribution.java @@ -18,6 +18,7 @@ public interface FileDistribution { void sendDeployedFiles(String hostName, Set<FileReference> fileReferences); void reloadDeployFileDistributor(); + // TODO: Remove when 6.150 is the oldest version used void limitSendingOfDeployedFilesTo(Collection<String> hostNames); void removeDeploymentsThatHaveDifferentApplicationId(Collection<String> targetHostnames); diff --git a/config-model-fat/CMakeLists.txt b/config-model-fat/CMakeLists.txt new file mode 100644 index 00000000000..1e2364556dc --- /dev/null +++ b/config-model-fat/CMakeLists.txt @@ -0,0 +1,5 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +install_java_artifact(config-model-fat) + +install(FILES src/main/resources/config-models.xml + DESTINATION conf/configserver-app) diff --git a/config-model/CMakeLists.txt b/config-model/CMakeLists.txt new file mode 100644 index 00000000000..274ab8a763b --- /dev/null +++ b/config-model/CMakeLists.txt @@ -0,0 +1,10 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +install_fat_java_artifact(config-model) + +vespa_install_script(src/main/perl/vespa-deploy bin) +vespa_install_script(src/main/perl/vespa-expand-config.pl bin) +vespa_install_script(src/main/perl/vespa-replicate-log-stream bin) +vespa_install_script(src/main/sh/vespa-validate-application bin) + +install(DIRECTORY src/main/resources/schema DESTINATION share/vespa PATTERN ".gitignore" EXCLUDE) +install(DIRECTORY src/main/resources/schema DESTINATION share/vespa/schema/version/6.x PATTERN ".gitignore" EXCLUDE) diff --git a/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributor.java b/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributor.java index e07327b1666..8860f5c2249 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributor.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributor.java @@ -102,7 +102,6 @@ public class FileDistributor { } } dbHandler.sendDeployedFiles(fileSourceHost, allFilesToSend()); - dbHandler.limitSendingOfDeployedFilesTo(union(getTargetHostnames(), fileSourceHost)); dbHandler.removeDeploymentsThatHaveDifferentApplicationId(getTargetHostnames()); } diff --git a/config-provisioning/CMakeLists.txt b/config-provisioning/CMakeLists.txt new file mode 100644 index 00000000000..829ba87fab8 --- /dev/null +++ b/config-provisioning/CMakeLists.txt @@ -0,0 +1,3 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +install_fat_java_artifact(config-provisioning) +install_config_definition(src/main/resources/configdefinitions/flavors.def config.provisioning.flavors.def) diff --git a/config-proxy/CMakeLists.txt b/config-proxy/CMakeLists.txt new file mode 100644 index 00000000000..a87f10573be --- /dev/null +++ b/config-proxy/CMakeLists.txt @@ -0,0 +1,6 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +install_fat_java_artifact(config-proxy) + +vespa_install_script(src/main/sh/vespa-config-ctl.sh vespa-config-ctl bin) +vespa_install_script(src/main/sh/vespa-config-loadtester.sh vespa-config-loadtester bin) +vespa_install_script(src/main/sh/vespa-config-verification.sh vespa-config-verification bin) diff --git a/configdefinitions/CMakeLists.txt b/configdefinitions/CMakeLists.txt index d8f89e04cc1..ee78759254a 100644 --- a/configdefinitions/CMakeLists.txt +++ b/configdefinitions/CMakeLists.txt @@ -8,3 +8,5 @@ vespa_define_module( LIBS src/vespa ) + +install_fat_java_artifact(configdefinitions) diff --git a/configdefinitions/src/vespa/CMakeLists.txt b/configdefinitions/src/vespa/CMakeLists.txt index 016739f4594..4ed4dc06d41 100644 --- a/configdefinitions/src/vespa/CMakeLists.txt +++ b/configdefinitions/src/vespa/CMakeLists.txt @@ -5,66 +5,66 @@ vespa_add_library(configdefinitions DEPENDS ) vespa_generate_config(configdefinitions application-id.def) -install(FILES application-id.def RENAME cloud.config.application-id.def DESTINATION var/db/vespa/config_server/serverdb/classes) +install_config_definition(application-id.def cloud.config.application-id.def) vespa_generate_config(configdefinitions attributes.def) -install(FILES attributes.def RENAME vespa.config.search.attributes.def DESTINATION var/db/vespa/config_server/serverdb/classes) +install_config_definition(attributes.def vespa.config.search.attributes.def) vespa_generate_config(configdefinitions cluster-info.def) -install(FILES cluster-info.def RENAME cloud.config.cluster-info.def DESTINATION var/db/vespa/config_server/serverdb/classes) +install_config_definition(cluster-info.def cloud.config.cluster-info.def) vespa_generate_config(configdefinitions cluster-list.def) -install(FILES cluster-list.def RENAME cloud.config.cluster-list.def DESTINATION var/db/vespa/config_server/serverdb/classes) +install_config_definition(cluster-list.def cloud.config.cluster-list.def) vespa_generate_config(configdefinitions configserver.def) -install(FILES configserver.def RENAME cloud.config.configserver.def DESTINATION var/db/vespa/config_server/serverdb/classes) +install_config_definition(configserver.def cloud.config.configserver.def) vespa_generate_config(configdefinitions dispatch.def) -install(FILES dispatch.def RENAME vespa.config.search.dispatch.def DESTINATION var/db/vespa/config_server/serverdb/classes) +install_config_definition(dispatch.def vespa.config.search.dispatch.def) vespa_generate_config(configdefinitions fleetcontroller.def) -install(FILES fleetcontroller.def RENAME vespa.config.content.fleetcontroller.def DESTINATION var/db/vespa/config_server/serverdb/classes) +install_config_definition(fleetcontroller.def vespa.config.content.fleetcontroller.def) vespa_generate_config(configdefinitions ilscripts.def) -install(FILES ilscripts.def RENAME vespa.configdefinition.ilscripts.def DESTINATION var/db/vespa/config_server/serverdb/classes) +install_config_definition(ilscripts.def vespa.configdefinition.ilscripts.def) vespa_generate_config(configdefinitions imported-fields.def) -install(FILES imported-fields.def RENAME vespa.config.search.imported-fields.def DESTINATION var/db/vespa/config_server/serverdb/classes) +install_config_definition(imported-fields.def vespa.config.search.imported-fields.def) vespa_generate_config(configdefinitions indexschema.def) -install(FILES indexschema.def RENAME vespa.config.search.indexschema.def DESTINATION var/db/vespa/config_server/serverdb/classes) +install_config_definition(indexschema.def vespa.config.search.indexschema.def) vespa_generate_config(configdefinitions lb-services.def) -install(FILES lb-services.def RENAME cloud.config.lb-services.def DESTINATION var/db/vespa/config_server/serverdb/classes) +install_config_definition(lb-services.def cloud.config.lb-services.def) vespa_generate_config(configdefinitions load-type.def) -install(FILES load-type.def RENAME vespa.config.content.load-type.def DESTINATION var/db/vespa/config_server/serverdb/classes) +install_config_definition(load-type.def vespa.config.content.load-type.def) vespa_generate_config(configdefinitions messagetyperouteselectorpolicy.def) -install(FILES messagetyperouteselectorpolicy.def RENAME vespa.config.content.messagetyperouteselectorpolicy.def DESTINATION var/db/vespa/config_server/serverdb/classes) +install_config_definition(messagetyperouteselectorpolicy.def vespa.config.content.messagetyperouteselectorpolicy.def) vespa_generate_config(configdefinitions model.def) -install(FILES model.def RENAME cloud.config.model.def DESTINATION var/db/vespa/config_server/serverdb/classes) +install_config_definition(model.def cloud.config.model.def) vespa_generate_config(configdefinitions orchestrator.def) -install(FILES orchestrator.def RENAME vespa.orchestrator.config.orchestrator.def DESTINATION var/db/vespa/config_server/serverdb/classes) +install_config_definition(orchestrator.def vespa.orchestrator.config.orchestrator.def) vespa_generate_config(configdefinitions persistence.def) -install(FILES persistence.def RENAME vespa.config.content.persistence.def DESTINATION var/db/vespa/config_server/serverdb/classes) +install_config_definition(persistence.def vespa.config.content.persistence.def) vespa_generate_config(configdefinitions rank-profiles.def) -install(FILES rank-profiles.def RENAME vespa.config.search.rank-profiles.def DESTINATION var/db/vespa/config_server/serverdb/classes) +install_config_definition(rank-profiles.def vespa.config.search.rank-profiles.def) vespa_generate_config(configdefinitions routing.def) -install(FILES routing.def RENAME cloud.config.routing.def DESTINATION var/db/vespa/config_server/serverdb/classes) +install_config_definition(routing.def cloud.config.routing.def) vespa_generate_config(configdefinitions routing-provider.def) -install(FILES routing-provider.def RENAME cloud.config.routing-provider.def DESTINATION var/db/vespa/config_server/serverdb/classes) +install_config_definition(routing-provider.def cloud.config.routing-provider.def) vespa_generate_config(configdefinitions sentinel.def) -install(FILES sentinel.def RENAME cloud.config.sentinel.def DESTINATION var/db/vespa/config_server/serverdb/classes) +install_config_definition(sentinel.def cloud.config.sentinel.def) vespa_generate_config(configdefinitions slobroks.def) -install(FILES slobroks.def RENAME cloud.config.slobroks.def DESTINATION var/db/vespa/config_server/serverdb/classes) +install_config_definition(slobroks.def cloud.config.slobroks.def) vespa_generate_config(configdefinitions specialtokens.def) -install(FILES specialtokens.def RENAME vespa.configdefinition.specialtokens.def DESTINATION var/db/vespa/config_server/serverdb/classes) +install_config_definition(specialtokens.def vespa.configdefinition.specialtokens.def) vespa_generate_config(configdefinitions stor-devices.def) -install(FILES stor-devices.def RENAME vespa.config.storage.stor-devices.def DESTINATION var/db/vespa/config_server/serverdb/classes) +install_config_definition(stor-devices.def vespa.config.storage.stor-devices.def) vespa_generate_config(configdefinitions stor-distribution.def) -install(FILES stor-distribution.def RENAME vespa.config.content.stor-distribution.def DESTINATION var/db/vespa/config_server/serverdb/classes) +install_config_definition(stor-distribution.def vespa.config.content.stor-distribution.def) vespa_generate_config(configdefinitions stor-filestor.def) -install(FILES stor-filestor.def RENAME vespa.config.content.stor-filestor.def DESTINATION var/db/vespa/config_server/serverdb/classes) +install_config_definition(stor-filestor.def vespa.config.content.stor-filestor.def) vespa_generate_config(configdefinitions stor-memfilepersistence.def) -install(FILES stor-memfilepersistence.def RENAME vespa.config.storage.stor-memfilepersistence.def DESTINATION var/db/vespa/config_server/serverdb/classes) +install_config_definition(stor-memfilepersistence.def vespa.config.storage.stor-memfilepersistence.def) vespa_generate_config(configdefinitions summary.def) -install(FILES summary.def RENAME vespa.config.search.summary.def DESTINATION var/db/vespa/config_server/serverdb/classes) +install_config_definition(summary.def vespa.config.search.summary.def) vespa_generate_config(configdefinitions summarymap.def) -install(FILES summarymap.def RENAME vespa.config.search.summarymap.def DESTINATION var/db/vespa/config_server/serverdb/classes) +install_config_definition(summarymap.def vespa.config.search.summarymap.def) vespa_generate_config(configdefinitions upgrading.def) -install(FILES upgrading.def RENAME vespa.config.content.upgrading.def DESTINATION var/db/vespa/config_server/serverdb/classes) +install_config_definition(upgrading.def vespa.config.content.upgrading.def) vespa_generate_config(configdefinitions ymon.def) -install(FILES ymon.def RENAME cloud.config.ymon.def DESTINATION var/db/vespa/config_server/serverdb/classes) +install_config_definition(ymon.def cloud.config.ymon.def) vespa_generate_config(configdefinitions zookeeper-server.def) -install(FILES zookeeper-server.def RENAME cloud.config.zookeeper-server.def DESTINATION var/db/vespa/config_server/serverdb/classes) +install_config_definition(zookeeper-server.def cloud.config.zookeeper-server.def) vespa_generate_config(configdefinitions zookeepers.def) -install(FILES zookeepers.def RENAME cloud.config.zookeepers.def DESTINATION var/db/vespa/config_server/serverdb/classes) +install_config_definition(zookeepers.def cloud.config.zookeepers.def) diff --git a/configserver/CMakeLists.txt b/configserver/CMakeLists.txt index e54a93b3d28..8134fde2105 100644 --- a/configserver/CMakeLists.txt +++ b/configserver/CMakeLists.txt @@ -1,9 +1,21 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +install_fat_java_artifact(configserver) + vespa_install_script(src/main/sh/vespa-configserver-remove-state bin) vespa_install_script(src/main/sh/start-filedistribution libexec/vespa) vespa_install_script(src/main/sh/ping-configserver libexec/vespa) vespa_install_script(src/main/sh/start-configserver libexec/vespa) vespa_install_script(src/main/sh/start-logd libexec/vespa) vespa_install_script(src/main/sh/stop-configserver libexec/vespa) + install(DIRECTORY src/main/resources/logd DESTINATION conf) install(DIRECTORY src/main/resources/configserver-app DESTINATION conf) + +install(CODE "execute_process(COMMAND mkdir -p $ENV{DESTDIR}/${CMAKE_INSTALL_PREFIX}/conf/configserver-app/components)") +install(CODE "execute_process(COMMAND mkdir -p $ENV{DESTDIR}/${CMAKE_INSTALL_PREFIX}/conf/configserver-app/config-models)") +install(CODE "execute_process(COMMAND ln -snf ${CMAKE_INSTALL_PREFIX}/lib/jars/config-model-fat.jar $ENV{DESTDIR}/${CMAKE_INSTALL_PREFIX}/conf/configserver-app/components/config-model-fat.jar)") +install(CODE "execute_process(COMMAND ln -snf ${CMAKE_INSTALL_PREFIX}/lib/jars/configserver-jar-with-dependencies.jar $ENV{DESTDIR}/${CMAKE_INSTALL_PREFIX}/conf/configserver-app/components/configserver.jar)") +install(CODE "execute_process(COMMAND ln -snf ${CMAKE_INSTALL_PREFIX}/lib/jars/orchestrator-jar-with-dependencies.jar $ENV{DESTDIR}/${CMAKE_INSTALL_PREFIX}/conf/configserver-app/components/orchestrator.jar)") +install(CODE "execute_process(COMMAND ln -snf ${CMAKE_INSTALL_PREFIX}/lib/jars/node-repository-jar-with-dependencies.jar $ENV{DESTDIR}/${CMAKE_INSTALL_PREFIX}/conf/configserver-app/components/node-repository.jar)") +install(CODE "execute_process(COMMAND ln -snf ${CMAKE_INSTALL_PREFIX}/lib/jars/zkfacade-jar-with-dependencies.jar $ENV{DESTDIR}/${CMAKE_INSTALL_PREFIX}/conf/configserver-app/components/zkfacade.jar)") +install(CODE "execute_process(COMMAND ln -snf ${CMAKE_INSTALL_PREFIX}/conf/configserver-app/components $ENV{DESTDIR}/${CMAKE_INSTALL_PREFIX}/lib/jars/config-models)") diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpHandler.java index c13d1b3fcfa..a1fd3ffd729 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpHandler.java @@ -70,8 +70,8 @@ public class HttpHandler extends LoggingRequestHandler { } } - // Override default, since we need a higher timeout - // TODO: Make configurable? Should be higher than timeouts used by clients + // Override default, since we need a higher timeout for some calls + // TODO: Review and see if overriding only in SessionPrepareHandler is enough @Override public Duration getTimeout() { return Duration.ofSeconds(910); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionPrepareHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionPrepareHandler.java index bfc5714467e..03a3f3556e4 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionPrepareHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionPrepareHandler.java @@ -52,6 +52,11 @@ public class SessionPrepareHandler extends SessionHandler { } @Override + public Duration getTimeout() { + return zookeeperBarrierTimeout.plus(Duration.ofSeconds(10)); + } + + @Override protected HttpResponse handlePUT(HttpRequest request) { Tenant tenant = getExistingTenant(request); TenantName tenantName = tenant.getName(); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelsBuilder.java b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelsBuilder.java index 6bd21e9cfb6..af4d998c347 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelsBuilder.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelsBuilder.java @@ -7,6 +7,7 @@ import com.yahoo.config.model.api.HostProvisioner; import com.yahoo.config.model.api.ModelContext; import com.yahoo.config.model.api.ModelFactory; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.ApplicationLockException; import com.yahoo.config.provision.OutOfCapacityException; import com.yahoo.config.provision.AllocatedHosts; import com.yahoo.config.provision.Rotation; @@ -87,7 +88,7 @@ public abstract class ModelsBuilder<MODELRESULT extends ModelResult> { if (allApplicationModels.size() > 0 && allApplicationModels.get(0).getModel().skipOldConfigModels(now)) break; } - catch (OutOfCapacityException e) { + catch (OutOfCapacityException | ApplicationLockException e) { // Don't wrap this exception, and don't try to load other model versions as this is (most likely) // caused by the state of the system, not the model version/application combination throw e; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java index beb62cf3ac9..31be18d9b22 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java @@ -100,6 +100,8 @@ public class SessionPreparer { preparation.distribute(); preparation.reloadDeployFileDistributor(); } + log.log(LogLevel.DEBUG, () -> "time used " + params.getTimeoutBudget().timesUsed() + + " : " + params.getApplicationId()); return preparation.result(); } catch (OutOfCapacityException e) { throw e; diff --git a/configserver/src/main/sh/start-configserver b/configserver/src/main/sh/start-configserver index 4b0191e145a..7d3f305a107 100755 --- a/configserver/src/main/sh/start-configserver +++ b/configserver/src/main/sh/start-configserver @@ -80,6 +80,7 @@ fixddir () { chmod 755 $1 } +fixddir ${VESPA_HOME}/conf/zookeeper fixfile ${VESPA_HOME}/conf/zookeeper/zookeeper.cfg fixfile ${VESPA_HOME}/var/zookeeper/myid fixddir ${VESPA_HOME}/var/zookeeper/version-2 diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java index 265eaa501a2..5dc529e3381 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java @@ -134,7 +134,6 @@ public class SessionPreparerTest extends TestWithCurator { public void require_that_application_is_prepared() throws Exception { preparer.prepare(getContext(getApplicationPackage(testApp)), getLogger(), new PrepareParams.Builder().build(), Optional.empty(), tenantPath, Instant.now()); assertThat(fileDistributionFactory.mockFileDistributionProvider.getMockFileDBHandler().sendDeployedFilesCalled, is(2)); - assertThat(fileDistributionFactory.mockFileDistributionProvider.getMockFileDBHandler().limitSendingOfDeployedFilesToCalled, is(2)); // Should be called only once no matter how many model versions are built assertThat(fileDistributionFactory.mockFileDistributionProvider.getMockFileDBHandler().reloadDeployFileDistributorCalled, is(1)); assertTrue(configCurator.exists(sessionsPath.append(ConfigCurator.USERAPP_ZK_SUBPATH).append("services.xml").getAbsolute())); diff --git a/container-accesslogging/CMakeLists.txt b/container-accesslogging/CMakeLists.txt new file mode 100644 index 00000000000..fb2ba9ac031 --- /dev/null +++ b/container-accesslogging/CMakeLists.txt @@ -0,0 +1,2 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +install_config_definition(src/main/resources/configdefinitions/access-log.def container.core.access-log.def) diff --git a/container-core/CMakeLists.txt b/container-core/CMakeLists.txt new file mode 100644 index 00000000000..2d5723865eb --- /dev/null +++ b/container-core/CMakeLists.txt @@ -0,0 +1,16 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +install_config_definition(src/main/resources/configdefinitions/application-metadata.def container.core.application-metadata.def) +install_config_definition(src/main/resources/configdefinitions/container-document.def container.core.document.container-document.def) +install_config_definition(src/main/resources/configdefinitions/container-http.def container.core.container-http.def) +install_config_definition(src/main/resources/configdefinitions/diagnostics.def container.core.diagnostics.def) +install_config_definition(src/main/resources/configdefinitions/health-monitor.def container.jdisc.config.health-monitor.def) +install_config_definition(src/main/resources/configdefinitions/http-filter.def container.core.http.http-filter.def) +install_config_definition(src/main/resources/configdefinitions/metrics-presentation.def metrics.metrics-presentation.def) +install_config_definition(src/main/resources/configdefinitions/mockservice.def container.handler.test.mockservice.def) +install_config_definition(src/main/resources/configdefinitions/qr-logging.def container.core.qr-logging.def) +install_config_definition(src/main/resources/configdefinitions/qr-searchers.def container.qr-searchers.def) +install_config_definition(src/main/resources/configdefinitions/qr-templates.def container.core.qr-templates.def) +install_config_definition(src/main/resources/configdefinitions/qr.def container.qr.def) +install_config_definition(src/main/resources/configdefinitions/servlet-config.def container.servlet.servlet-config.def) +install_config_definition(src/main/resources/configdefinitions/threadpool.def container.handler.threadpool.def) +install_config_definition(src/main/resources/configdefinitions/vip-status.def container.core.vip-status.def) diff --git a/container-dependencies-enforcer/pom.xml b/container-dependencies-enforcer/pom.xml index 4720135b609..2c608280fbe 100644 --- a/container-dependencies-enforcer/pom.xml +++ b/container-dependencies-enforcer/pom.xml @@ -5,13 +5,24 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> <modelVersion>4.0.0</modelVersion> - <groupId>com.yahoo.vespa</groupId> + <parent> + <groupId>com.yahoo.vespa</groupId> + <artifactId>parent</artifactId> + <version>6-SNAPSHOT</version> + </parent> + <artifactId>container-dependencies-enforcer</artifactId> <version>6-SNAPSHOT</version> <packaging>pom</packaging> <dependencies> <dependency> + <groupId>junit</groupId> + <artifactId>junit</artifactId> + <version>4.11</version> + <scope>test</scope> + </dependency> + <dependency> <groupId>com.yahoo.vespa</groupId> <artifactId>application</artifactId> <version>${project.version}</version> @@ -57,64 +68,67 @@ </excludes> <includes> <include>com.yahoo.vespa</include> - <include>aopalliance:aopalliance:1.0:jar:provided</include> - <include>com.fasterxml.jackson.core:jackson-annotations:2.8.3:jar:provided</include> - <include>com.fasterxml.jackson.core:jackson-core:2.8.3:jar:provided</include> - <include>com.fasterxml.jackson.core:jackson-databind:2.8.3:jar:provided</include> - <include>com.fasterxml.jackson.datatype:jackson-datatype-jdk8:2.8.3:jar:provided</include> - <include>com.fasterxml.jackson.jaxrs:jackson-jaxrs-base:2.5.4:jar:provided</include> - <include>com.fasterxml.jackson.jaxrs:jackson-jaxrs-json-provider:2.5.4:jar:provided</include> - <include>com.fasterxml.jackson.module:jackson-module-jaxb-annotations:2.5.4:jar:provided</include> - <include>com.google.code.findbugs:annotations:1.3.9:jar:provided</include> - <include>com.google.code.findbugs:jsr305:1.3.9:jar:provided</include> - <include>com.google.guava:guava:18.0:jar:provided</include> - <include>com.google.inject.extensions:guice-assistedinject:3.0:jar:provided</include> - <include>com.google.inject.extensions:guice-multibindings:3.0:jar:provided</include> - <include>com.google.inject:guice:3.0:jar:provided:no_aop</include> - <include>commons-codec:commons-codec:1.4:jar:provided</include> - <include>commons-daemon:commons-daemon:1.0.3:jar:provided</include> - <include>commons-logging:commons-logging:1.1.1:jar:provided</include> - <include>javax.annotation:javax.annotation-api:1.2:jar:provided</include> - <include>javax.inject:javax.inject:1:jar:provided</include> - <include>javax.servlet:javax.servlet-api:3.1.0:jar:provided</include> - <include>javax.validation:validation-api:1.1.0.Final:jar:provided</include> - <include>javax.ws.rs:javax.ws.rs-api:2.0.1:jar:provided</include> - <include>net.jcip:jcip-annotations:1.0:jar:provided</include> - <include>net.jpountz.lz4:lz4:1.3.0:jar:provided</include> - <include>org.apache.felix:org.apache.felix.framework:4.2.1:jar:provided</include> - <include>org.apache.felix:org.apache.felix.log:1.0.1:jar:provided</include> - <include>org.apache.felix:org.apache.felix.main:4.2.1:jar:provided</include> - <include>org.apache.httpcomponents:httpclient:4.3.6:jar:provided</include> - <include>org.apache.httpcomponents:httpcore:4.3.3:jar:provided</include> - <include>org.eclipse.jetty:jetty-http:9.4.6.v20170531:jar:provided</include> - <include>org.eclipse.jetty:jetty-io:9.4.6.v20170531:jar:provided</include> - <include>org.eclipse.jetty:jetty-util:9.4.6.v20170531:jar:provided</include> - <include>org.glassfish.hk2.external:aopalliance-repackaged:2.5.0-b05:jar:provided</include> - <include>org.glassfish.hk2.external:javax.inject:2.5.0-b05:jar:provided</include> - <include>org.glassfish.hk2:hk2-api:2.5.0-b05:jar:provided</include> - <include>org.glassfish.hk2:hk2-locator:2.5.0-b05:jar:provided</include> - <include>org.glassfish.hk2:hk2-utils:2.5.0-b05:jar:provided</include> - <include>org.glassfish.hk2:osgi-resource-locator:1.0.1:jar:provided</include> - <include>org.glassfish.jersey.bundles.repackaged:jersey-guava:2.23.2:jar:provided</include> - <include>org.glassfish.jersey.containers:jersey-container-servlet-core:2.23.2:jar:provided</include> - <include>org.glassfish.jersey.containers:jersey-container-servlet:2.23.2:jar:provided</include> - <include>org.glassfish.jersey.core:jersey-client:2.23.2:jar:provided</include> - <include>org.glassfish.jersey.core:jersey-common:2.23.2:jar:provided</include> - <include>org.glassfish.jersey.core:jersey-server:2.23.2:jar:provided</include> - <include>org.glassfish.jersey.ext:jersey-entity-filtering:2.23.2:jar:provided</include> - <include>org.glassfish.jersey.ext:jersey-proxy-client:2.23.2:jar:provided</include> - <include>org.glassfish.jersey.media:jersey-media-jaxb:2.23.2:jar:provided</include> - <include>org.glassfish.jersey.media:jersey-media-json-jackson:2.23.2:jar:provided</include> - <include>org.glassfish.jersey.media:jersey-media-multipart:2.23.2:jar:provided</include> - <include>org.javassist:javassist:3.20.0-GA:jar:provided</include> - <include>org.json:json:20090211:jar:provided</include> - <include>org.jvnet.mimepull:mimepull:1.9.6:jar:provided</include> - <include>org.scala-lang.modules:scala-parser-combinators_2.11:1.0.1:jar:provided</include> - <include>org.slf4j:jcl-over-slf4j:1.7.5:jar:provided</include> - <include>org.slf4j:log4j-over-slf4j:1.7.5:jar:provided</include> - <include>org.slf4j:slf4j-api:1.7.5:jar:provided</include> - <include>org.slf4j:slf4j-jdk14:1.7.5:jar:provided</include> - <include>xml-apis:xml-apis:1.4.01:jar:provided</include> + <include>aopalliance:aopalliance:[1.0]:jar:provided</include> + <include>com.fasterxml.jackson.core:jackson-annotations:[${jackson2.version}]:jar:provided</include> + <include>com.fasterxml.jackson.core:jackson-core:[${jackson2.version}]:jar:provided</include> + <include>com.fasterxml.jackson.core:jackson-databind:[${jackson2.version}]:jar:provided</include> + <include>com.fasterxml.jackson.datatype:jackson-datatype-jdk8:[${jackson2.version}]:jar:provided</include> + + <!-- Use version range for jax deps, because jersey and junit affect the versions. --> + <include>com.fasterxml.jackson.jaxrs:jackson-jaxrs-base:[2.5.4, ${jackson2.version}]:jar:provided</include> + <include>com.fasterxml.jackson.jaxrs:jackson-jaxrs-json-provider:[2.5.4, ${jackson2.version}]:jar:provided</include> + <include>com.fasterxml.jackson.module:jackson-module-jaxb-annotations:[2.5.4, ${jackson2.version}]:jar:provided</include> + + <include>com.google.code.findbugs:annotations:[1.3.9]:jar:provided</include> + <include>com.google.code.findbugs:jsr305:[1.3.9]:jar:provided</include> + <include>com.google.guava:guava:[18.0]:jar:provided</include> + <include>com.google.inject.extensions:guice-assistedinject:[3.0]:jar:provided</include> + <include>com.google.inject.extensions:guice-multibindings:[3.0]:jar:provided</include> + <include>com.google.inject:guice:[3.0]:jar:provided:no_aop</include> + <include>commons-codec:commons-codec:[1.4]:jar:provided</include> + <include>commons-daemon:commons-daemon:[1.0.3]:jar:provided</include> + <include>commons-logging:commons-logging:[1.1.1]:jar:provided</include> + <include>javax.annotation:javax.annotation-api:[1.2]:jar:provided</include> + <include>javax.inject:javax.inject:[1]:jar:provided</include> + <include>javax.servlet:javax.servlet-api:[3.1.0]:jar:provided</include> + <include>javax.validation:validation-api:[1.1.0.Final]:jar:provided</include> + <include>javax.ws.rs:javax.ws.rs-api:[${javax.ws.rs-api.version}]:jar:provided</include> + <include>net.jcip:jcip-annotations:[1.0]:jar:provided</include> + <include>net.jpountz.lz4:lz4:[1.3.0]:jar:provided</include> + <include>org.apache.felix:org.apache.felix.framework:[4.2.1]:jar:provided</include> + <include>org.apache.felix:org.apache.felix.log:[1.0.1]:jar:provided</include> + <include>org.apache.felix:org.apache.felix.main:[4.2.1]:jar:provided</include> + <include>org.apache.httpcomponents:httpclient:[4.3.6]:jar:provided</include> + <include>org.apache.httpcomponents:httpcore:[4.3.3]:jar:provided</include> + <include>org.eclipse.jetty:jetty-http:[${jetty.version}]:jar:provided</include> + <include>org.eclipse.jetty:jetty-io:[${jetty.version}]:jar:provided</include> + <include>org.eclipse.jetty:jetty-util:[${jetty.version}]:jar:provided</include> + <include>org.glassfish.hk2.external:aopalliance-repackaged:[2.5.0-b05]:jar:provided</include> + <include>org.glassfish.hk2.external:javax.inject:[2.5.0-b05]:jar:provided</include> + <include>org.glassfish.hk2:hk2-api:[2.5.0-b05]:jar:provided</include> + <include>org.glassfish.hk2:hk2-locator:[2.5.0-b05]:jar:provided</include> + <include>org.glassfish.hk2:hk2-utils:[2.5.0-b05]:jar:provided</include> + <include>org.glassfish.hk2:osgi-resource-locator:[1.0.1]:jar:provided</include> + <include>org.glassfish.jersey.bundles.repackaged:jersey-guava:[${jersey2.version}]:jar:provided</include> + <include>org.glassfish.jersey.containers:jersey-container-servlet-core:[${jersey2.version}]:jar:provided</include> + <include>org.glassfish.jersey.containers:jersey-container-servlet:[${jersey2.version}]:jar:provided</include> + <include>org.glassfish.jersey.core:jersey-client:[${jersey2.version}]:jar:provided</include> + <include>org.glassfish.jersey.core:jersey-common:[${jersey2.version}]:jar:provided</include> + <include>org.glassfish.jersey.core:jersey-server:[${jersey2.version}]:jar:provided</include> + <include>org.glassfish.jersey.ext:jersey-entity-filtering:[${jersey2.version}]:jar:provided</include> + <include>org.glassfish.jersey.ext:jersey-proxy-client:[${jersey2.version}]:jar:provided</include> + <include>org.glassfish.jersey.media:jersey-media-jaxb:[${jersey2.version}]:jar:provided</include> + <include>org.glassfish.jersey.media:jersey-media-json-jackson:[${jersey2.version}]:jar:provided</include> + <include>org.glassfish.jersey.media:jersey-media-multipart:[${jersey2.version}]:jar:provided</include> + <include>org.javassist:javassist:[3.20.0-GA]:jar:provided</include> + <include>org.json:json:[20090211]:jar:provided</include> + <include>org.jvnet.mimepull:mimepull:[1.9.6]:jar:provided</include> + <include>org.scala-lang.modules:scala-parser-combinators_2.11:[1.0.1]:jar:provided</include> + <include>org.slf4j:jcl-over-slf4j:[1.7.5]:jar:provided</include> + <include>org.slf4j:log4j-over-slf4j:[1.7.5]:jar:provided</include> + <include>org.slf4j:slf4j-api:[1.7.5]:jar:provided</include> + <include>org.slf4j:slf4j-jdk14:[1.7.5]:jar:provided</include> + <include>xml-apis:xml-apis:[1.4.01]:jar:provided</include> </includes> </bannedDependencies> </rules> diff --git a/container-di/CMakeLists.txt b/container-di/CMakeLists.txt new file mode 100644 index 00000000000..c2b033baa92 --- /dev/null +++ b/container-di/CMakeLists.txt @@ -0,0 +1,5 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +install_config_definition(src/main/resources/configdefinitions/bundles.def container.bundles.def) +install_config_definition(src/main/resources/configdefinitions/components.def container.components.def) +install_config_definition(src/main/resources/configdefinitions/jersey-bundles.def container.di.config.jersey-bundles.def) +install_config_definition(src/main/resources/configdefinitions/jersey-injection.def container.di.config.jersey-injection.def) diff --git a/container-disc/CMakeLists.txt b/container-disc/CMakeLists.txt new file mode 100644 index 00000000000..1b661020166 --- /dev/null +++ b/container-disc/CMakeLists.txt @@ -0,0 +1,12 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +install_fat_java_artifact(container-disc) + +vespa_install_script(src/main/sh/vespa-start-container-daemon.sh vespa-start-container-daemon bin) + +install_config_definition(src/main/resources/configdefinitions/container.jdisc.config.http-server.def) +install_config_definition(src/main/resources/configdefinitions/jdisc-bindings.def container.jdisc.jdisc-bindings.def) +install_config_definition(src/main/resources/configdefinitions/jersey-connection.def container.config.jersey.jersey-connection.def) +install_config_definition(src/main/resources/configdefinitions/jersey-init.def container.config.jersey.jersey-init.def) +install_config_definition(src/main/resources/configdefinitions/jersey-web-app-pool.def container.config.jersey.jersey-web-app-pool.def) +install_config_definition(src/main/resources/configdefinitions/metric-defaults.def container.jdisc.config.metric-defaults.def) +install_config_definition(src/main/resources/configdefinitions/score-board.def jdisc.metrics.yamasconsumer.cloud.score-board.def) diff --git a/container-jersey2/CMakeLists.txt b/container-jersey2/CMakeLists.txt new file mode 100644 index 00000000000..d2490563372 --- /dev/null +++ b/container-jersey2/CMakeLists.txt @@ -0,0 +1,2 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +install_fat_java_artifact(container-jersey2) diff --git a/container-messagebus/CMakeLists.txt b/container-messagebus/CMakeLists.txt new file mode 100644 index 00000000000..3cd62d83b73 --- /dev/null +++ b/container-messagebus/CMakeLists.txt @@ -0,0 +1,3 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +install_config_definition(src/main/resources/configdefinitions/container-mbus.def container.jdisc.container-mbus.def) +install_config_definition(src/main/resources/configdefinitions/session.def container.jdisc.config.session.def) diff --git a/container-search-and-docproc/CMakeLists.txt b/container-search-and-docproc/CMakeLists.txt new file mode 100644 index 00000000000..29bbe5bdb0f --- /dev/null +++ b/container-search-and-docproc/CMakeLists.txt @@ -0,0 +1,4 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +install_fat_java_artifact(container-search-and-docproc) + +install_config_definition(src/main/resources/configdefinitions/application-userdata.def container.handler.observability.application-userdata.def) diff --git a/container-search/CMakeLists.txt b/container-search/CMakeLists.txt new file mode 100644 index 00000000000..dcf6c3461d3 --- /dev/null +++ b/container-search/CMakeLists.txt @@ -0,0 +1,28 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +install_config_definition(src/main/resources/configdefinitions/cluster.def search.config.cluster.def) +install_config_definition(src/main/resources/configdefinitions/documentdb-info.def prelude.fastsearch.documentdb-info.def) +install_config_definition(src/main/resources/configdefinitions/emulation.def prelude.emulation.def) +install_config_definition(src/main/resources/configdefinitions/federation.def search.federation.federation.def) +install_config_definition(src/main/resources/configdefinitions/fs4.def container.search.fs4.def) +install_config_definition(src/main/resources/configdefinitions/index-info.def search.config.index-info.def) +install_config_definition(src/main/resources/configdefinitions/keyvalue.def prelude.searcher.keyvalue.def) +install_config_definition(src/main/resources/configdefinitions/legacy-emulation.def container.search.legacy-emulation.def) +install_config_definition(src/main/resources/configdefinitions/lowercasing.def search.querytransform.lowercasing.def) +install_config_definition(src/main/resources/configdefinitions/measure-qps.def search.statistics.measure-qps.def) +install_config_definition(src/main/resources/configdefinitions/page-templates.def search.pagetemplates.page-templates.def) +install_config_definition(src/main/resources/configdefinitions/provider.def search.federation.provider.def) +install_config_definition(src/main/resources/configdefinitions/qr-binary-cache-region.def search.cache.qr-binary-cache-region.def) +install_config_definition(src/main/resources/configdefinitions/qr-binary-cache.def search.cache.qr-binary-cache.def) +install_config_definition(src/main/resources/configdefinitions/qr-monitor.def prelude.cluster.qr-monitor.def) +install_config_definition(src/main/resources/configdefinitions/qr-quotetable.def prelude.searcher.qr-quotetable.def) +install_config_definition(src/main/resources/configdefinitions/qr-start.def search.config.qr-start.def) +install_config_definition(src/main/resources/configdefinitions/query-profiles.def search.query.profile.config.query-profiles.def) +install_config_definition(src/main/resources/configdefinitions/rate-limiting.def search.config.rate-limiting.def) +install_config_definition(src/main/resources/configdefinitions/resolvers.def search.pagetemplates.resolvers.def) +install_config_definition(src/main/resources/configdefinitions/rewrites.def search.query.rewrite.rewrites.def) +install_config_definition(src/main/resources/configdefinitions/search-nodes.def search.config.dispatchprototype.search-nodes.def) +install_config_definition(src/main/resources/configdefinitions/search-with-renderer-handler.def search.handler.search-with-renderer-handler.def) +install_config_definition(src/main/resources/configdefinitions/searchchain-forward.def search.federation.searchchain-forward.def) +install_config_definition(src/main/resources/configdefinitions/semantic-rules.def prelude.semantics.semantic-rules.def) +install_config_definition(src/main/resources/configdefinitions/strict-contracts.def search.federation.strict-contracts.def) +install_config_definition(src/main/resources/configdefinitions/timing-searcher.def search.statistics.timing-searcher.def) diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/MetricsService.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/MetricsService.java index 5547f94c8cf..1ceb064ad44 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/MetricsService.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/MetricsService.java @@ -4,7 +4,7 @@ package com.yahoo.vespa.hosted.controller.api.integration; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Zone; -import java.util.List; +import java.util.Map; /** * A service which returns metric values on request @@ -17,7 +17,7 @@ public interface MetricsService { DeploymentMetrics getDeploymentMetrics(ApplicationId application, Zone zone); - List<ClusterCostMetrics> getClusterCostMetrics(ApplicationId application, Zone zone); + Map<String, SystemMetrics> getSystemMetrics(ApplicationId application, Zone zone); class DeploymentMetrics { @@ -67,22 +67,30 @@ public interface MetricsService { } - class CostMetrics { + class SystemMetrics { private final double cpuUtil; private final double memUtil; private final double diskUtil; - public CostMetrics(double cpuUtil, double memUtil, double diskUtil) { + /** + * @param cpuUtil percentage of system cpu utilization + * @param memUtil percentage of system memory utilization + * @param diskUtil percentage of system disk utilization + */ + public SystemMetrics(double cpuUtil, double memUtil, double diskUtil) { this.cpuUtil = cpuUtil; this.memUtil = memUtil; this.diskUtil = diskUtil; } + /** @return the percentage of cpu utilization **/ public double cpuUtil() { return cpuUtil; } + /** @return the percentage of memory utilization **/ public double memUtil() { return memUtil; } + /** @return the percentage of disk utilization **/ public double diskUtil() { return diskUtil; } public static class Builder { @@ -102,25 +110,9 @@ public interface MetricsService { this.diskUtil = diskUtil; } - public CostMetrics build() { return new CostMetrics(cpuUtil, memUtil, diskUtil); } + public SystemMetrics build() { return new SystemMetrics(cpuUtil, memUtil, diskUtil); } } } - class ClusterCostMetrics { - - private final String clusterId; - private final CostMetrics costMetrics; - - public ClusterCostMetrics(String clusterId, CostMetrics costMetrics) { - this.clusterId = clusterId; - this.costMetrics = costMetrics; - } - - public String clusterId() { return clusterId; } - - public CostMetrics costMetrics() { return costMetrics; } - - } - } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java index ccfcc0e4b0d..2ce2e480fc5 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java @@ -278,7 +278,7 @@ public class ApplicationController { Version version; if (options.deployCurrentVersion) version = application.currentVersion(controller, zone); - else if (application.deploymentJobs().isSelfTriggering()) // legacy mode: let the client decide + else if (canDeployDirectlyTo(zone, options)) version = options.vespaVersion.map(Version::new).orElse(controller.systemVersion()); else if ( ! application.deploying().isPresent() && ! zone.environment().isManuallyDeployed()) return unexpectedDeployment(applicationId, zone, applicationPackage); @@ -286,10 +286,8 @@ public class ApplicationController { version = application.currentDeployVersion(controller, zone); // Ensure that the deploying change is tested - // FIXME: For now only for non-self-triggering applications - VESPA-8418 - if ( ! application.deploymentJobs().isSelfTriggering() && - ! zone.environment().isManuallyDeployed() && - ! application.deploymentJobs().isDeployableTo(zone.environment(), application.deploying())) + if (! canDeployDirectlyTo(zone, options) && + ! application.deploymentJobs().isDeployableTo(zone.environment(), application.deploying())) throw new IllegalArgumentException("Rejecting deployment of " + application + " to " + zone + " as pending " + application.deploying().get() + " is untested"); @@ -305,9 +303,8 @@ public class ApplicationController { application = application.withProjectId(options.screwdriverBuildJob.get().screwdriverId.value()); if (application.deploying().isPresent() && application.deploying().get() instanceof Change.ApplicationChange) application = application.withDeploying(Optional.of(Change.ApplicationChange.of(revision))); - if ( ! triggeredWith(revision, application, jobType) && !zone.environment().isManuallyDeployed() && jobType != null) { + if ( ! triggeredWith(revision, application, jobType) && !canDeployDirectlyTo(zone, options) && jobType != null) { // Triggering information is used to store which changes were made or attempted - // - For self-triggered applications we don't have any trigger information, so we add it here. // - For all applications, we don't have complete control over which revision is actually built, // so we update it here with what we actually triggered if necessary application = application.with(application.deploymentJobs() @@ -590,6 +587,13 @@ public class ApplicationController { return curator.lock(application, Duration.ofMinutes(10)); } + /** Returns whether a direct deployment to given zone is allowed */ + private static boolean canDeployDirectlyTo(Zone zone, DeployOptions options) { + return !options.screwdriverBuildJob.isPresent() || + options.screwdriverBuildJob.get().screwdriverId == null || + zone.environment().isManuallyDeployed(); + } + private static final class ApplicationRotation { private final ImmutableSet<String> cnames; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentJobs.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentJobs.java index b4ca00243b7..af8617cbf05 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentJobs.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentJobs.java @@ -31,25 +31,23 @@ public class DeploymentJobs { private final Optional<Long> projectId; private final ImmutableMap<JobType, JobStatus> status; private final Optional<String> jiraIssueId; - private final boolean selfTriggering; // TODO: Remove this when no projects are self-triggering. /** Creates an empty set of deployment jobs */ public DeploymentJobs(long projectId) { - this(Optional.of(projectId), ImmutableMap.of(), Optional.empty(),true); + this(Optional.of(projectId), ImmutableMap.of(), Optional.empty()); } - public DeploymentJobs(Optional<Long> projectId, Collection<JobStatus> jobStatusEntries, Optional<String> jiraIssueId, boolean selfTriggering) { - this(projectId, asMap(jobStatusEntries), jiraIssueId, selfTriggering); + public DeploymentJobs(Optional<Long> projectId, Collection<JobStatus> jobStatusEntries, Optional<String> jiraIssueId) { + this(projectId, asMap(jobStatusEntries), jiraIssueId); } - private DeploymentJobs(Optional<Long> projectId, Map<JobType, JobStatus> status, Optional<String> jiraIssueId, boolean selfTriggering) { + private DeploymentJobs(Optional<Long> projectId, Map<JobType, JobStatus> status, Optional<String> jiraIssueId) { Objects.requireNonNull(projectId, "projectId cannot be null"); Objects.requireNonNull(status, "status cannot be null"); Objects.requireNonNull(jiraIssueId, "jiraIssueId cannot be null"); this.projectId = projectId; this.status = ImmutableMap.copyOf(status); this.jiraIssueId = jiraIssueId; - this.selfTriggering = selfTriggering; } private static Map<JobType, JobStatus> asMap(Collection<JobStatus> jobStatusEntries) { @@ -66,7 +64,7 @@ public class DeploymentJobs { if (job == null) job = JobStatus.initial(report.jobType()); return job.withCompletion(report.jobError(), notificationTime, controller); }); - return new DeploymentJobs(Optional.of(report.projectId()), status, jiraIssueId, report.selfTriggering()); + return new DeploymentJobs(Optional.of(report.projectId()), status, jiraIssueId); } public DeploymentJobs withTriggering(JobType jobType, @@ -81,33 +79,26 @@ public class DeploymentJobs { change.isPresent() && change.get() instanceof Change.VersionChange, triggerTime); }); - return new DeploymentJobs(projectId, status, jiraIssueId, selfTriggering); + return new DeploymentJobs(projectId, status, jiraIssueId); } public DeploymentJobs withProjectId(long projectId) { - return new DeploymentJobs(Optional.of(projectId), status, jiraIssueId, selfTriggering); + return new DeploymentJobs(Optional.of(projectId), status, jiraIssueId); } public DeploymentJobs withJiraIssueId(Optional<String> jiraIssueId) { - return new DeploymentJobs(projectId, status, jiraIssueId, selfTriggering); + return new DeploymentJobs(projectId, status, jiraIssueId); } public DeploymentJobs without(JobType job) { Map<JobType, JobStatus> status = new HashMap<>(this.status); status.remove(job); - return new DeploymentJobs(projectId, status, jiraIssueId, selfTriggering); - } - - public DeploymentJobs asSelfTriggering(boolean selfTriggering) { - return new DeploymentJobs(projectId, status, jiraIssueId, selfTriggering); + return new DeploymentJobs(projectId, status, jiraIssueId); } /** Returns an immutable map of the status entries in this */ public Map<JobType, JobStatus> jobStatus() { return status; } - /** Returns whether this application's deployment jobs trigger each other, and should be left alone, or not. */ - public boolean isSelfTriggering() { return selfTriggering; } - /** Returns whether this has some job status which is not a success */ public boolean hasFailures() { return status.values().stream().anyMatch(jobStatus -> ! jobStatus.isSuccess()); @@ -281,19 +272,18 @@ public class DeploymentJobs { private final long projectId; private final long buildNumber; private final Optional<JobError> jobError; - private final boolean selfTriggering; public JobReport(ApplicationId applicationId, JobType jobType, long projectId, long buildNumber, - Optional<JobError> jobError, boolean selfTriggering) { + Optional<JobError> jobError) { Objects.requireNonNull(applicationId, "applicationId cannot be null"); Objects.requireNonNull(jobType, "jobType cannot be null"); Objects.requireNonNull(jobError, "jobError cannot be null"); + this.applicationId = applicationId; this.projectId = projectId; this.buildNumber = buildNumber; this.jobType = jobType; this.jobError = jobError; - this.selfTriggering = selfTriggering; } public ApplicationId applicationId() { return applicationId; } @@ -302,7 +292,6 @@ public class DeploymentJobs { public long buildNumber() { return buildNumber; } public boolean success() { return !jobError.isPresent(); } public Optional<JobError> jobError() { return jobError; } - public boolean selfTriggering() { return selfTriggering; } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java index c43225fbd2b..268965850ba 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java @@ -287,8 +287,8 @@ public class DeploymentTrigger { return application; } - if (application.deploymentJobs().isSelfTriggering()) { - log.info("Not triggering " + jobType + " for self-triggering " + application); + // Ignore applications that are not associated with a project + if (!application.deploymentJobs().projectId().isPresent()) { return application; } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java index 607ad4fd9f0..14fce0987b7 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java @@ -61,7 +61,6 @@ public class ApplicationSerializer { private final String projectIdField = "projectId"; private final String jobStatusField = "jobStatus"; private final String jiraIssueIdField = "jiraIssueId"; - private final String selfTriggeringField = "selfTriggering"; // JobStatus field private final String jobTypeField = "jobType"; @@ -125,7 +124,6 @@ public class ApplicationSerializer { deploymentJobs.projectId().ifPresent(projectId -> cursor.setLong(projectIdField, projectId)); jobStatusToSlime(deploymentJobs.jobStatus().values(), cursor.setArray(jobStatusField)); deploymentJobs.jiraIssueId().ifPresent(jiraIssueId -> cursor.setString(jiraIssueIdField, jiraIssueId)); - cursor.setBool(selfTriggeringField, deploymentJobs.isSelfTriggering()); } private void jobStatusToSlime(Collection<JobStatus> jobStatuses, Cursor jobStatusArray) { @@ -218,9 +216,8 @@ public class ApplicationSerializer { Optional<Long> projectId = optionalLong(object.field(projectIdField)); List<JobStatus> jobStatusList = jobStatusListFromSlime(object.field(jobStatusField)); Optional<String> jiraIssueKey = optionalString(object.field(jiraIssueIdField)); - boolean selfTriggering = object.field(selfTriggeringField).asBool(); - return new DeploymentJobs(projectId, jobStatusList, jiraIssueKey, selfTriggering); + return new DeploymentJobs(projectId, jobStatusList, jiraIssueKey); } private Optional<Change> changeFromSlime(Inspector object) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java index 9d6aa7d3632..7aef1e413aa 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java @@ -194,6 +194,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler { Path path = new Path(request.getUri().getPath()); if (path.matches("/application/v4/tenant/{tenant}")) return deleteTenant(path.get("tenant"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}")) return deleteApplication(path.get("tenant"), path.get("application"), request); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/deploying")) return cancelDeploy(path.get("tenant"), path.get("application")); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}")) return deactivate(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region")); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/global-rotation/override")) return setGlobalRotationOverride(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), true, request); @@ -679,7 +680,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler { Application application = controller.applications().require(id); if (application.deploying().isPresent()) throw new IllegalArgumentException("Can not start a deployment of " + application + " at this time: " + - application.deploying() + " is in progress"); + application.deploying().get() + " is in progress"); Version version = decideDeployVersion(request); if ( ! systemHasVersion(version)) @@ -687,14 +688,26 @@ public class ApplicationApiHandler extends LoggingRequestHandler { "Version is not active in this system. " + "Active versions: " + controller.versionStatus().versions()); - // Since we manually triggered it we don't want this to be self-triggering for the time being - controller.applications().store(application.with(application.deploymentJobs().asSelfTriggering(false)), lock); - controller.applications().deploymentTrigger().triggerChange(application.id(), new Change.VersionChange(version)); return new MessageResponse("Triggered deployment of " + application + " on version " + version); } } - + + /** Cancel any ongoing change for given application */ + private HttpResponse cancelDeploy(String tenantName, String applicationName) { + ApplicationId id = ApplicationId.from(tenantName, applicationName, "default"); + try (Lock lock = controller.applications().lock(id)) { + Application application = controller.applications().require(id); + Optional<Change> change = application.deploying(); + if (!change.isPresent()) { + return new MessageResponse("No deployment in progress for " + application + " at this time"); + } + controller.applications().deploymentTrigger().cancelChange(id); + return new MessageResponse("Cancelled " + change.get() + " for " + application); + } + } + + /** Schedule restart of deployment, or specific host in a deployment */ private HttpResponse restart(String tenantName, String applicationName, String instanceName, String environment, String region, HttpRequest request) { DeploymentId deploymentId = new DeploymentId(ApplicationId.from(tenantName, applicationName, instanceName), new Zone(Environment.from(environment), RegionName.from(region))); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/screwdriver/ScrewdriverApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/screwdriver/ScrewdriverApiHandler.java index 3cc4d8c7f5c..a02bb6b373e 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/screwdriver/ScrewdriverApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/screwdriver/ScrewdriverApiHandler.java @@ -103,23 +103,6 @@ public class ScrewdriverApiHandler extends LoggingRequestHandler { return new SlimeJsonResponse(slime); } - /** - * Parse a JSON blob of the form: - * { - * "tenant" : String - * "application" : String - * "instance" : String - * "jobName" : String - * "projectId" : long - * "success" : boolean - * "selfTriggering": boolean - * "vespaVersion" : String - * } - * and notify the controller of the report. - * - * @param request The JSON blob. - * @return 200 - */ private HttpResponse handleJobReportPost(HttpRequest request) { controller.applications().notifyJobCompletion(toJobReport(toSlime(request.getData()).get())); return new StringResponse("ok"); @@ -147,8 +130,7 @@ public class ScrewdriverApiHandler extends LoggingRequestHandler { JobType.fromId(report.field("jobName").asString()), report.field("projectId").asLong(), report.field("buildNumber").asLong(), - jobError, - report.field("selfTriggering").asBool() + jobError ); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ConfigServerClientMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ConfigServerClientMock.java index 6018c99206e..13525969358 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ConfigServerClientMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ConfigServerClientMock.java @@ -51,8 +51,17 @@ public class ConfigServerClientMock extends AbstractComponent implements ConfigS /** The exception to throw on the next prepare run, or null to continue normally */ private RuntimeException prepareException = null; - /** The version given in the previous prepare call, or null if no call has been made */ - public Optional<Version> lastPrepareVersion = null; + private Optional<Version> lastPrepareVersion = Optional.empty(); + + /** The version given in the previous prepare call, or empty if no call has been made */ + public Optional<Version> lastPrepareVersion() { + return lastPrepareVersion; + } + + /** Return map of applications that may have been activated */ + public Map<ApplicationId, Boolean> activated() { + return Collections.unmodifiableMap(applicationActivated); + } @Override public PreparedApplication prepare(DeploymentId deployment, DeployOptions deployOptions, Set<String> rotationCnames, Set<Rotation> rotations, byte[] content) { @@ -201,4 +210,5 @@ public class ConfigServerClientMock extends AbstractComponent implements ConfigS ? endpoints.get(endpoint) : result; } + } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java index 91fca0d37d1..36b09708399 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java @@ -17,16 +17,10 @@ import com.yahoo.vespa.curator.Lock; import com.yahoo.vespa.hosted.controller.api.Tenant; import com.yahoo.vespa.hosted.controller.api.application.v4.model.DeployOptions; import com.yahoo.vespa.hosted.controller.api.application.v4.model.EndpointStatus; -import com.yahoo.vespa.hosted.controller.api.application.v4.model.GitRevision; -import com.yahoo.vespa.hosted.controller.api.application.v4.model.ScrewdriverBuildJob; import com.yahoo.vespa.hosted.controller.api.identifiers.AthensDomain; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; -import com.yahoo.vespa.hosted.controller.api.identifiers.GitBranch; -import com.yahoo.vespa.hosted.controller.api.identifiers.GitCommit; -import com.yahoo.vespa.hosted.controller.api.identifiers.GitRepository; import com.yahoo.vespa.hosted.controller.api.identifiers.Property; import com.yahoo.vespa.hosted.controller.api.identifiers.PropertyId; -import com.yahoo.vespa.hosted.controller.api.identifiers.ScrewdriverId; import com.yahoo.vespa.hosted.controller.api.identifiers.TenantId; import com.yahoo.vespa.hosted.controller.api.identifiers.UserGroup; import com.yahoo.vespa.hosted.controller.api.integration.BuildService.BuildJob; @@ -101,7 +95,7 @@ public class ControllerTest { // staging job - succeeding Version version1 = Version.fromString("6.1"); // Set in config server mock Application app1 = tester.createApplication("app1", "tenant1", 1, 11L); - applications.notifyJobCompletion(mockReport(app1, component, true, false)); + applications.notifyJobCompletion(mockReport(app1, component, true)); assertFalse("Revision is currently not known", ((Change.ApplicationChange)tester.controller().applications().require(app1.id()).deploying().get()).revision().isPresent()); tester.deployAndNotify(app1, applicationPackage, true, systemTest); @@ -143,7 +137,7 @@ public class ControllerTest { tester.clock().advance(Duration.ofSeconds(1)); // system and staging test job - succeeding - applications.notifyJobCompletion(mockReport(app1, component, true, false)); + applications.notifyJobCompletion(mockReport(app1, component, true)); tester.deployAndNotify(app1, applicationPackage, true, systemTest); assertStatus(JobStatus.initial(systemTest) .withTriggering(version1, revision, false, tester.clock().instant()) @@ -170,7 +164,7 @@ public class ControllerTest { .environment(Environment.prod) .region("us-east-3") .build(); - applications.notifyJobCompletion(mockReport(app1, component, true, false)); + applications.notifyJobCompletion(mockReport(app1, component, true)); try { tester.deploy(systemTest, app1, applicationPackage); fail("Expected exception due to unallowed production deployment removal"); @@ -207,17 +201,16 @@ public class ControllerTest { Version systemVersion = tester.controller().versionStatus().systemVersion().get().versionNumber(); Application app1 = tester.createApplication("application1", "tenant1", 1, 1L); - applications.store(app1.with(app1.deploymentJobs().asSelfTriggering(false)), applications.lock(app1.id())); // First deployment: An application change - applications.notifyJobCompletion(mockReport(app1, component, true, false)); + applications.notifyJobCompletion(mockReport(app1, component, true)); tester.deployAndNotify(app1, applicationPackage, true, systemTest); tester.deployAndNotify(app1, applicationPackage, true, stagingTest); tester.deployAndNotify(app1, applicationPackage, true, productionUsWest1); app1 = applications.require(app1.id()); assertEquals("First deployment gets system version", systemVersion, app1.deployedVersion().get()); - assertEquals(systemVersion, tester.configServer().lastPrepareVersion.get()); + assertEquals(systemVersion, tester.configServer().lastPrepareVersion().get()); // Unexpected deployment tester.deploy(productionUsWest1, app1, applicationPackage); @@ -233,14 +226,14 @@ public class ControllerTest { .region("us-west-1") .region("us-east-3") .build(); - applications.notifyJobCompletion(mockReport(app1, component, true, false)); + applications.notifyJobCompletion(mockReport(app1, component, true)); tester.deployAndNotify(app1, applicationPackage, true, systemTest); tester.deployAndNotify(app1, applicationPackage, true, stagingTest); tester.deployAndNotify(app1, applicationPackage, true, productionUsWest1); app1 = applications.require(app1.id()); assertEquals("Application change preserves version", systemVersion, app1.deployedVersion().get()); - assertEquals(systemVersion, tester.configServer().lastPrepareVersion.get()); + assertEquals(systemVersion, tester.configServer().lastPrepareVersion().get()); // A deployment to the new region gets the same version applicationPackage = new ApplicationPackageBuilder() @@ -251,7 +244,7 @@ public class ControllerTest { tester.deployAndNotify(app1, applicationPackage, true, productionUsEast3); app1 = applications.require(app1.id()); assertEquals("Application change preserves version", systemVersion, app1.deployedVersion().get()); - assertEquals(systemVersion, tester.configServer().lastPrepareVersion.get()); + assertEquals(systemVersion, tester.configServer().lastPrepareVersion().get()); // Version upgrade changes system version Change.VersionChange change = new Change.VersionChange(newSystemVersion); @@ -263,7 +256,7 @@ public class ControllerTest { app1 = applications.require(app1.id()); assertEquals("Version upgrade changes version", newSystemVersion, app1.deployedVersion().get()); - assertEquals(newSystemVersion, tester.configServer().lastPrepareVersion.get()); + assertEquals(newSystemVersion, tester.configServer().lastPrepareVersion().get()); } /** Adds a new version, higher than the current system version, makes it the system version and returns it */ @@ -404,23 +397,6 @@ public class ControllerTest { } @Test - public void selfTriggeringApplicationIsNotTriggered() { - ControllerTester tester = new ControllerTester(); - ApplicationController applications = tester.controller().applications(); - - // Create application and report completion from component job - long projectId = 1; - TenantId tenant = tester.createTenant("tenant", "domain", 1L); - Application application = tester.createApplication(tenant, "application", "default", projectId); - applications.notifyJobCompletion(mockReport(application, component, true, true)); - - // Only component completion status is persisted and no further jobs are triggered - assertEquals(1, applications.get(application.id()).get().deploymentJobs().jobStatus().size()); - assertStatus(JobStatus.initial(component).withCompletion(Optional.empty(), tester.clock().instant(), tester.controller()), - application.id(), tester.controller()); - } - - @Test public void requeueOutOfCapacityStagingJob() { DeploymentTester tester = new DeploymentTester(); @@ -491,19 +467,18 @@ public class ControllerTest { assertEquals(expectedStatus, existingStatus); } - private JobReport mockReport(Application application, JobType jobType, Optional<JobError> jobError, boolean selfTriggering) { + private JobReport mockReport(Application application, JobType jobType, Optional<JobError> jobError) { return new JobReport( application.id(), jobType, application.deploymentJobs().projectId().get(), 42, - jobError, - selfTriggering + jobError ); } - private JobReport mockReport(Application application, JobType jobType, boolean success, boolean selfTriggering) { - return mockReport(application, jobType, JobError.from(success), selfTriggering); + private JobReport mockReport(Application application, JobType jobType, boolean success) { + return mockReport(application, jobType, JobError.from(success)); } @Test @@ -533,51 +508,12 @@ public class ControllerTest { } @Test - public void testLegacyDeployments() { - // Setup system - DeploymentTester tester = new DeploymentTester(); - ApplicationController applications = tester.controller().applications(); - ApplicationPackage applicationPackage = new ApplicationPackageBuilder() - .environment(Environment.prod) - .region("us-east-3") - .build(); - Version systemVersion = tester.controller().versionStatus().systemVersion().get().versionNumber(); - - Application app1 = tester.createApplication("application1", "tenant1", 1, 1L); - applications.store(app1.with(app1.deploymentJobs().asSelfTriggering(true)), applications.lock(app1.id())); - - // Scenario: App already on 6.0, Upgrade to 6.1 (systemversion) - Zone prodZone = new Zone(Environment.prod, RegionName.from("us-east-3")); - Zone stagingZone = new Zone(Environment.staging, RegionName.from("us-east-3")); - Version existingVersion = Version.fromString("6.0"); - - // Add deployment on existing version - legacyDeploy(tester.controller(), app1, applicationPackage, prodZone, Optional.of(existingVersion), false); - - // Add dev/perf deployment on old version to verify that this does not affect Initialize staging step. VESPA-8469 - Version devVersion = Version.fromString("5.0"); - legacyDeploy(tester.controller(), app1, applicationPackage, new Zone(Environment.dev, RegionName.from("us-east-1")), Optional.of(devVersion), false); - legacyDeploy(tester.controller(), app1, applicationPackage, new Zone(Environment.perf, RegionName.from("us-east-3")), Optional.of(devVersion), false); - - // Initialize staging on existing version - legacyDeploy(tester.controller(), app1, applicationPackage, stagingZone, Optional.of(systemVersion), true); - app1 = applications.require(app1.id()); - assertEquals(existingVersion, app1.currentDeployVersion(tester.controller(), stagingZone)); - - // Upgrade to the new version in staging - legacyDeploy(tester.controller(), app1, applicationPackage, stagingZone, Optional.of(systemVersion), false); - app1 = applications.require(app1.id()); - assertEquals(systemVersion, app1.currentDeployVersion(tester.controller(), stagingZone)); - } - - @Test public void testDeployUntestedChangeFails() { ControllerTester tester = new ControllerTester(); ApplicationController applications = tester.controller().applications();TenantId tenant = tester.createTenant("tenant1", "domain1", 11L); Application app = tester.createApplication(tenant, "app1", "default", 1); - app = app.withDeploying(Optional.of(new Change.VersionChange(Version.fromString("6.3")))) - .with(app.deploymentJobs().asSelfTriggering(false)); + app = app.withDeploying(Optional.of(new Change.VersionChange(Version.fromString("6.3")))); applications.store(app, applications.lock(app.id())); try { tester.deploy(app, new Zone(Environment.prod, RegionName.from("us-east-3"))); @@ -587,16 +523,6 @@ public class ControllerTest { } } - private void legacyDeploy(Controller controller, Application application, ApplicationPackage applicationPackage, Zone zone, Optional<Version> version, boolean deployCurrentVersion) { - ScrewdriverId app1ScrewdriverId = new ScrewdriverId(String.valueOf(application.deploymentJobs().projectId().get())); - GitRevision app1RevisionId = new GitRevision(new GitRepository("repo"), new GitBranch("master"), new GitCommit("commit1")); - controller.applications().deployApplication(application.id(), - zone, - applicationPackage, - new DeployOptions(Optional.of(new ScrewdriverBuildJob(app1ScrewdriverId, app1RevisionId)), version, false, deployCurrentVersion)); - - } - @Test public void testCleanupOfStaleDeploymentData() throws IOException { DeploymentTester tester = new DeploymentTester(); @@ -679,4 +605,31 @@ public class ControllerTest { assertEquals("fake-global-rotation-tenant1.app1", record.get().value()); } + @Test + public void testDeployWithoutProjectId() { + DeploymentTester tester = new DeploymentTester(); + tester.controllerTester().zoneRegistry().setSystem(SystemName.cd); + ApplicationPackage applicationPackage = new ApplicationPackageBuilder() + .environment(Environment.prod) + .region("cd-us-central-1") + .build(); + + // Create application + Application app = tester.createApplication("app1", "tenant1", 1, 2L); + + // Direct deploy is allowed when project ID is missing + Zone zone = new Zone(Environment.prod, RegionName.from("cd-us-central-1")); + // Same options as used in our integration tests + DeployOptions options = new DeployOptions(Optional.empty(), Optional.empty(), false, + false); + tester.controller().applications().deployApplication(app.id(), zone, applicationPackage, options); + + assertTrue("Application deployed and activated", + tester.controllerTester().configServer().activated().getOrDefault(app.id(), false)); + + assertTrue("No job status added", + tester.applications().require(app.id()).deploymentJobs().jobStatus().isEmpty()); + + } + } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java index 44309b43a5f..fb8b9df40ba 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java @@ -214,8 +214,7 @@ public class DeploymentTester { jobType, application.deploymentJobs().projectId().get(), 42, - jobError, - false + jobError ); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/MockBuildService.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/MockBuildService.java index b9e66b354de..0293ea08d65 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/MockBuildService.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/MockBuildService.java @@ -161,8 +161,7 @@ public class MockBuildService implements BuildService { jobType, projectId, 42, - JobError.from(success), - false + JobError.from(success) )); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/PolledBuildSystemTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/PolledBuildSystemTest.java index 779af370ff4..c869bd90924 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/PolledBuildSystemTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/PolledBuildSystemTest.java @@ -2,12 +2,9 @@ package com.yahoo.vespa.hosted.controller.deployment; import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.Environment; -import com.yahoo.vespa.curator.mock.MockCurator; -import com.yahoo.vespa.hosted.controller.ControllerTester; import com.yahoo.vespa.hosted.controller.api.integration.BuildService.BuildJob; +import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; import com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType; -import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; import com.yahoo.vespa.hosted.controller.persistence.MockCuratorDb; import org.junit.Test; import org.junit.runner.RunWith; @@ -37,15 +34,16 @@ public class PolledBuildSystemTest { @Test public void throttle_capacity_constrained_jobs() { - ControllerTester tester = new ControllerTester(); + DeploymentTester tester = new DeploymentTester(); BuildSystem buildSystem = new PolledBuildSystem(tester.controller(), new MockCuratorDb()); - long fooProjectId = 1; - long barProjectId = 2; - ApplicationId foo = tester.createAndDeploy("tenant1", "domain1", "app1", - Environment.prod, fooProjectId).id(); - ApplicationId bar = tester.createAndDeploy("tenant2", "domain2", "app2", - Environment.prod, barProjectId).id(); + int fooProjectId = 1; + int barProjectId = 2; + ApplicationPackage applicationPackage = new ApplicationPackageBuilder() + .region("us-west-1") + .build(); + ApplicationId foo = tester.createAndDeploy("app1", fooProjectId, applicationPackage).id(); + ApplicationId bar = tester.createAndDeploy("app2", barProjectId, applicationPackage).id(); // Trigger jobs in capacity constrained environment buildSystem.addJob(foo, jobType, false); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/MockMetricsService.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/MockMetricsService.java index 79b4c5f6d6a..b30d90c10f4 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/MockMetricsService.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/MockMetricsService.java @@ -5,7 +5,7 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Zone; import java.util.Collections; -import java.util.List; +import java.util.Map; /** * @author bratseth @@ -23,10 +23,8 @@ public class MockMetricsService implements com.yahoo.vespa.hosted.controller.api } @Override - public List<ClusterCostMetrics> getClusterCostMetrics(ApplicationId application, Zone zone) { - CostMetrics costMetrics = new CostMetrics(55.54, 69.90, 34.59); - ClusterCostMetrics clusterCostMetrics = new ClusterCostMetrics("default", costMetrics); - return Collections.singletonList(clusterCostMetrics); + public Map<String, SystemMetrics> getSystemMetrics(ApplicationId application, Zone zone) { + return Collections.emptyMap(); } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentExpirerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentExpirerTest.java index ee4f3631b54..ef0b05f9bb2 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentExpirerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentExpirerTest.java @@ -1,16 +1,22 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.maintenance; -import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.Zone; -import com.yahoo.vespa.hosted.controller.ControllerTester; +import com.yahoo.vespa.hosted.controller.Application; +import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; +import com.yahoo.vespa.hosted.controller.application.Deployment; +import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; +import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; import com.yahoo.vespa.hosted.controller.persistence.MockCuratorDb; +import org.junit.Before; import org.junit.Test; import java.io.IOException; import java.time.Duration; +import java.util.List; +import java.util.stream.Collectors; import static org.junit.Assert.assertEquals; @@ -19,28 +25,53 @@ import static org.junit.Assert.assertEquals; */ public class DeploymentExpirerTest { + private DeploymentTester tester; + + @Before + public void before() { + tester = new DeploymentTester(); + } + @Test public void testDeploymentExpiry() throws IOException, InterruptedException { - ControllerTester tester = new ControllerTester(); - tester.zoneRegistry().setDeploymentTimeToLive(new Zone(Environment.dev, RegionName.from("us-east-1")), Duration.ofDays(14)); + tester.controllerTester().zoneRegistry().setDeploymentTimeToLive( + new Zone(Environment.dev, RegionName.from("us-east-1")), + Duration.ofDays(14) + ); DeploymentExpirer expirer = new DeploymentExpirer(tester.controller(), Duration.ofDays(10), tester.clock(), new JobControl(new MockCuratorDb())); - ApplicationId devApp = tester.createAndDeploy("tenant1", "domain1", "app1", Environment.dev, 123).id(); - ApplicationId prodApp = tester.createAndDeploy("tenant2", "domain2", "app2", Environment.prod, 456).id(); + Application devApp = tester.createApplication("app1", "tenant1", 123L, 1L); + Application prodApp = tester.createApplication("app2", "tenant2", 456L, 2L); + + // Deploy dev + tester.controllerTester().deploy(devApp, tester.controllerTester().toZone(Environment.dev)); - assertEquals(1, tester.controller().applications().get(devApp).get().deployments().size()); - assertEquals(1, tester.controller().applications().get(prodApp).get().deployments().size()); + // Deploy prod + ApplicationPackage prodAppPackage = new ApplicationPackageBuilder() + .region("us-west-1") + .build(); + tester.deployCompletely(prodApp, prodAppPackage); + + assertEquals(1, permanentDeployments(devApp).size()); + assertEquals(1, permanentDeployments(prodApp).size()); // Not expired at first expirer.maintain(); - assertEquals(1, tester.controller().applications().get(devApp).get().deployments().size()); - assertEquals(1, tester.controller().applications().get(prodApp).get().deployments().size()); + assertEquals(1, permanentDeployments(devApp).size()); + assertEquals(1, permanentDeployments(prodApp).size()); // The dev application is removed tester.clock().advance(Duration.ofDays(15)); expirer.maintain(); - assertEquals(0, tester.controller().applications().get(devApp).get().deployments().size()); - assertEquals(1, tester.controller().applications().get(prodApp).get().deployments().size()); + assertEquals(0, permanentDeployments(devApp).size()); + assertEquals(1, permanentDeployments(prodApp).size()); + } + + private List<Deployment> permanentDeployments(Application application) { + return tester.controller().applications().get(application.id()).get().deployments().values().stream() + .filter(deployment -> deployment.zone().environment() != Environment.test && + deployment.zone().environment() != Environment.staging) + .collect(Collectors.toList()); } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java index 8584a62383b..5be030b4fd9 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java @@ -57,7 +57,7 @@ public class UpgraderTest { assertEquals("New system version: Should upgrade Canaries", 2, tester.buildSystem().jobs().size()); tester.completeUpgrade(canary0, version, "canary"); - assertEquals(version, tester.configServer().lastPrepareVersion.get()); + assertEquals(version, tester.configServer().lastPrepareVersion().get()); tester.updateVersionStatus(version); tester.upgrader().maintain(); @@ -107,7 +107,7 @@ public class UpgraderTest { assertEquals("New system version: Should upgrade Canaries", 2, tester.buildSystem().jobs().size()); tester.completeUpgrade(canary0, version, "canary"); - assertEquals(version, tester.configServer().lastPrepareVersion.get()); + assertEquals(version, tester.configServer().lastPrepareVersion().get()); tester.updateVersionStatus(version); tester.upgrader().maintain(); @@ -187,7 +187,7 @@ public class UpgraderTest { assertEquals("New system version: Should upgrade Canaries", 2, tester.buildSystem().jobs().size()); tester.completeUpgrade(canary0, version, "canary"); - assertEquals(version, tester.configServer().lastPrepareVersion.get()); + assertEquals(version, tester.configServer().lastPrepareVersion().get()); tester.updateVersionStatus(version); tester.upgrader().maintain(); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/testdata/application-without-project-id.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/testdata/application-without-project-id.json index 912d1c2f4dc..63832531c7d 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/testdata/application-without-project-id.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/testdata/application-without-project-id.json @@ -13,8 +13,7 @@ "at": 1506330088050 } } - ], - "selfTriggering": false + ] }, "outstandingChangeField": false } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/testdata/canary-with-stale-data.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/testdata/canary-with-stale-data.json index 323889c7c45..9864ce502f6 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/testdata/canary-with-stale-data.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/testdata/canary-with-stale-data.json @@ -288,8 +288,7 @@ "at": 1493033800469 } } - ], - "selfTriggering": false + ] }, "outstandingChangeField": false } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java index 1aaf41350f2..f0a2019d41e 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java @@ -67,7 +67,7 @@ public class ApplicationSerializerTest { .withTriggering(Version.fromString("5.6.6"), Optional.empty(), true, Instant.ofEpochMilli(5)) .withCompletion(Optional.of(JobError.unknown), Instant.ofEpochMilli(6), tester.controller())); - DeploymentJobs deploymentJobs = new DeploymentJobs(projectId, statusList, Optional.empty(), false); + DeploymentJobs deploymentJobs = new DeploymentJobs(projectId, statusList, Optional.empty()); Application original = new Application(ApplicationId.from("t1", "a1", "i1"), deploymentSpec, @@ -98,7 +98,6 @@ public class ApplicationSerializerTest { assertEquals( original.deploymentJobs().jobStatus().get(DeploymentJobs.JobType.stagingTest), serialized.deploymentJobs().jobStatus().get(DeploymentJobs.JobType.stagingTest)); assertEquals(original.deploymentJobs().failingSince(), serialized.deploymentJobs().failingSince()); - assertEquals(original.deploymentJobs().isSelfTriggering(), serialized.deploymentJobs().isSelfTriggering()); assertEquals(original.hasOutstandingChange(), serialized.hasOutstandingChange()); @@ -167,8 +166,7 @@ public class ApplicationSerializerTest { " \"at\": 1505725189469\n" + " }\n" + " }\n" + - " ],\n" + - " \"selfTriggering\": false\n" + + " ]\n" + " }\n" + "}\n"; } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerControllerTester.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerControllerTester.java index ef606a0eced..ed7378ac6b5 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerControllerTester.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerControllerTester.java @@ -89,8 +89,8 @@ public class ContainerControllerTester { public void notifyJobCompletion(ApplicationId applicationId, long projectId, boolean success, DeploymentJobs.JobType job) { controller().applications().notifyJobCompletion(new DeploymentJobs.JobReport(applicationId, job, projectId, 42, - success ? Optional.empty() : Optional.of(DeploymentJobs.JobError.unknown), - false)); + success ? Optional.empty() : Optional.of(DeploymentJobs.JobError.unknown) + )); } public AthensDomain addTenantAthensDomain(String domainName, String userName) { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java index 7d1700270ea..50ad84028f1 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java @@ -9,6 +9,9 @@ import com.yahoo.vespa.hosted.controller.api.identifiers.AthensDomain; import com.yahoo.vespa.hosted.controller.api.identifiers.UserId; import com.yahoo.vespa.hosted.controller.api.integration.athens.Athens; import com.yahoo.vespa.hosted.controller.api.integration.athens.AthensPrincipal; +import com.yahoo.vespa.hosted.controller.api.integration.athens.mock.AthensDbMock; +import com.yahoo.vespa.hosted.controller.api.integration.athens.mock.AthensMock; +import com.yahoo.vespa.hosted.controller.api.integration.athens.mock.ZmsClientFactoryMock; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerException; import com.yahoo.vespa.hosted.controller.api.integration.cost.ApplicationCost; import com.yahoo.vespa.hosted.controller.api.integration.cost.ClusterCost; @@ -19,9 +22,6 @@ import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; import com.yahoo.vespa.hosted.controller.restapi.ContainerControllerTester; import com.yahoo.vespa.hosted.controller.restapi.ContainerTester; import com.yahoo.vespa.hosted.controller.restapi.ControllerContainerTest; -import com.yahoo.vespa.hosted.controller.api.integration.athens.mock.AthensMock; -import com.yahoo.vespa.hosted.controller.api.integration.athens.mock.AthensDbMock; -import com.yahoo.vespa.hosted.controller.api.integration.athens.mock.ZmsClientFactoryMock; import org.apache.http.HttpEntity; import org.apache.http.entity.ContentType; import org.apache.http.entity.mime.MultipartEntityBuilder; @@ -32,7 +32,6 @@ import java.io.File; import java.io.IOException; import java.io.UncheckedIOException; import java.net.URI; -import java.net.URISyntaxException; import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.HashMap; @@ -42,6 +41,7 @@ import java.util.Optional; /** * @author bratseth + * @author mpolden */ public class ApplicationApiTest extends ControllerContainerTest { @@ -54,7 +54,7 @@ public class ApplicationApiTest extends ControllerContainerTest { private static final String athensScrewdriverDomain = "screwdriver-domain"; @Test - public void testApplicationApi() throws IOException { + public void testApplicationApi() throws Exception { ContainerControllerTester controllerTester = new ContainerControllerTester(container, responseFiles); ContainerTester tester = controllerTester.containerTester(); tester.updateSystemVersion(); @@ -104,6 +104,14 @@ public class ApplicationApiTest extends ControllerContainerTest { tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/deploying", "6.1.0", Request.Method.POST), new File("application-deployment.json")); + // DELETE (cancel) ongoing change + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/deploying", "", Request.Method.DELETE), + new File("application-deployment-cancelled.json")); + + // DELETE (cancel) again is a no-op + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/deploying", "", Request.Method.DELETE), + new File("application-deployment-cancelled-no-op.json")); + // POST (deploy) an application to a zone - manual user deployment HttpEntity entity = createApplicationDeployData(applicationPackage, Optional.empty()); tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/dev/region/us-west-1/instance/default/deploy", @@ -118,6 +126,10 @@ public class ApplicationApiTest extends ControllerContainerTest { addScrewdriverUserToDomain("screwdriveruser1", "domain1"); // (Necessary but not provided in this API) + // Trigger deployment + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/deploying", "6.1.0", Request.Method.POST), + new File("application-deployment.json")); + // ... systemtest tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/test/region/test-region/instance/default/", createApplicationDeployData(applicationPackage, Optional.of(screwdriverProjectId)), @@ -267,9 +279,39 @@ public class ApplicationApiTest extends ControllerContainerTest { tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-west-1/instance/default/promote", "", Request.Method.POST), "{\"message\":\"Successfully copied environment hosted-instance_tenant1_application1_placeholder_component_default to hosted-instance_tenant1_application1_us-west-1_prod_default\"}"); } + + @Test + public void testDeployDirectly() throws Exception { + // Setup + ContainerControllerTester controllerTester = new ContainerControllerTester(container, responseFiles); + ContainerTester tester = controllerTester.containerTester(); + tester.updateSystemVersion(); + addTenantAthensDomain(athensUserDomain, "mytenant"); + addScrewdriverUserToDomain("screwdriveruser1", "domain1"); + + // Create tenant + tester.assertResponse(request("/application/v4/tenant/tenant1", + "{\"athensDomain\":\"domain1\", \"property\":\"property1\"}", + Request.Method.POST), + new File("tenant-without-applications.json")); + + // Create application + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1", + "", + Request.Method.POST), + new File("application-reference.json")); + + // POST (deploy) an application to a prod zone - allowed when project ID is not specified + HttpEntity entity = createApplicationDeployData(applicationPackage, Optional.empty()); + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/corp-us-east-1/instance/default/deploy", + entity, + Request.Method.POST, + athensScrewdriverDomain, "screwdriveruser1"), + new File("deploy-result.json")); + } @Test - public void testErrorResponses() throws IOException, URISyntaxException { + public void testErrorResponses() throws Exception { ContainerTester tester = new ContainerTester(container, responseFiles); tester.updateSystemVersion(); addTenantAthensDomain("domain1", "mytenant"); @@ -385,7 +427,7 @@ public class ApplicationApiTest extends ControllerContainerTest { } @Test - public void testAuthorization() throws IOException, URISyntaxException { + public void testAuthorization() throws Exception { ContainerTester tester = new ContainerTester(container, responseFiles); String authorizedUser = "mytenant"; String unauthorizedUser = "othertenant"; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-deployment-cancelled-no-op.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-deployment-cancelled-no-op.json new file mode 100644 index 00000000000..91d3e64d6db --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-deployment-cancelled-no-op.json @@ -0,0 +1 @@ +{"message":"No deployment in progress for application 'tenant1.application1' at this time"} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-deployment-cancelled.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-deployment-cancelled.json new file mode 100644 index 00000000000..d1e1ebe94fd --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-deployment-cancelled.json @@ -0,0 +1 @@ +{"message":"Cancelled version change to 6.1 for application 'tenant1.application1'"} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/screwdriver/ScrewdriverApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/screwdriver/ScrewdriverApiTest.java index bdfd0f9794f..5425b37d787 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/screwdriver/ScrewdriverApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/screwdriver/ScrewdriverApiTest.java @@ -74,7 +74,7 @@ public class ScrewdriverApiTest extends ControllerContainerTest { // Notifying about unknown job fails tester.containerTester().assertResponse(new Request("http://localhost:8080/screwdriver/v1/jobreport", jsonReport(app.id(), JobType.productionUsEast3, projectId, 1L, - Optional.empty(), false, true) + Optional.empty()) .getBytes(StandardCharsets.UTF_8), Request.Method.POST), new File("unexpected-completion.json"), 400); @@ -141,21 +141,19 @@ public class ScrewdriverApiTest extends ControllerContainerTest { private void notifyCompletion(ApplicationId app, long projectId, JobType jobType, Optional<JobError> error) throws IOException { assertResponse(new Request("http://localhost:8080/screwdriver/v1/jobreport", - jsonReport(app, jobType, projectId, 1L, error, false, true).getBytes(StandardCharsets.UTF_8), + jsonReport(app, jobType, projectId, 1L, error).getBytes(StandardCharsets.UTF_8), Request.Method.POST), 200, "ok"); } private static String jsonReport(ApplicationId applicationId, JobType jobType, long projectId, long buildNumber, - Optional<JobError> jobError, boolean selfTriggering, boolean gitChanges) { + Optional<JobError> jobError) { return "{\n" + " \"projectId\" : " + projectId + ",\n" + " \"jobName\" :\"" + jobType.id() + "\",\n" + " \"buildNumber\" : " + buildNumber + ",\n" + jobError.map(message -> " \"jobError\" : \"" + message + "\",\n").orElse("") + - " \"selfTriggering\": " + selfTriggering + ",\n" + - " \"gitChanges\" : " + gitChanges + ",\n" + " \"tenant\" :\"" + applicationId.tenant().value() + "\",\n" + " \"application\" :\"" + applicationId.application().value() + "\",\n" + " \"instance\" :\"" + applicationId.instance().value() + "\"\n" + diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/VersionStatusTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/VersionStatusTest.java index c4a3bd9cd81..6071776edf5 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/VersionStatusTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/VersionStatusTest.java @@ -274,8 +274,7 @@ public class VersionStatusTest { jobType, application.deploymentJobs().projectId().get(), 42, - JobError.from(success), - false + JobError.from(success) ); } diff --git a/defaults/CMakeLists.txt b/defaults/CMakeLists.txt index 2b7f719d297..c42e5402688 100644 --- a/defaults/CMakeLists.txt +++ b/defaults/CMakeLists.txt @@ -6,3 +6,5 @@ vespa_define_module( APPS src/apps/printdefault ) + +install_fat_java_artifact(defaults) @@ -9,5 +9,5 @@ fi VERSION="$1" mkdir -p ~/rpmbuild/{SOURCES,SPECS} -GZIP=-1 tar -zcf ~/rpmbuild/SOURCES/vespa-$VERSION.tar.gz --transform "flags=r;s,^,vespa-$VERSION/," * +GZIP=-1 tar -zcf ~/rpmbuild/SOURCES/vespa-$VERSION.tar.gz --exclude target --exclude cmake-build-debug --transform "flags=r;s,^,vespa-$VERSION/," * sed -e "s,VESPA_VERSION,$VERSION," < dist/vespa.spec > ~/rpmbuild/SPECS/vespa-$VERSION.spec diff --git a/dist/post_install.sh b/dist/post_install.sh deleted file mode 100755 index fb0e19370ee..00000000000 --- a/dist/post_install.sh +++ /dev/null @@ -1,76 +0,0 @@ -#!/bin/bash -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -if [ $# -ne 1 ]; then - echo "Usage: $0 <install prefix>" - exit 1 -fi - -declare -r PREFIX="$1" -declare -r INSTALLPATH="$DESTDIR/$PREFIX" - -# Rewrite config def file names -for path in $INSTALLPATH/var/db/vespa/config_server/serverdb/classes/*.def; do - dir=$(dirname $path) - filename=$(basename $path) - namespace=$(grep '^ *namespace *=' $path | sed 's/ *namespace *= *//') - if [ "$namespace" ]; then - case $filename in - $namespace.*) - ;; - *) - mv $path $dir/$namespace.$filename ;; - esac - fi -done - -mkdir -p $INSTALLPATH/conf/configserver/ -mkdir -p $INSTALLPATH/conf/configserver-app/ -mkdir -p $INSTALLPATH/conf/configserver-app/config-models/ -mkdir -p $INSTALLPATH/conf/configserver-app/components/ -mkdir -p $INSTALLPATH/conf/filedistributor/ -mkdir -p $INSTALLPATH/conf/node-admin-app/ -mkdir -p $INSTALLPATH/conf/node-admin-app/components/ -mkdir -p $INSTALLPATH/conf/zookeeper/ -mkdir -p $INSTALLPATH/libexec/jdisc_core/ -mkdir -p $INSTALLPATH/libexec/vespa/modelplugins/ -mkdir -p $INSTALLPATH/libexec/vespa/plugins/qrs/ -mkdir -p $INSTALLPATH/logs/jdisc_core/ -mkdir -p $INSTALLPATH/logs/vespa/ -mkdir -p $INSTALLPATH/logs/vespa/ -mkdir -p $INSTALLPATH/logs/vespa/configserver/ -mkdir -p $INSTALLPATH/logs/vespa/search/ -mkdir -p $INSTALLPATH/logs/vespa/qrs/ -mkdir -p $INSTALLPATH/share/vespa/ -mkdir -p $INSTALLPATH/share/vespa/schema/version/6.x/schema/ -mkdir -p $INSTALLPATH/tmp/vespa/ -mkdir -p $INSTALLPATH/var/db/jdisc/logcontrol/ -mkdir -p $INSTALLPATH/var/db/vespa/ -mkdir -p $INSTALLPATH/var/db/vespa/config_server/serverdb/applications/ -mkdir -p $INSTALLPATH/var/db/vespa/logcontrol/ -mkdir -p $INSTALLPATH/var/jdisc_container/ -mkdir -p $INSTALLPATH/var/jdisc_core/ -mkdir -p $INSTALLPATH/var/run/ -mkdir -p $INSTALLPATH/var/spool/vespa/ -mkdir -p $INSTALLPATH/var/spool/master/inbox/ -mkdir -p $INSTALLPATH/var/vespa/bundlecache/ -mkdir -p $INSTALLPATH/var/vespa/cache/config/ -mkdir -p $INSTALLPATH/var/vespa/cmdlines/ -mkdir -p $INSTALLPATH/var/zookeeper/version-2/ -mkdir -p $INSTALLPATH/sbin - -ln -sf $PREFIX/lib/jars/config-model-fat.jar $INSTALLPATH/conf/configserver-app/components/config-model-fat.jar -ln -sf $PREFIX/lib/jars/configserver-jar-with-dependencies.jar $INSTALLPATH/conf/configserver-app/components/configserver.jar -ln -sf $PREFIX/lib/jars/orchestrator-jar-with-dependencies.jar $INSTALLPATH/conf/configserver-app/components/orchestrator.jar -ln -sf $PREFIX/lib/jars/node-repository-jar-with-dependencies.jar $INSTALLPATH/conf/configserver-app/components/node-repository.jar -ln -sf $PREFIX/lib/jars/zkfacade-jar-with-dependencies.jar $INSTALLPATH/conf/configserver-app/components/zkfacade.jar -ln -snf $PREFIX/conf/configserver-app/components $INSTALLPATH/lib/jars/config-models -ln -sf vespa-storaged-bin $INSTALLPATH/sbin/vespa-distributord-bin - -# Setup default enviroment -mkdir -p $INSTALLPATH/conf/vespa -cat > $INSTALLPATH/conf/vespa/default-env.txt <<EOF -fallback VESPA_HOME $PREFIX -override VESPA_USER vespa -EOF - diff --git a/dist/vespa.spec b/dist/vespa.spec index eae6a0b3817..a3e0ec4711f 100644 --- a/dist/vespa.spec +++ b/dist/vespa.spec @@ -90,7 +90,6 @@ Requires: zookeeper >= 3.4.9 %define _extra_link_directory /opt/vespa-libtorrent/lib;/opt/vespa-cppunit/lib %define _extra_include_directory /opt/vespa-libtorrent/include;/opt/vespa-cppunit/include %define _vespa_boost_lib_suffix %{nil} -%define _vespa_cxx_abi_flags -D_GLIBCXX_USE_CXX11_ABI=1 %endif Requires: java-1.8.0-openjdk Requires: openssl @@ -122,7 +121,6 @@ cmake3 -DCMAKE_INSTALL_PREFIX=%{_prefix} \ -DCMAKE_INSTALL_RPATH="%{_prefix}/lib64%{?_extra_link_directory:;%{_extra_link_directory}};/usr/lib/jvm/java-1.8.0/jre/lib/amd64/server" \ %{?_vespa_llvm_version:-DVESPA_LLVM_VERSION="%{_vespa_llvm_version}"} \ %{?_vespa_boost_lib_suffix:-DVESPA_BOOST_LIB_SUFFIX="%{_vespa_boost_lib_suffix}"} \ - %{?_vespa_cxx_abi_flags:-DVESPA_CXX_ABI_FLAGS="%{_vespa_cxx_abi_flags}"} \ . make %{_smp_mflags} diff --git a/docker-api/CMakeLists.txt b/docker-api/CMakeLists.txt new file mode 100644 index 00000000000..25957c81e4c --- /dev/null +++ b/docker-api/CMakeLists.txt @@ -0,0 +1,2 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +install_config_definition(src/main/resources/configdefinitions/docker.def vespa.hosted.dockerapi.docker.def) diff --git a/docker-api/pom.xml b/docker-api/pom.xml index b3f6d1ec12f..fc374a12dd2 100644 --- a/docker-api/pom.xml +++ b/docker-api/pom.xml @@ -26,7 +26,7 @@ <dependency> <groupId>com.github.docker-java</groupId> <artifactId>docker-java</artifactId> - <version>3.0.8</version> + <version>3.0.13</version> <exclusions> <exclusion> <groupId>org.slf4j</groupId> diff --git a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/DockerImpl.java b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/DockerImpl.java index db10a85bb45..a6f8783a22c 100644 --- a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/DockerImpl.java +++ b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/DockerImpl.java @@ -419,7 +419,7 @@ public class DockerImpl implements Docker { @Override public void buildImage(File dockerfile, DockerImage image) { try { - dockerClient.buildImageCmd(dockerfile).withTag(image.asString()) + dockerClient.buildImageCmd(dockerfile).withTags(Collections.singleton(image.asString())) .exec(new BuildImageResultCallback()).awaitImageId(); } catch (RuntimeException e) { numberOfDockerDaemonFails.add(); diff --git a/docproc/CMakeLists.txt b/docproc/CMakeLists.txt new file mode 100644 index 00000000000..bacb45df319 --- /dev/null +++ b/docproc/CMakeLists.txt @@ -0,0 +1,4 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +install_config_definition(src/main/resources/configdefinitions/docproc.def config.docproc.docproc.def) +install_config_definition(src/main/resources/configdefinitions/schemamapping.def config.docproc.schemamapping.def) +install_config_definition(src/main/resources/configdefinitions/splitter-joiner-document-processor.def config.docproc.splitter-joiner-document-processor.def) diff --git a/docprocs/CMakeLists.txt b/docprocs/CMakeLists.txt new file mode 100644 index 00000000000..8786a77cbf8 --- /dev/null +++ b/docprocs/CMakeLists.txt @@ -0,0 +1,2 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +install_fat_java_artifact(docprocs) diff --git a/document/CMakeLists.txt b/document/CMakeLists.txt index ca2ee029c87..72f4ee445fb 100644 --- a/document/CMakeLists.txt +++ b/document/CMakeLists.txt @@ -40,3 +40,5 @@ vespa_define_module( src/tests/struct_anno src/tests/tensor_fieldvalue ) + +install_java_artifact(document) diff --git a/document/src/vespa/document/config/CMakeLists.txt b/document/src/vespa/document/config/CMakeLists.txt index 32bbce210a5..fc711c5e57a 100644 --- a/document/src/vespa/document/config/CMakeLists.txt +++ b/document/src/vespa/document/config/CMakeLists.txt @@ -4,7 +4,5 @@ vespa_add_library(document_documentconfig OBJECT DEPENDS ) vespa_generate_config(document_documentconfig documenttypes.def) -install(FILES documenttypes.def RENAME document.documenttypes.def - DESTINATION var/db/vespa/config_server/serverdb/classes) -install(FILES documentmanager.def RENAME document.config.documentmanager.def - DESTINATION var/db/vespa/config_server/serverdb/classes) +install_config_definition(documenttypes.def document.documenttypes.def) +install_config_definition(documentmanager.def document.config.documentmanager.def) diff --git a/documentapi/src/vespa/documentapi/messagebus/policies/CMakeLists.txt b/documentapi/src/vespa/documentapi/messagebus/policies/CMakeLists.txt index 143310d1f67..f1a691bc46d 100644 --- a/documentapi/src/vespa/documentapi/messagebus/policies/CMakeLists.txt +++ b/documentapi/src/vespa/documentapi/messagebus/policies/CMakeLists.txt @@ -20,4 +20,4 @@ vespa_add_library(documentapi_documentapipolicies OBJECT DEPENDS ) vespa_generate_config(documentapi_documentapipolicies ../../../../main/resources/configdefinitions/documentrouteselectorpolicy.def) -install(FILES ../../../../main/resources/configdefinitions/documentrouteselectorpolicy.def RENAME documentapi.messagebus.protocol.documentrouteselectorpolicy.def DESTINATION var/db/vespa/config_server/serverdb/classes) +install_config_definition(../../../../main/resources/configdefinitions/documentrouteselectorpolicy.def documentapi.messagebus.protocol.documentrouteselectorpolicy.def) diff --git a/eval/CMakeLists.txt b/eval/CMakeLists.txt index 19a614464a1..89e8a72e330 100644 --- a/eval/CMakeLists.txt +++ b/eval/CMakeLists.txt @@ -6,6 +6,7 @@ vespa_define_module( APPS src/apps/eval_expr src/apps/make_tensor_binary_format_test_spec + src/apps/tensor_conformance TESTS src/tests/eval/aggr diff --git a/eval/src/apps/make_tensor_binary_format_test_spec/make_tensor_binary_format_test_spec.cpp b/eval/src/apps/make_tensor_binary_format_test_spec/make_tensor_binary_format_test_spec.cpp index 5040ae35ff9..a7695408a85 100644 --- a/eval/src/apps/make_tensor_binary_format_test_spec/make_tensor_binary_format_test_spec.cpp +++ b/eval/src/apps/make_tensor_binary_format_test_spec/make_tensor_binary_format_test_spec.cpp @@ -188,7 +188,6 @@ void make_matrix_test(Cursor &test, size_t x_size, size_t y_size) { //----------------------------------------------------------------------------- void make_map_test(Cursor &test, const Dict &x_dict_in) { - TensorSpec spec("tensor(x{})"); nbostream sparse_base = make_sparse(); sparse_base.putInt1_4Bytes(1); sparse_base.writeSmallString("x"); @@ -200,6 +199,7 @@ void make_map_test(Cursor &test, const Dict &x_dict_in) { mixed_base.putInt1_4Bytes(x_dict_in.size()); auto x_perm = make_permutations(x_dict_in); for (const Dict &x_dict: x_perm) { + TensorSpec spec("tensor(x{})"); nbostream sparse = sparse_base; nbostream mixed = mixed_base; for (vespalib::string x: x_dict) { @@ -214,13 +214,13 @@ void make_map_test(Cursor &test, const Dict &x_dict_in) { add_binary(test, {sparse, mixed}); } if (x_dict_in.empty()) { + TensorSpec spec("tensor(x{})"); set_tensor(test, spec); add_binary(test, {sparse_base, mixed_base}); } } void make_mesh_test(Cursor &test, const Dict &x_dict_in, const vespalib::string &y) { - TensorSpec spec("tensor(x{},y{})"); nbostream sparse_base = make_sparse(); sparse_base.putInt1_4Bytes(2); sparse_base.writeSmallString("x"); @@ -234,6 +234,7 @@ void make_mesh_test(Cursor &test, const Dict &x_dict_in, const vespalib::string mixed_base.putInt1_4Bytes(x_dict_in.size() * 1); auto x_perm = make_permutations(x_dict_in); for (const Dict &x_dict: x_perm) { + TensorSpec spec("tensor(x{},y{})"); nbostream sparse = sparse_base; nbostream mixed = mixed_base; for (vespalib::string x: x_dict) { @@ -250,6 +251,7 @@ void make_mesh_test(Cursor &test, const Dict &x_dict_in, const vespalib::string add_binary(test, {sparse, mixed}); } if (x_dict_in.empty()) { + TensorSpec spec("tensor(x{},y{})"); set_tensor(test, spec); add_binary(test, {sparse_base, mixed_base}); } @@ -264,7 +266,6 @@ void make_vector_map_test(Cursor &test, auto type_str = vespalib::make_string("tensor(%s{},%s[%zu])", mapped_name.c_str(), indexed_name.c_str(), indexed_size); ValueType type = ValueType::from_spec(type_str); - TensorSpec spec(type.to_spec()); // ensures type string is normalized nbostream mixed_base = make_mixed(); mixed_base.putInt1_4Bytes(1); mixed_base.writeSmallString(mapped_name); @@ -274,6 +275,7 @@ void make_vector_map_test(Cursor &test, mixed_base.putInt1_4Bytes(mapped_dict.size()); auto mapped_perm = make_permutations(mapped_dict); for (const Dict &dict: mapped_perm) { + TensorSpec spec(type.to_spec()); // ensures type string is normalized nbostream mixed = mixed_base; for (vespalib::string label: dict) { mixed.writeSmallString(label); @@ -287,6 +289,7 @@ void make_vector_map_test(Cursor &test, add_binary(test, mixed); } if (mapped_dict.empty()) { + TensorSpec spec(type.to_spec()); // ensures type string is normalized set_tensor(test, spec); add_binary(test, mixed_base); } diff --git a/eval/src/apps/tensor_conformance/.gitignore b/eval/src/apps/tensor_conformance/.gitignore new file mode 100644 index 00000000000..3e87a05826c --- /dev/null +++ b/eval/src/apps/tensor_conformance/.gitignore @@ -0,0 +1 @@ +/vespa-tensor-conformance diff --git a/eval/src/apps/tensor_conformance/CMakeLists.txt b/eval/src/apps/tensor_conformance/CMakeLists.txt new file mode 100644 index 00000000000..76ababd9f5e --- /dev/null +++ b/eval/src/apps/tensor_conformance/CMakeLists.txt @@ -0,0 +1,8 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(vespa-tensor-conformance + SOURCES + generate.cpp + tensor_conformance.cpp + DEPENDS + vespaeval +) diff --git a/eval/src/apps/tensor_conformance/generate.cpp b/eval/src/apps/tensor_conformance/generate.cpp new file mode 100644 index 00000000000..45ff6243d81 --- /dev/null +++ b/eval/src/apps/tensor_conformance/generate.cpp @@ -0,0 +1,18 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "generate.h" + +using TensorSpec = vespalib::eval::TensorSpec; + +TensorSpec spec(double value) { return TensorSpec("double").add({}, value); } + +void +Generator::generate(TestBuilder &dst) +{ + // smoke tests with expected result + dst.add("a+a", {{"a", spec(2.0)}}, spec(4.0)); + dst.add("a*b", {{"a", spec(2.0)}, {"b", spec(3.0)}}, spec(6.0)); + dst.add("(a+b)*(a-b)", {{"a", spec(5.0)}, {"b", spec(2.0)}}, spec(21.0)); + // smoke test without expected result + dst.add("(a-b)/(a+b)", {{"a", spec(5.0)}, {"b", spec(2.0)}}); +} diff --git a/eval/src/apps/tensor_conformance/generate.h b/eval/src/apps/tensor_conformance/generate.h new file mode 100644 index 00000000000..d20d085f00c --- /dev/null +++ b/eval/src/apps/tensor_conformance/generate.h @@ -0,0 +1,22 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <vespa/eval/eval/tensor_spec.h> +#include <map> + +struct TestBuilder { + using TensorSpec = vespalib::eval::TensorSpec; + // add test with undefined expected result + virtual void add(const vespalib::string &expression, + const std::map<vespalib::string,TensorSpec> &inputs) = 0; + // add test with pre-defined expected result + virtual void add(const vespalib::string &expression, + const std::map<vespalib::string,TensorSpec> &inputs, + const TensorSpec &expect) = 0; + virtual ~TestBuilder() {} +}; + +struct Generator { + static void generate(TestBuilder &out); +}; diff --git a/eval/src/apps/tensor_conformance/tensor_conformance.cpp b/eval/src/apps/tensor_conformance/tensor_conformance.cpp new file mode 100644 index 00000000000..593e4439b0a --- /dev/null +++ b/eval/src/apps/tensor_conformance/tensor_conformance.cpp @@ -0,0 +1,318 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/vespalib/data/slime/slime.h> +#include <vespa/vespalib/data/slime/json_format.h> +#include <vespa/vespalib/objects/nbostream.h> +#include <vespa/vespalib/util/stringfmt.h> +#include <vespa/eval/eval/tensor_spec.h> +#include <vespa/eval/eval/tensor.h> +#include <vespa/eval/eval/function.h> +#include <vespa/eval/eval/interpreted_function.h> +#include <vespa/eval/eval/tensor_engine.h> +#include <vespa/eval/eval/simple_tensor_engine.h> +#include <vespa/eval/tensor/default_tensor_engine.h> +#include <vespa/eval/eval/value_type.h> +#include <vespa/eval/eval/value.h> +#include <unistd.h> + +#include "generate.h" + +using namespace vespalib; +using namespace vespalib::eval; +using namespace vespalib::slime::convenience; +using slime::JsonFormat; +using tensor::DefaultTensorEngine; + +constexpr size_t CHUNK_SIZE = 16384; +constexpr bool not_compact = false; + +//----------------------------------------------------------------------------- + +size_t num_tests = 0; +std::map<vespalib::string,size_t> result_map; + +vespalib::string result_stats() { + vespalib::string stats; + for (const auto &entry: result_map) { + if (!stats.empty()) { + stats += ", "; + } + stats += make_string("%s: %zu", entry.first.c_str(), entry.second); + } + return stats; +} + +//----------------------------------------------------------------------------- + +class StdIn : public Input { +private: + bool _eof = false; + SimpleBuffer _input; +public: + ~StdIn() {} + Memory obtain() override { + if ((_input.get().size == 0) && !_eof) { + WritableMemory buf = _input.reserve(CHUNK_SIZE); + ssize_t res = read(STDIN_FILENO, buf.data, buf.size); + _eof = (res == 0); + assert(res >= 0); // fail on stdio read errors + _input.commit(res); + } + return _input.obtain(); + } + Input &evict(size_t bytes) override { + _input.evict(bytes); + return *this; + } +}; + +class StdOut : public Output { +private: + SimpleBuffer _output; +public: + ~StdOut() {} + WritableMemory reserve(size_t bytes) override { + return _output.reserve(bytes); + } + Output &commit(size_t bytes) override { + _output.commit(bytes); + Memory buf = _output.obtain(); + ssize_t res = write(STDOUT_FILENO, buf.data, buf.size); + assert(res == ssize_t(buf.size)); // fail on stdout write failures + _output.evict(res); + return *this; + } +}; + +//----------------------------------------------------------------------------- + +uint8_t unhex(char c) { + if (c >= '0' && c <= '9') { + return (c - '0'); + } + if (c >= 'A' && c <= 'F') { + return ((c - 'A') + 10); + } + TEST_ERROR("bad hex char"); + return 0; +} + +void extract_data_from_string(Memory hex_dump, nbostream &data) { + if ((hex_dump.size > 2) && (hex_dump.data[0] == '0') && (hex_dump.data[1] == 'x')) { + for (size_t i = 2; i < (hex_dump.size - 1); i += 2) { + data << uint8_t((unhex(hex_dump.data[i]) << 4) | unhex(hex_dump.data[i + 1])); + } + } +} + +nbostream extract_data(const Inspector &value) { + nbostream data; + if (value.asString().size > 0) { + extract_data_from_string(value.asString(), data); + } else { + Memory buf = value.asData(); + data.write(buf.data, buf.size); + } + return data; +} + +//----------------------------------------------------------------------------- + +TensorSpec to_spec(const Value &value) { + if (value.is_error()) { + return TensorSpec("error"); + } else if (value.is_double()) { + return TensorSpec("double").add({}, value.as_double()); + } else { + ASSERT_TRUE(value.is_tensor()); + auto tensor = value.as_tensor(); + return tensor->engine().to_spec(*tensor); + } +} + +const Value &to_value(const TensorSpec &spec, const TensorEngine &engine, Stash &stash) { + if (spec.type() == "error") { + return stash.create<ErrorValue>(); + } else if (spec.type() == "double") { + double value = 0.0; + for (const auto &cell: spec.cells()) { + value += cell.second; + } + return stash.create<DoubleValue>(value); + } else { + ASSERT_TRUE(starts_with(spec.type(), "tensor(")); + return stash.create<TensorValue>(engine.create(spec)); + } +} + +void insert_value(Cursor &cursor, const vespalib::string &name, const TensorSpec &spec) { + Stash stash; + nbostream data; + const Value &value = to_value(spec, SimpleTensorEngine::ref(), stash); + SimpleTensorEngine::ref().encode(value, data, stash); + cursor.setData(name, Memory(data.peek(), data.size())); +} + +TensorSpec extract_value(const Inspector &inspector) { + Stash stash; + nbostream data = extract_data(inspector); + return to_spec(SimpleTensorEngine::ref().decode(data, stash)); +} + +//----------------------------------------------------------------------------- + +TensorSpec eval_expr(const Inspector &test, const TensorEngine &engine) { + Stash stash; + Function fun = Function::parse(test["expression"].asString().make_string()); + std::vector<Value::CREF> param_values; + std::vector<ValueType> param_types; + for (size_t i = 0; i < fun.num_params(); ++i) { + param_values.emplace_back(to_value(extract_value(test["inputs"][fun.param_name(i)]), engine, stash)); + } + for (size_t i = 0; i < fun.num_params(); ++i) { + param_types.emplace_back(param_values[i].get().type()); + } + NodeTypes types(fun, param_types); + InterpretedFunction ifun(engine, fun, types); + InterpretedFunction::Context ctx(ifun); + InterpretedFunction::SimpleObjectParams params(param_values); + return to_spec(ifun.eval(ctx, params)); +} + +//----------------------------------------------------------------------------- + +std::vector<vespalib::string> extract_fields(const Inspector &object) { + struct FieldExtractor : slime::ObjectTraverser { + std::vector<vespalib::string> result; + void field(const Memory &symbol, const Inspector &) override { + result.push_back(symbol.make_string()); + } + } extractor; + object.traverse(extractor); + return std::move(extractor.result); +}; + +void dump_test(const Inspector &test) { + fprintf(stderr, "expression: '%s'\n", test["expression"].asString().make_string().c_str()); + for (const auto &input: extract_fields(test["inputs"])) { + auto value = extract_value(test["inputs"][input]); + fprintf(stderr, "input '%s': %s\n", input.c_str(), value.to_string().c_str()); + } +} + +//----------------------------------------------------------------------------- + +class MyTestBuilder : public TestBuilder { +private: + Output &_out; + void build_test(Cursor &test, const vespalib::string &expression, + const std::map<vespalib::string,TensorSpec> &input_map) + { + test.setString("expression", expression); + Cursor &inputs = test.setObject("inputs"); + for (const auto &input: input_map) { + insert_value(inputs, input.first, input.second); + } + } +public: + MyTestBuilder(Output &out) : _out(out) {} + void add(const vespalib::string &expression, + const std::map<vespalib::string,TensorSpec> &inputs) override + { + Slime slime; + build_test(slime.setObject(), expression, inputs); + insert_value(slime.get().setObject("result"), "expect", + eval_expr(slime.get(), SimpleTensorEngine::ref())); + JsonFormat::encode(slime, _out, not_compact); + ++num_tests; + } + void add(const vespalib::string &expression, + const std::map<vespalib::string,TensorSpec> &inputs, + const TensorSpec &expect) override + { + Slime slime; + build_test(slime.setObject(), expression, inputs); + insert_value(slime.get().setObject("result"), "expect", expect); + if (!EXPECT_EQUAL(eval_expr(slime.get(), SimpleTensorEngine::ref()), expect)) { + dump_test(slime.get()); + } + JsonFormat::encode(slime, _out, not_compact); + ++num_tests; + } +}; + +void generate(Output &out) { + MyTestBuilder my_test_builder(out); + Generator::generate(my_test_builder); +} + +//----------------------------------------------------------------------------- + +void evaluate(Input &in, Output &out) { + while (in.obtain().size > 0) { + Slime slime; + if (JsonFormat::decode(in, slime)) { + ++num_tests; + insert_value(slime.get()["result"], "prod_cpp", + eval_expr(slime.get(), DefaultTensorEngine::ref())); + JsonFormat::encode(slime, out, not_compact); + } + } +} + +//----------------------------------------------------------------------------- + +void verify(Input &in) { + while (in.obtain().size > 0) { + Slime slime; + if (JsonFormat::decode(in, slime)) { + ++num_tests; + TensorSpec reference_result = eval_expr(slime.get(), SimpleTensorEngine::ref()); + for (const auto &result: extract_fields(slime.get()["result"])) { + ++result_map[result]; + TEST_STATE(make_string("verifying result: '%s'", result.c_str()).c_str()); + if (!EXPECT_EQUAL(reference_result, extract_value(slime.get()["result"][result]))) { + dump_test(slime.get()); + } + } + } + } +} + +//----------------------------------------------------------------------------- + +int usage(const char *self) { + fprintf(stderr, "usage: %s <mode>\n", self); + fprintf(stderr, " <mode>: which mode to activate\n"); + fprintf(stderr, " 'generate': write test cases to stdout\n"); + fprintf(stderr, " 'evaluate': read test cases from stdin, annotate them with\n"); + fprintf(stderr, " results from various implementations and write\n"); + fprintf(stderr, " them to stdout\n"); + fprintf(stderr, " 'verify': read annotated test cases from stdin and verify\n"); + fprintf(stderr, " that all results are as expected\n"); + return 1; +} + +int main(int argc, char **argv) { + StdIn std_in; + StdOut std_out; + if (argc != 2) { + return usage(argv[0]); + } + vespalib::string mode = argv[1]; + TEST_MASTER.init(make_string("vespa-tensor-conformance-%s", mode.c_str()).c_str()); + if (mode == "generate") { + generate(std_out); + fprintf(stderr, "generated %zu test cases\n", num_tests); + } else if (mode == "evaluate") { + evaluate(std_in, std_out); + fprintf(stderr, "evaluated %zu test cases\n", num_tests); + } else if (mode == "verify") { + verify(std_in); + fprintf(stderr, "verified %zu test cases (%s)\n", num_tests, result_stats().c_str()); + } else { + TEST_ERROR(make_string("unknown mode: %s", mode.c_str()).c_str()); + } + return (TEST_MASTER.fini() ? 0 : 1); +} diff --git a/eval/src/apps/tensor_conformance/test_spec.json b/eval/src/apps/tensor_conformance/test_spec.json new file mode 100644 index 00000000000..a7c906cfb85 --- /dev/null +++ b/eval/src/apps/tensor_conformance/test_spec.json @@ -0,0 +1,39 @@ +{ + "expression": "a+a", + "inputs": { + "a": "0x02004000000000000000" + }, + "result": { + "expect": "0x02004010000000000000" + } +} +{ + "expression": "a*b", + "inputs": { + "a": "0x02004000000000000000", + "b": "0x02004008000000000000" + }, + "result": { + "expect": "0x02004018000000000000" + } +} +{ + "expression": "(a+b)*(a-b)", + "inputs": { + "a": "0x02004014000000000000", + "b": "0x02004000000000000000" + }, + "result": { + "expect": "0x02004035000000000000" + } +} +{ + "expression": "(a-b)/(a+b)", + "inputs": { + "a": "0x02004014000000000000", + "b": "0x02004000000000000000" + }, + "result": { + "expect": "0x02003FDB6DB6DB6DB6DB" + } +} diff --git a/fileacquirer/CMakeLists.txt b/fileacquirer/CMakeLists.txt index 1ae83a6f6c2..7a366aa3882 100644 --- a/fileacquirer/CMakeLists.txt +++ b/fileacquirer/CMakeLists.txt @@ -9,3 +9,5 @@ vespa_define_module( LIBS src/vespa/fileacquirer ) + +install_config_definition(src/main/resources/configdefinitions/filedistributorrpc.def cloud.config.filedistribution.filedistributorrpc.def) diff --git a/filedistribution/src/vespa/filedistribution/distributor/CMakeLists.txt b/filedistribution/src/vespa/filedistribution/distributor/CMakeLists.txt index 5345cba1e3f..f85ab85fb39 100644 --- a/filedistribution/src/vespa/filedistribution/distributor/CMakeLists.txt +++ b/filedistribution/src/vespa/filedistribution/distributor/CMakeLists.txt @@ -12,4 +12,4 @@ vespa_add_library(filedistribution_distributor STATIC ) target_compile_options(filedistribution_distributor PRIVATE -DTORRENT_DISABLE_ENCRYPTION -DTORRENT_DISABLE_DHT -DWITH_SHIPPED_GEOIP_H -DBOOST_ASIO_HASH_MAP_BUCKETS=1021 -DBOOST_EXCEPTION_DISABLE -DBOOST_ASIO_ENABLE_CANCELIO -DBOOST_ASIO_DYN_LINK -DTORRENT_LINKING_SHARED) vespa_generate_config(filedistribution_distributor filedistributor.def) -install(FILES filedistributor.def RENAME cloud.config.filedistribution.filedistributor.def DESTINATION var/db/vespa/config_server/serverdb/classes) +install_config_definition(filedistributor.def cloud.config.filedistribution.filedistributor.def) diff --git a/filedistribution/src/vespa/filedistribution/distributor/filedownloadermanager.cpp b/filedistribution/src/vespa/filedistribution/distributor/filedownloadermanager.cpp index 0d11c0c1528..669cc550003 100644 --- a/filedistribution/src/vespa/filedistribution/distributor/filedownloadermanager.cpp +++ b/filedistribution/src/vespa/filedistribution/distributor/filedownloadermanager.cpp @@ -1,8 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include "filedownloadermanager.h" -#include <iterator> -#include <sstream> +#include "filedownloadermanager.h" #include <thread> #include <vespa/log/log.h> diff --git a/filedistribution/src/vespa/filedistribution/manager/filedistributionmanager.cpp b/filedistribution/src/vespa/filedistribution/manager/filedistributionmanager.cpp index f1c74e4a000..954cce23205 100644 --- a/filedistribution/src/vespa/filedistribution/manager/filedistributionmanager.cpp +++ b/filedistribution/src/vespa/filedistribution/manager/filedistributionmanager.cpp @@ -9,8 +9,6 @@ #include <vespa/filedistribution/model/filedistributionmodel.h> #include <vespa/filedistribution/model/zkfiledbmodel.h> #include <vespa/filedistribution/model/mockfiledistributionmodel.h> -#include <vespa/filedistribution/model/zkfacade.h> -#include <memory> using namespace filedistribution; diff --git a/filedistribution/src/vespa/filedistribution/model/CMakeLists.txt b/filedistribution/src/vespa/filedistribution/model/CMakeLists.txt index 7136aba475b..5b92aa4086d 100644 --- a/filedistribution/src/vespa/filedistribution/model/CMakeLists.txt +++ b/filedistribution/src/vespa/filedistribution/model/CMakeLists.txt @@ -17,4 +17,4 @@ vespa_add_library(filedistribution_filedistributionmodel STATIC vespa_generate_config(filedistribution_filedistributionmodel filereferences.def) vespa_add_target_external_dependency(filedistribution_filedistributionmodel zookeeper_mt) -install(FILES filereferences.def RENAME cloud.config.filedistribution.filereferences.def DESTINATION var/db/vespa/config_server/serverdb/classes) +install_config_definition(filereferences.def cloud.config.filedistribution.filereferences.def) diff --git a/filedistribution/src/vespa/filedistribution/model/deployedfilestodownload.cpp b/filedistribution/src/vespa/filedistribution/model/deployedfilestodownload.cpp index 9443d256c92..13daecbe5c1 100644 --- a/filedistribution/src/vespa/filedistribution/model/deployedfilestodownload.cpp +++ b/filedistribution/src/vespa/filedistribution/model/deployedfilestodownload.cpp @@ -2,8 +2,6 @@ #include "deployedfilestodownload.h" #include <vespa/filedistribution/common/logfwd.h> -#include <sstream> -#include <iterator> using filedistribution::DeployedFilesToDownload; using filedistribution::Path; diff --git a/filedistribution/src/vespa/filedistribution/model/filedbmodel.h b/filedistribution/src/vespa/filedistribution/model/filedbmodel.h index 9326e822dc7..c556c703b6d 100644 --- a/filedistribution/src/vespa/filedistribution/model/filedbmodel.h +++ b/filedistribution/src/vespa/filedistribution/model/filedbmodel.h @@ -1,8 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include <string> -#include <vector> #include <vespa/filedistribution/common/buffer.h> #include <vespa/filedistribution/common/exception.h> diff --git a/filedistribution/src/vespa/filedistribution/model/filedistributionmodelimpl.cpp b/filedistribution/src/vespa/filedistribution/model/filedistributionmodelimpl.cpp index ad415c11928..9ce31e0dd3a 100644 --- a/filedistribution/src/vespa/filedistribution/model/filedistributionmodelimpl.cpp +++ b/filedistribution/src/vespa/filedistribution/model/filedistributionmodelimpl.cpp @@ -4,7 +4,6 @@ #include "zkfiledbmodel.h" #include "deployedfilestodownload.h" #include "filedistributionmodelimpl.h" -#include <vespa/vespalib/util/stringfmt.h> #include <boost/filesystem.hpp> #include <zookeeper/zookeeper.h> diff --git a/filedistribution/src/vespa/filedistribution/model/zkfacade.cpp b/filedistribution/src/vespa/filedistribution/model/zkfacade.cpp index ffc82caf4b2..e07f0684584 100644 --- a/filedistribution/src/vespa/filedistribution/model/zkfacade.cpp +++ b/filedistribution/src/vespa/filedistribution/model/zkfacade.cpp @@ -6,7 +6,6 @@ #include <vespa/defaults.h> #include <vespa/vespalib/util/sync.h> #include <vespa/vespalib/text/stringtokenizer.h> -#include <vespa/vespalib/util/stringfmt.h> #include <zookeeper/zookeeper.h> #include <sstream> #include <thread> diff --git a/filedistribution/src/vespa/filedistribution/model/zkfiledbmodel.cpp b/filedistribution/src/vespa/filedistribution/model/zkfiledbmodel.cpp index 92777ef7b18..9931b104010 100644 --- a/filedistribution/src/vespa/filedistribution/model/zkfiledbmodel.cpp +++ b/filedistribution/src/vespa/filedistribution/model/zkfiledbmodel.cpp @@ -5,10 +5,7 @@ #include "zkfiledbmodel.h" #include "deployedfilestodownload.h" #include <vespa/filedistribution/common/logfwd.h> -#include <vespa/vespalib/util/stringfmt.h> #include <sys/file.h> -#include <ostream> -#include <algorithm> namespace fs = boost::filesystem; diff --git a/functions.cmake b/functions.cmake index 38ae339cd53..1c3d1ff7a6e 100644 --- a/functions.cmake +++ b/functions.cmake @@ -536,3 +536,23 @@ function(__export_include_directories TARGET) target_include_directories(${TARGET} PUBLIC ${LOCAL_INCLUDE_DIRS}) endif() endfunction() + +function(install_config_definition) + if(ARGC GREATER 1) + install(FILES ${ARGV0} RENAME ${ARGV1} DESTINATION var/db/vespa/config_server/serverdb/classes) + else() + install(FILES ${ARGV0} DESTINATION var/db/vespa/config_server/serverdb/classes) + endif() +endfunction() + +function(install_java_artifact NAME) + install(FILES "target/${NAME}.jar" DESTINATION lib/jars/) +endfunction() + +function(install_java_artifact_dependencies NAME) + install(DIRECTORY "target/dependency/" DESTINATION lib/jars FILES_MATCHING PATTERN "*.jar") +endfunction() + +function(install_fat_java_artifact NAME) + install(FILES "target/${NAME}-jar-with-dependencies.jar" DESTINATION lib/jars/) +endfunction() diff --git a/install_java.cmake b/install_java.cmake deleted file mode 100644 index e0611306b5f..00000000000 --- a/install_java.cmake +++ /dev/null @@ -1,161 +0,0 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -function(install_java_artifact NAME) - install(FILES "${NAME}/target/${NAME}.jar" DESTINATION lib/jars/) -endfunction() - -function(install_java_artifact_dependencies NAME) - install(DIRECTORY "${NAME}/target/dependency/" DESTINATION lib/jars FILES_MATCHING PATTERN "*.jar") -endfunction() - -function(install_fat_java_artifact NAME) - install(FILES "${NAME}/target/${NAME}-jar-with-dependencies.jar" DESTINATION lib/jars/) -endfunction() - -install_java_artifact(config-model-fat) -install_java_artifact(document) -install_java_artifact(jdisc_jetty) -install_java_artifact_dependencies(jdisc_jetty) -install_java_artifact_dependencies(vespa_jersey2) -install_java_artifact(searchlib) -install_java_artifact(vespajlib) - -install_fat_java_artifact(application-preprocessor) -install_fat_java_artifact(clustercontroller-apps) -install_fat_java_artifact(clustercontroller-apputil) -install_fat_java_artifact(clustercontroller-utils) -install_fat_java_artifact(clustercontroller-core) -install_fat_java_artifact(component) -install_fat_java_artifact(config-bundle) -install_fat_java_artifact(config-model-api) -install_fat_java_artifact(config-model) -install_fat_java_artifact(config-provisioning) -install_fat_java_artifact(config-proxy) -install_fat_java_artifact(configdefinitions) -install_fat_java_artifact(configserver) -install_fat_java_artifact(container-disc) -install_fat_java_artifact(container-jersey2) -install_fat_java_artifact(container-search-and-docproc) -install_fat_java_artifact(defaults) -install_fat_java_artifact(docprocs) -install_fat_java_artifact(jdisc_core) -install_fat_java_artifact(jdisc_http_service) -install_fat_java_artifact(logserver) -install_fat_java_artifact(node-repository) -install_fat_java_artifact(orchestrator) -install_fat_java_artifact(persistence) -install_fat_java_artifact(searchlib) -install_fat_java_artifact(simplemetrics) -install_fat_java_artifact(standalone-container) -install_fat_java_artifact(vespa-http-client) -install_fat_java_artifact(vespaclient-container-plugin) -install_fat_java_artifact(vespaclient-java) -install_fat_java_artifact(zkfacade) - -vespa_install_script(application-preprocessor/src/main/sh/vespa-preprocess-application bin) -vespa_install_script(config-proxy/src/main/sh/vespa-config-ctl.sh vespa-config-ctl bin) -vespa_install_script(config-proxy/src/main/sh/vespa-config-loadtester.sh vespa-config-loadtester bin) -vespa_install_script(config-proxy/src/main/sh/vespa-config-verification.sh vespa-config-verification bin) -vespa_install_script(config-model/src/main/perl/vespa-deploy bin) -vespa_install_script(config-model/src/main/perl/vespa-expand-config.pl bin) -vespa_install_script(config-model/src/main/perl/vespa-replicate-log-stream bin) -vespa_install_script(config-model/src/main/sh/vespa-validate-application bin) -vespa_install_script(container-disc/src/main/sh/vespa-start-container-daemon.sh vespa-start-container-daemon bin) -vespa_install_script(searchlib/src/main/sh/vespa-gbdt-converter bin) -vespa_install_script(searchlib/src/main/sh/vespa-treenet-converter bin) -vespa_install_script(vespaclient-java/src/main/sh/vespa-document-statistics.sh vespa-document-statistics bin) -vespa_install_script(vespaclient-java/src/main/sh/vespa-stat.sh vespa-stat bin) -vespa_install_script(vespaclient-java/src/main/sh/vespa-query-profile-dump-tool.sh vespa-query-profile-dump-tool bin) -vespa_install_script(vespaclient-java/src/main/sh/vespa-summary-benchmark.sh vespa-summary-benchmark bin) -vespa_install_script(vespaclient-java/src/main/sh/vespa-destination.sh vespa-destination bin) -vespa_install_script(vespaclient-java/src/main/sh/vespa-feeder.sh vespa-feeder bin) -vespa_install_script(vespaclient-java/src/main/sh/vespa-get.sh vespa-get bin) -vespa_install_script(vespaclient-java/src/main/sh/vespa-visit.sh vespa-visit bin) -vespa_install_script(vespaclient-java/src/main/sh/vespa-visit-target.sh vespa-visit-target bin) - -vespa_install_script(logserver/bin/logserver-start.sh vespa-logserver-start bin) - -install(DIRECTORY config-model/src/main/resources/schema DESTINATION share/vespa PATTERN ".gitignore" EXCLUDE) -install(DIRECTORY config-model/src/main/resources/schema DESTINATION share/vespa/schema/version/6.x PATTERN ".gitignore" EXCLUDE) - -install(FILES jdisc_core/src/main/perl/vespa-jdisc-logfmt.1 DESTINATION man/man1) - -install(FILES - config-model-fat/src/main/resources/config-models.xml - node-repository/src/main/config/node-repository.xml - DESTINATION conf/configserver-app) - -install(FILES - chain/src/main/resources/configdefinitions/chains.def - config-provisioning/src/main/resources/configdefinitions/flavors.def - container-accesslogging/src/main/resources/configdefinitions/access-log.def - container-core/src/main/resources/configdefinitions/application-metadata.def - container-core/src/main/resources/configdefinitions/container-document.def - container-core/src/main/resources/configdefinitions/container-http.def - container-core/src/main/resources/configdefinitions/diagnostics.def - container-core/src/main/resources/configdefinitions/health-monitor.def - container-core/src/main/resources/configdefinitions/http-filter.def - container-core/src/main/resources/configdefinitions/metrics-presentation.def - container-core/src/main/resources/configdefinitions/mockservice.def - container-core/src/main/resources/configdefinitions/qr-logging.def - container-core/src/main/resources/configdefinitions/qr-searchers.def - container-core/src/main/resources/configdefinitions/qr-templates.def - container-core/src/main/resources/configdefinitions/qr.def - container-core/src/main/resources/configdefinitions/servlet-config.def - container-core/src/main/resources/configdefinitions/threadpool.def - container-core/src/main/resources/configdefinitions/vip-status.def - container-di/src/main/resources/configdefinitions/bundles.def - container-di/src/main/resources/configdefinitions/components.def - container-di/src/main/resources/configdefinitions/jersey-bundles.def - container-di/src/main/resources/configdefinitions/jersey-injection.def - container-disc/src/main/resources/configdefinitions/container.jdisc.config.http-server.def - container-disc/src/main/resources/configdefinitions/jdisc-bindings.def - container-disc/src/main/resources/configdefinitions/jersey-connection.def - container-disc/src/main/resources/configdefinitions/jersey-init.def - container-disc/src/main/resources/configdefinitions/jersey-web-app-pool.def - container-disc/src/main/resources/configdefinitions/metric-defaults.def - container-disc/src/main/resources/configdefinitions/score-board.def - container-messagebus/src/main/resources/configdefinitions/container-mbus.def - container-messagebus/src/main/resources/configdefinitions/session.def - container-search-and-docproc/src/main/resources/configdefinitions/application-userdata.def - container-search/src/main/resources/configdefinitions/cluster.def - container-search/src/main/resources/configdefinitions/documentdb-info.def - container-search/src/main/resources/configdefinitions/emulation.def - container-search/src/main/resources/configdefinitions/federation.def - container-search/src/main/resources/configdefinitions/fs4.def - container-search/src/main/resources/configdefinitions/index-info.def - container-search/src/main/resources/configdefinitions/keyvalue.def - container-search/src/main/resources/configdefinitions/legacy-emulation.def - container-search/src/main/resources/configdefinitions/lowercasing.def - container-search/src/main/resources/configdefinitions/measure-qps.def - container-search/src/main/resources/configdefinitions/page-templates.def - container-search/src/main/resources/configdefinitions/provider.def - container-search/src/main/resources/configdefinitions/qr-binary-cache-region.def - container-search/src/main/resources/configdefinitions/qr-binary-cache.def - container-search/src/main/resources/configdefinitions/qr-monitor.def - container-search/src/main/resources/configdefinitions/qr-quotetable.def - container-search/src/main/resources/configdefinitions/qr-start.def - container-search/src/main/resources/configdefinitions/query-profiles.def - container-search/src/main/resources/configdefinitions/rate-limiting.def - container-search/src/main/resources/configdefinitions/resolvers.def - container-search/src/main/resources/configdefinitions/rewrites.def - container-search/src/main/resources/configdefinitions/search-nodes.def - container-search/src/main/resources/configdefinitions/search-with-renderer-handler.def - container-search/src/main/resources/configdefinitions/searchchain-forward.def - container-search/src/main/resources/configdefinitions/semantic-rules.def - container-search/src/main/resources/configdefinitions/strict-contracts.def - container-search/src/main/resources/configdefinitions/timing-searcher.def - docproc/src/main/resources/configdefinitions/docproc.def - docproc/src/main/resources/configdefinitions/schemamapping.def - docproc/src/main/resources/configdefinitions/splitter-joiner-document-processor.def - fileacquirer/src/main/resources/configdefinitions/filedistributorrpc.def - jdisc_http_service/src/main/resources/configdefinitions/jdisc.http.client.http-client.def - jdisc_http_service/src/main/resources/configdefinitions/jdisc.http.connector.def - jdisc_http_service/src/main/resources/configdefinitions/jdisc.http.server.def - jdisc_http_service/src/main/resources/configdefinitions/jdisc.http.servlet-paths.def - persistence/src/main/resources/configdefinitions/persistence-rpc.def - simplemetrics/src/main/resources/configdefinitions/manager.def - statistics/src/main/resources/configdefinitions/statistics.def - vespaclient-core/src/main/resources/configdefinitions/feeder.def - vespaclient-core/src/main/resources/configdefinitions/spooler.def - docker-api/src/main/resources/configdefinitions/docker.def - DESTINATION var/db/vespa/config_server/serverdb/classes) diff --git a/jdisc_core/CMakeLists.txt b/jdisc_core/CMakeLists.txt new file mode 100644 index 00000000000..b56d6ce1e5b --- /dev/null +++ b/jdisc_core/CMakeLists.txt @@ -0,0 +1,4 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +install_fat_java_artifact(jdisc_core) + +install(FILES src/main/perl/vespa-jdisc-logfmt.1 DESTINATION man/man1) diff --git a/jdisc_http_service/CMakeLists.txt b/jdisc_http_service/CMakeLists.txt new file mode 100644 index 00000000000..8ac0b5e80fb --- /dev/null +++ b/jdisc_http_service/CMakeLists.txt @@ -0,0 +1,8 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +install_fat_java_artifact(jdisc_http_service) + +install_config_definition(src/main/resources/configdefinitions/jdisc.http.client.http-client.def) +install_config_definition(src/main/resources/configdefinitions/jdisc.http.connector.def) +install_config_definition(src/main/resources/configdefinitions/jdisc.http.server.def) +install_config_definition(src/main/resources/configdefinitions/jdisc.http.servlet-paths.def) + diff --git a/jdisc_jetty/CMakeLists.txt b/jdisc_jetty/CMakeLists.txt new file mode 100644 index 00000000000..9059f68d953 --- /dev/null +++ b/jdisc_jetty/CMakeLists.txt @@ -0,0 +1,3 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +install_java_artifact(jdisc_jetty) +install_java_artifact_dependencies(jdisc_jetty) diff --git a/logd/src/logd/CMakeLists.txt b/logd/src/logd/CMakeLists.txt index 03ade1d66c8..be3fe2d1794 100644 --- a/logd/src/logd/CMakeLists.txt +++ b/logd/src/logd/CMakeLists.txt @@ -12,4 +12,4 @@ vespa_add_library(logd STATIC DEPENDS ) vespa_generate_config(logd ../main/resources/configdefinitions/logd.def) -install(FILES ../main/resources/configdefinitions/logd.def RENAME cloud.config.log.logd.def DESTINATION var/db/vespa/config_server/serverdb/classes) +install_config_definition(../main/resources/configdefinitions/logd.def cloud.config.log.logd.def) diff --git a/logserver/CMakeLists.txt b/logserver/CMakeLists.txt new file mode 100644 index 00000000000..736878948fb --- /dev/null +++ b/logserver/CMakeLists.txt @@ -0,0 +1,4 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +install_fat_java_artifact(logserver) + +vespa_install_script(bin/logserver-start.sh vespa-logserver-start bin) diff --git a/messagebus/src/vespa/messagebus/CMakeLists.txt b/messagebus/src/vespa/messagebus/CMakeLists.txt index 7c922f36a8c..9ff3bae67c8 100644 --- a/messagebus/src/vespa/messagebus/CMakeLists.txt +++ b/messagebus/src/vespa/messagebus/CMakeLists.txt @@ -37,4 +37,4 @@ vespa_add_library(messagebus DEPENDS ) vespa_generate_config(messagebus ../../main/config/messagebus.def) -install(FILES ../../main/config/messagebus.def RENAME messagebus.messagebus.def DESTINATION var/db/vespa/config_server/serverdb/classes) +install_config_definition(../../main/config/messagebus.def messagebus.messagebus.def) diff --git a/metrics/src/vespa/metrics/CMakeLists.txt b/metrics/src/vespa/metrics/CMakeLists.txt index 27a1f25e887..6eae8cd75e4 100644 --- a/metrics/src/vespa/metrics/CMakeLists.txt +++ b/metrics/src/vespa/metrics/CMakeLists.txt @@ -24,4 +24,4 @@ vespa_add_library(metrics DEPENDS ) vespa_generate_config(metrics metricsmanager.def) -install(FILES metricsmanager.def RENAME metrics.metricsmanager.def DESTINATION var/db/vespa/config_server/serverdb/classes) +install_config_definition(metricsmanager.def metrics.metricsmanager.def) diff --git a/node-repository/CMakeLists.txt b/node-repository/CMakeLists.txt new file mode 100644 index 00000000000..24a8f7e4177 --- /dev/null +++ b/node-repository/CMakeLists.txt @@ -0,0 +1,5 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +install_fat_java_artifact(node-repository) + +install(FILES src/main/config/node-repository.xml + DESTINATION conf/configserver-app) diff --git a/orchestrator/CMakeLists.txt b/orchestrator/CMakeLists.txt new file mode 100644 index 00000000000..5ff6a7ac576 --- /dev/null +++ b/orchestrator/CMakeLists.txt @@ -0,0 +1,2 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +install_fat_java_artifact(orchestrator) diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/ServiceMonitorInstanceLookupService.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/ServiceMonitorInstanceLookupService.java index b7f1ec56541..31f4f1430d3 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/ServiceMonitorInstanceLookupService.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/ServiceMonitorInstanceLookupService.java @@ -2,11 +2,11 @@ package com.yahoo.vespa.orchestrator; import com.google.inject.Inject; -import com.yahoo.vespa.applicationmodel.ApplicationInstanceReference; import com.yahoo.vespa.applicationmodel.ApplicationInstance; +import com.yahoo.vespa.applicationmodel.ApplicationInstanceReference; import com.yahoo.vespa.applicationmodel.HostName; +import com.yahoo.vespa.service.monitor.ServiceMonitor; import com.yahoo.vespa.service.monitor.ServiceMonitorStatus; -import com.yahoo.vespa.service.monitor.SlobrokAndConfigIntersector; import java.util.List; import java.util.Map; @@ -21,24 +21,24 @@ import java.util.stream.Collectors; */ public class ServiceMonitorInstanceLookupService implements InstanceLookupService { - private final SlobrokAndConfigIntersector slobrokAndConfigIntersector; + private final ServiceMonitor serviceMonitor; @Inject - public ServiceMonitorInstanceLookupService(SlobrokAndConfigIntersector slobrokAndConfigIntersector) { - this.slobrokAndConfigIntersector = slobrokAndConfigIntersector; + public ServiceMonitorInstanceLookupService(ServiceMonitor serviceMonitor) { + this.serviceMonitor = serviceMonitor; } @Override public Optional<ApplicationInstance<ServiceMonitorStatus>> findInstanceById(ApplicationInstanceReference applicationInstanceReference) { Map<ApplicationInstanceReference, ApplicationInstance<ServiceMonitorStatus>> instanceMap - = slobrokAndConfigIntersector.queryStatusOfAllApplicationInstances(); + = serviceMonitor.queryStatusOfAllApplicationInstances(); return Optional.ofNullable(instanceMap.get(applicationInstanceReference)); } @Override public Optional<ApplicationInstance<ServiceMonitorStatus>> findInstanceByHost(HostName hostName) { Map<ApplicationInstanceReference, ApplicationInstance<ServiceMonitorStatus>> instanceMap - = slobrokAndConfigIntersector.queryStatusOfAllApplicationInstances(); + = serviceMonitor.queryStatusOfAllApplicationInstances(); List<ApplicationInstance<ServiceMonitorStatus>> applicationInstancesUsingHost = instanceMap.entrySet().stream() .filter(entry -> applicationInstanceUsesHost(entry.getValue(), hostName)) .map(Map.Entry::getValue) @@ -56,7 +56,7 @@ public class ServiceMonitorInstanceLookupService implements InstanceLookupServic @Override public Set<ApplicationInstanceReference> knownInstances() { - return slobrokAndConfigIntersector.queryStatusOfAllApplicationInstances().keySet(); + return serviceMonitor.queryStatusOfAllApplicationInstances().keySet(); } private static boolean applicationInstanceUsesHost(ApplicationInstance<ServiceMonitorStatus> applicationInstance, diff --git a/persistence/CMakeLists.txt b/persistence/CMakeLists.txt index b773fccc09a..b4cb36e8bcf 100644 --- a/persistence/CMakeLists.txt +++ b/persistence/CMakeLists.txt @@ -29,3 +29,6 @@ vespa_define_module( src/tests/proxy src/tests/spi ) + +install_fat_java_artifact(persistence) +install_config_definition(src/main/resources/configdefinitions/persistence-rpc.def persistence.persistence-rpc.def) diff --git a/searchcore/src/tests/proton/attribute/attributeflush_test.cpp b/searchcore/src/tests/proton/attribute/attributeflush_test.cpp index ecb39af61bc..ce0a6bffc0d 100644 --- a/searchcore/src/tests/proton/attribute/attributeflush_test.cpp +++ b/searchcore/src/tests/proton/attribute/attributeflush_test.cpp @@ -577,7 +577,7 @@ Test::requireThatFlushedAttributeCanBeLoaded(const HwInfo &hwInfo) { constexpr uint32_t numDocs = 100; BaseFixture f(hwInfo); - vespalib::string attrName(hwInfo.slowDisk() ? "a11slow" : "a11fast"); + vespalib::string attrName(hwInfo.disk().slow() ? "a11slow" : "a11fast"); { AttributeManagerFixture amf(f); AttributeManager &am = amf._m; @@ -606,8 +606,8 @@ Test::requireThatFlushedAttributeCanBeLoaded(const HwInfo &hwInfo) void Test::requireThatFlushedAttributeCanBeLoaded() { - TEST_DO(requireThatFlushedAttributeCanBeLoaded(HwInfo(false))); - TEST_DO(requireThatFlushedAttributeCanBeLoaded(HwInfo(true))); + TEST_DO(requireThatFlushedAttributeCanBeLoaded(HwInfo(HwInfo::Disk(0, false, false), HwInfo::Memory(0)))); + TEST_DO(requireThatFlushedAttributeCanBeLoaded(HwInfo(HwInfo::Disk(0, true, false), HwInfo::Memory(0)))); } int diff --git a/searchcore/src/tests/proton/common/hw_info_sampler/hw_info_sampler_test.cpp b/searchcore/src/tests/proton/common/hw_info_sampler/hw_info_sampler_test.cpp index f9745d7fd03..c39a8163e00 100644 --- a/searchcore/src/tests/proton/common/hw_info_sampler/hw_info_sampler_test.cpp +++ b/searchcore/src/tests/proton/common/hw_info_sampler/hw_info_sampler_test.cpp @@ -18,6 +18,7 @@ namespace { const vespalib::string test_dir = "temp"; constexpr uint64_t sampleLen = 1024 * 1024 * 40; +constexpr bool sharedDisk = false; long time_point_to_long(Clock::time_point tp) { @@ -44,11 +45,11 @@ struct Fixture TEST_F("Test that hw_info_sampler uses override info", Fixture) { - Config samplerCfg(75.0, 100.0, sampleLen); + Config samplerCfg(0, 75.0, 100.0, sampleLen, sharedDisk, 0); HwInfoSampler sampler(test_dir, samplerCfg); EXPECT_EQUAL(75.0, sampler.diskWriteSpeed()); EXPECT_NOT_EQUAL(0, time_point_to_long(sampler.sampleTime())); - EXPECT_TRUE(sampler.hwInfo().slowDisk()); + EXPECT_TRUE(sampler.hwInfo().disk().slow()); } TEST_F("Test that hw_info_sampler uses saved info", Fixture) @@ -57,16 +58,16 @@ TEST_F("Test that hw_info_sampler uses saved info", Fixture) builder.disk.writespeed = 72.0; builder.disk.sampletime = time_point_to_long(Clock::now()); f.writeConfig(builder); - Config samplerCfg(0.0, 70.0, sampleLen); + Config samplerCfg(0, 0.0, 70.0, sampleLen, sharedDisk, 0); HwInfoSampler sampler(test_dir, samplerCfg); EXPECT_EQUAL(builder.disk.writespeed, sampler.diskWriteSpeed()); EXPECT_EQUAL(builder.disk.sampletime, time_point_to_long(sampler.sampleTime())); - EXPECT_FALSE(sampler.hwInfo().slowDisk()); + EXPECT_FALSE(sampler.hwInfo().disk().slow()); } TEST_F("Test that hw_info_sampler can sample disk write speed", Fixture) { - Config samplerCfg(0.0, 100.0, sampleLen); + Config samplerCfg(0, 0.0, 100.0, sampleLen, sharedDisk, 0); HwInfoSampler sampler(test_dir, samplerCfg); ASSERT_NOT_EQUAL(0.0, sampler.diskWriteSpeed()); ASSERT_NOT_EQUAL(0, time_point_to_long(sampler.sampleTime())); @@ -76,6 +77,34 @@ TEST_F("Test that hw_info_sampler can sample disk write speed", Fixture) time_point_to_long(sampler2.sampleTime())); } +TEST_F("require that disk size can be specified", Fixture) +{ + Config samplerCfg(1024, 1.0, 0.0, sampleLen, sharedDisk, 0); + HwInfoSampler sampler(test_dir, samplerCfg); + EXPECT_EQUAL(1024u, sampler.hwInfo().disk().sizeBytes()); +} + +TEST_F("require that disk size can be sampled", Fixture) +{ + Config samplerCfg(0, 1.0, 0.0, sampleLen, sharedDisk, 0); + HwInfoSampler sampler(test_dir, samplerCfg); + EXPECT_GREATER(sampler.hwInfo().disk().sizeBytes(), 0u); +} + +TEST_F("require that memory size can be specified", Fixture) +{ + Config samplerCfg(0, 1.0, 0.0, sampleLen, sharedDisk, 1024); + HwInfoSampler sampler(test_dir, samplerCfg); + EXPECT_EQUAL(1024u, sampler.hwInfo().memory().sizeBytes()); +} + +TEST_F("require that memory size can be sampled", Fixture) +{ + Config samplerCfg(0, 1.0, 0.0, sampleLen, sharedDisk, 0); + HwInfoSampler sampler(test_dir, samplerCfg); + EXPECT_GREATER(sampler.hwInfo().memory().sizeBytes(), 0u); +} + TEST_MAIN() { vespalib::rmdir(test_dir, true); diff --git a/searchcore/src/tests/proton/server/disk_mem_usage_filter/disk_mem_usage_filter_test.cpp b/searchcore/src/tests/proton/server/disk_mem_usage_filter/disk_mem_usage_filter_test.cpp index 5d8a647f675..cfd1c37e4d1 100644 --- a/searchcore/src/tests/proton/server/disk_mem_usage_filter/disk_mem_usage_filter_test.cpp +++ b/searchcore/src/tests/proton/server/disk_mem_usage_filter/disk_mem_usage_filter_test.cpp @@ -1,9 +1,11 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/vespalib/testkit/testapp.h> +#include <vespa/searchcore/proton/common/hw_info.h> #include <vespa/searchcore/proton/server/disk_mem_usage_filter.h> using proton::DiskMemUsageFilter; +using proton::HwInfo; namespace fs = std::experimental::filesystem; @@ -16,9 +18,9 @@ struct Fixture using Config = DiskMemUsageFilter::Config; Fixture() - : _filter(64 * 1024 * 1024) + : _filter(HwInfo(HwInfo::Disk(100, false, false), HwInfo::Memory(64 * 1024 * 1024))) { - _filter.setDiskStats({.capacity = 100, .free = 100, .available=100}); + _filter.setDiskUsedSize(0); _filter.setMemoryStats(vespalib::ProcessMemoryStats(10000000, 10000001, 10000002, @@ -41,7 +43,7 @@ struct Fixture } void triggerDiskLimit() { - _filter.setDiskStats({.capacity = 100, .free = 20, .available=10}); + _filter.setDiskUsedSize(90); } void triggerMemoryLimit() @@ -76,7 +78,7 @@ TEST_F("Check that disk limit can be reached", Fixture) "action: \"add more content nodes\", " "reason: \"disk used (0.9) > disk limit (0.8)\", " "stats: { " - "capacity: 100, free: 20, available: 10, diskUsed: 0.9, diskLimit: 0.8}}"); + "capacity: 100, used: 90, diskUsed: 0.9, diskLimit: 0.8}}"); } TEST_F("Check that memory limit can be reached", Fixture) @@ -108,7 +110,7 @@ TEST_F("Check that both disk limit and memory limit can be reached", Fixture) "action: \"add more content nodes\", " "reason: \"disk used (0.9) > disk limit (0.8)\", " "stats: { " - "capacity: 100, free: 20, available: 10, diskUsed: 0.9, diskLimit: 0.8}}"); + "capacity: 100, used: 90, diskUsed: 0.9, diskLimit: 0.8}}"); } TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchcore/src/vespa/searchcore/config/CMakeLists.txt b/searchcore/src/vespa/searchcore/config/CMakeLists.txt index c7669efc876..3d62309161c 100644 --- a/searchcore/src/vespa/searchcore/config/CMakeLists.txt +++ b/searchcore/src/vespa/searchcore/config/CMakeLists.txt @@ -4,11 +4,11 @@ vespa_add_library(searchcore_fconfig STATIC DEPENDS ) vespa_generate_config(searchcore_fconfig partitions.def) -install(FILES partitions.def RENAME vespa.config.search.core.partitions.def DESTINATION var/db/vespa/config_server/serverdb/classes) +install_config_definition(partitions.def vespa.config.search.core.partitions.def) vespa_generate_config(searchcore_fconfig fdispatchrc.def) -install(FILES fdispatchrc.def RENAME vespa.config.search.core.fdispatchrc.def DESTINATION var/db/vespa/config_server/serverdb/classes) +install_config_definition(fdispatchrc.def vespa.config.search.core.fdispatchrc.def) vespa_generate_config(searchcore_fconfig proton.def) -install(FILES proton.def RENAME vespa.config.search.core.proton.def DESTINATION var/db/vespa/config_server/serverdb/classes) +install_config_definition(proton.def vespa.config.search.core.proton.def) vespa_generate_config(searchcore_fconfig ranking-constants.def) -install(FILES ranking-constants.def RENAME vespa.config.search.core.ranking-constants.def DESTINATION var/db/vespa/config_server/serverdb/classes) +install_config_definition(ranking-constants.def vespa.config.search.core.ranking-constants.def) vespa_generate_config(searchcore_fconfig hwinfo.def) diff --git a/searchcore/src/vespa/searchcore/proton/attribute/flushableattribute.cpp b/searchcore/src/vespa/searchcore/proton/attribute/flushableattribute.cpp index 43dc3237685..a658b11263a 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/flushableattribute.cpp +++ b/searchcore/src/vespa/searchcore/proton/attribute/flushableattribute.cpp @@ -92,7 +92,7 @@ FlushableAttribute::Flusher::saveAttribute() _syncToken); bool saveSuccess = true; if (_saver && _saver->hasGenerationGuard() && - _fattr._hwInfo.slowDisk()) { + _fattr._hwInfo.disk().slow()) { saveSuccess = _saver->save(_saveTarget); _saver.reset(); } diff --git a/searchcore/src/vespa/searchcore/proton/common/CMakeLists.txt b/searchcore/src/vespa/searchcore/proton/common/CMakeLists.txt index dec686d3070..cdca446f114 100644 --- a/searchcore/src/vespa/searchcore/proton/common/CMakeLists.txt +++ b/searchcore/src/vespa/searchcore/proton/common/CMakeLists.txt @@ -22,4 +22,5 @@ vespa_add_library(searchcore_pcommon STATIC DEPENDS searchcore_proton_metrics searchcore_fconfig + stdc++fs ) diff --git a/searchcore/src/vespa/searchcore/proton/common/hw_info.h b/searchcore/src/vespa/searchcore/proton/common/hw_info.h index 06efaec18f2..12b2d0cfbe5 100644 --- a/searchcore/src/vespa/searchcore/proton/common/hw_info.h +++ b/searchcore/src/vespa/searchcore/proton/common/hw_info.h @@ -2,6 +2,8 @@ #pragma once +#include <cstdint> + namespace proton { /* @@ -9,19 +11,48 @@ namespace proton { */ class HwInfo { - bool _slowDisk; +public: + class Disk { + private: + uint64_t _sizeBytes; + bool _slow; + bool _shared; + public: + Disk(uint64_t sizeBytes_, bool slow_, bool shared_) + : _sizeBytes(sizeBytes_), _slow(slow_), _shared(shared_) {} + uint64_t sizeBytes() const { return _sizeBytes; } + bool slow() const { return _slow; } + bool shared() const { return _shared; } + }; + + class Memory { + private: + uint64_t _sizeBytes; + public: + Memory(uint64_t sizeBytes_) : _sizeBytes(sizeBytes_) {} + uint64_t sizeBytes() const { return _sizeBytes; } + }; + +private: + Disk _disk; + Memory _memory; + public: HwInfo() - : _slowDisk(false) + : _disk(0, false, false), + _memory(0) { } - HwInfo(bool slowDisk_in) - : _slowDisk(slowDisk_in) + HwInfo(const Disk &disk_, + const Memory &memory_) + : _disk(disk_), + _memory(memory_) { } - bool slowDisk() const { return _slowDisk; } + const Disk &disk() const { return _disk; } + const Memory &memory() const { return _memory; } }; } diff --git a/searchcore/src/vespa/searchcore/proton/common/hw_info_sampler.cpp b/searchcore/src/vespa/searchcore/proton/common/hw_info_sampler.cpp index 73bb20c712a..ccb88714a20 100644 --- a/searchcore/src/vespa/searchcore/proton/common/hw_info_sampler.cpp +++ b/searchcore/src/vespa/searchcore/proton/common/hw_info_sampler.cpp @@ -1,13 +1,14 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "hw_info_sampler.h" -#include <vespa/config/config.h> #include <vespa/config/common/configholder.h> +#include <vespa/config/config.h> #include <vespa/config/file/filesource.h> -#include <vespa/searchcore/config/config-hwinfo.h> #include <vespa/config/print/fileconfigwriter.h> -#include <vespa/vespalib/io/fileutil.h> #include <vespa/fastos/file.h> +#include <vespa/searchcore/config/config-hwinfo.h> +#include <vespa/vespalib/io/fileutil.h> +#include <experimental/filesystem> using config::ConfigHandle; using config::ConfigSubscriber; @@ -22,6 +23,26 @@ namespace proton { namespace { +uint64_t +sampleDiskSizeBytes(const std::string &pathStr, const HwInfoSampler::Config &cfg) +{ + if (cfg.diskSizeBytes != 0) { + return cfg.diskSizeBytes; + } + std::experimental::filesystem::path path(pathStr); + auto space_info = std::experimental::filesystem::space(path); + return space_info.capacity; +} + +uint64_t +sampleMemorySizeBytes(const HwInfoSampler::Config &cfg) +{ + if (cfg.memorySizeBytes != 0) { + return cfg.memorySizeBytes; + } + return sysconf(_SC_PHYS_PAGES) * sysconf(_SC_PAGESIZE); +} + std::unique_ptr<HwinfoConfig> readConfig(const vespalib::string &path) { FileSpec spec(path + "/" + "hwinfo.cfg"); ConfigSubscriber s(spec); @@ -80,8 +101,29 @@ HwInfoSampler::HwInfoSampler(const vespalib::string &path, _sampleTime(), _diskWriteSpeed(0.0) { - if (config._diskWriteSpeedOverride != 0) { - _diskWriteSpeed = config._diskWriteSpeedOverride; + setDiskWriteSpeed(path, config); + setup(HwInfo::Disk(sampleDiskSizeBytes(path, config), + (_diskWriteSpeed < config.slowWriteSpeedLimit), + config.diskShared), + HwInfo::Memory(sampleMemorySizeBytes(config))); + +} + +HwInfoSampler::~HwInfoSampler() +{ +} + +void +HwInfoSampler::setup(const HwInfo::Disk &disk, const HwInfo::Memory &memory) +{ + _hwInfo = HwInfo(disk, memory); +} + +void +HwInfoSampler::setDiskWriteSpeed(const vespalib::string &path, const Config &config) +{ + if (config.diskWriteSpeedOverride != 0) { + _diskWriteSpeed = config.diskWriteSpeedOverride; _sampleTime = Clock::now(); } else { auto cfg = readConfig(path); @@ -89,28 +131,16 @@ HwInfoSampler::HwInfoSampler(const vespalib::string &path, _sampleTime = std::chrono::time_point<Clock>(std::chrono::seconds(cfg->disk.sampletime)); _diskWriteSpeed = cfg->disk.writespeed; } else { - sample(path, config); + sampleDiskWriteSpeed(path, config); } } - setup(config); -} - -HwInfoSampler::~HwInfoSampler() -{ -} - -void -HwInfoSampler::setup(const Config &config) -{ - bool slowDisk = _diskWriteSpeed < config._slowWriteSpeedLimit; - _hwInfo = HwInfo(slowDisk); } void -HwInfoSampler::sample(const vespalib::string &path, const Config &config) +HwInfoSampler::sampleDiskWriteSpeed(const vespalib::string &path, const Config &config) { size_t minDiskWriteLen = 1024u * 1024u; - size_t diskWriteLen = config._diskSampleWriteSize; + size_t diskWriteLen = config.diskSampleWriteSize; diskWriteLen = std::max(diskWriteLen, minDiskWriteLen); _sampleTime = Clock::now(); _diskWriteSpeed = measureDiskWriteSpeed(path, diskWriteLen); diff --git a/searchcore/src/vespa/searchcore/proton/common/hw_info_sampler.h b/searchcore/src/vespa/searchcore/proton/common/hw_info_sampler.h index 22af2d32786..b640fd50370 100644 --- a/searchcore/src/vespa/searchcore/proton/common/hw_info_sampler.h +++ b/searchcore/src/vespa/searchcore/proton/common/hw_info_sampler.h @@ -16,16 +16,25 @@ class HwInfoSampler { public: struct Config { - double _diskWriteSpeedOverride; - double _slowWriteSpeedLimit; - uint64_t _diskSampleWriteSize; - - Config(double diskWriteSpeedOverride, - double slowWriteSpeedLimit, - double diskSampleWriteSize) - : _diskWriteSpeedOverride(diskWriteSpeedOverride), - _slowWriteSpeedLimit(slowWriteSpeedLimit), - _diskSampleWriteSize(diskSampleWriteSize) + uint64_t diskSizeBytes; + double diskWriteSpeedOverride; + double slowWriteSpeedLimit; + uint64_t diskSampleWriteSize; + bool diskShared; + uint64_t memorySizeBytes; + + Config(uint64_t diskSizeBytes_, + double diskWriteSpeedOverride_, + double slowWriteSpeedLimit_, + double diskSampleWriteSize_, + bool diskShared_, + uint64_t memorySizeBytes_) + : diskSizeBytes(diskSizeBytes_), + diskWriteSpeedOverride(diskWriteSpeedOverride_), + slowWriteSpeedLimit(slowWriteSpeedLimit_), + diskSampleWriteSize(diskSampleWriteSize_), + diskShared(diskShared_), + memorySizeBytes(memorySizeBytes_) { } }; @@ -36,8 +45,9 @@ private: Clock::time_point _sampleTime; double _diskWriteSpeed; - void setup(const Config &config); - void sample(const vespalib::string &path, const Config &config); + void setup(const HwInfo::Disk &disk, const HwInfo::Memory &memory); + void setDiskWriteSpeed(const vespalib::string &path, const Config &config); + void sampleDiskWriteSpeed(const vespalib::string &path, const Config &config); public: HwInfoSampler(const vespalib::string &path, const Config &config); ~HwInfoSampler(); diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastoreflushtarget.cpp b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastoreflushtarget.cpp index 322e71a572c..3821237233e 100644 --- a/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastoreflushtarget.cpp +++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastoreflushtarget.cpp @@ -87,7 +87,7 @@ DocumentMetaStoreFlushTarget::Flusher::saveDocumentMetaStore() SerialNumFileHeaderContext fileHeaderContext(_dmsft._fileHeaderContext, _syncToken); bool saveSuccess = false; - if (_dmsft._hwInfo.slowDisk()) { + if (_dmsft._hwInfo.disk().slow()) { search::AttributeMemorySaveTarget memorySaveTarget; saveSuccess = _saver->save(memorySaveTarget); _saver.reset(); diff --git a/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_filter.cpp b/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_filter.cpp index d489f477df2..32b2b6aaba6 100644 --- a/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_filter.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_filter.cpp @@ -3,6 +3,7 @@ #include "disk_mem_usage_filter.h" #include "i_disk_mem_usage_listener.h" #include <vespa/log/log.h> +#include <vespa/searchcore/proton/common/hw_info.h> LOG_SETUP(".proton.server.disk_mem_usage_filter"); @@ -43,12 +44,12 @@ void makeDiskStatsMessage(std::ostream &os, double diskUsed, double diskLimit, - const DiskMemUsageFilter::space_info &diskStats) + const HwInfo &hwInfo, + uint64_t usedDiskSizeBytes) { os << "stats: { "; - os << "capacity: " << diskStats.capacity << ", "; - os << "free: " << diskStats.free << ", "; - os << "available: " << diskStats.available << ", "; + os << "capacity: " << hwInfo.disk().sizeBytes() << ", "; + os << "used: " << usedDiskSizeBytes << ", "; os << "diskUsed: " << diskUsed << ", "; os << "diskLimit: " << diskLimit << "}"; } @@ -57,31 +58,31 @@ void makeDiskLimitMessage(std::ostream &os, double diskUsed, double diskLimit, - const DiskMemUsageFilter::space_info &diskStats) + const HwInfo &hwInfo, + uint64_t usedDiskSizeBytes) { os << "diskLimitReached: { "; os << "action: \"add more content nodes\", "; os << "reason: \"disk used (" << diskUsed << ") > disk limit (" << diskLimit << ")\", "; - makeDiskStatsMessage(os, diskUsed, diskLimit, diskStats); + makeDiskStatsMessage(os, diskUsed, diskLimit, hwInfo, usedDiskSizeBytes); os << "}"; } - vespalib::string makeUnblockingMessage(double memoryUsed, double memoryLimit, const vespalib::ProcessMemoryStats &memoryStats, - uint64_t physicalMemory, + const HwInfo &hwInfo, double diskUsed, double diskLimit, - const DiskMemUsageFilter::space_info &diskStats) + uint64_t usedDiskSizeBytes) { std::ostringstream os; os << "memoryLimitOK: { "; - makeMemoryStatsMessage(os, memoryUsed, memoryLimit, memoryStats, physicalMemory); + makeMemoryStatsMessage(os, memoryUsed, memoryLimit, memoryStats, hwInfo.memory().sizeBytes()); os << "}, "; os << "diskLimitOK: { "; - makeDiskStatsMessage(os, diskUsed, diskLimit, diskStats); + makeDiskStatsMessage(os, diskUsed, diskLimit, hwInfo, usedDiskSizeBytes); os << "}"; return os.str(); } @@ -97,7 +98,7 @@ DiskMemUsageFilter::recalcState(const Guard &guard) if (memoryUsed > _config._memoryLimit) { hasMessage = true; makeMemoryLimitMessage(message, memoryUsed, - _config._memoryLimit, _memoryStats, _physicalMemory); + _config._memoryLimit, _memoryStats, _hwInfo.memory().sizeBytes()); } double diskUsed = getDiskUsedRatio(guard); if (diskUsed > _config._diskLimit) { @@ -105,7 +106,7 @@ DiskMemUsageFilter::recalcState(const Guard &guard) message << ", "; } hasMessage = true; - makeDiskLimitMessage(message, diskUsed, _config._diskLimit, _diskStats); + makeDiskLimitMessage(message, diskUsed, _config._diskLimit, _hwInfo, _diskUsedSizeBytes); } if (hasMessage) { if (_acceptWrite) { @@ -118,10 +119,10 @@ DiskMemUsageFilter::recalcState(const Guard &guard) vespalib::string unblockMsg = makeUnblockingMessage(memoryUsed, _config._memoryLimit, _memoryStats, - _physicalMemory, + _hwInfo, diskUsed, _config._diskLimit, - _diskStats); + _diskUsedSizeBytes); LOG(info, "Write operations are now un-blocked: '%s'", unblockMsg.c_str()); } _state = State(); @@ -137,23 +138,23 @@ DiskMemUsageFilter::getMemoryUsedRatio(const Guard &guard) const { (void) guard; uint64_t unscaledMemoryUsed = _memoryStats.getAnonymousRss(); - return static_cast<double>(unscaledMemoryUsed) / _physicalMemory; + return static_cast<double>(unscaledMemoryUsed) / _hwInfo.memory().sizeBytes(); } double DiskMemUsageFilter::getDiskUsedRatio(const Guard &guard) const { (void) guard; - double availableDiskSpaceRatio = static_cast<double>(_diskStats.available) / - static_cast<double>(_diskStats.capacity); - return 1.0 - availableDiskSpaceRatio; + double usedDiskSpaceRatio = static_cast<double>(_diskUsedSizeBytes) / + static_cast<double>(_hwInfo.disk().sizeBytes()); + return usedDiskSpaceRatio; } -DiskMemUsageFilter::DiskMemUsageFilter(uint64_t physicalMemory_in) +DiskMemUsageFilter::DiskMemUsageFilter(const HwInfo &hwInfo) : _lock(), + _hwInfo(hwInfo), _memoryStats(), - _physicalMemory(physicalMemory_in), - _diskStats(), + _diskUsedSizeBytes(), _config(), _state(), _acceptWrite(true), @@ -172,10 +173,10 @@ DiskMemUsageFilter::setMemoryStats(vespalib::ProcessMemoryStats memoryStats_in) } void -DiskMemUsageFilter::setDiskStats(space_info diskStats_in) +DiskMemUsageFilter::setDiskUsedSize(uint64_t diskUsedSizeBytes) { Guard guard(_lock); - _diskStats = diskStats_in; + _diskUsedSizeBytes = diskUsedSizeBytes; recalcState(guard); } @@ -194,11 +195,11 @@ DiskMemUsageFilter::getMemoryStats() const return _memoryStats; } -DiskMemUsageFilter::space_info -DiskMemUsageFilter::getDiskStats() const +uint64_t +DiskMemUsageFilter::getDiskUsedSize() const { Guard guard(_lock); - return _diskStats; + return _diskUsedSizeBytes; } DiskMemUsageFilter::Config diff --git a/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_filter.h b/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_filter.h index c06d2e2ab0a..4906348a54d 100644 --- a/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_filter.h +++ b/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_filter.h @@ -4,12 +4,12 @@ #include "i_disk_mem_usage_notifier.h" #include "disk_mem_usage_state.h" +#include <vespa/searchcore/proton/common/hw_info.h> #include <vespa/searchcore/proton/persistenceengine/i_resource_write_filter.h> #include <vespa/vespalib/util/process_memory_stats.h> -#include <mutex> #include <atomic> #include <experimental/filesystem> - +#include <mutex> namespace proton { @@ -39,10 +39,10 @@ public: }; private: - mutable Mutex _lock; // protect _memoryStats, _diskStats, _config, _state + mutable Mutex _lock; // protect _memoryStats, _usedDiskSizeBytes, _config, _state + HwInfo _hwInfo; vespalib::ProcessMemoryStats _memoryStats; - uint64_t _physicalMemory; - space_info _diskStats; + uint64_t _diskUsedSizeBytes; Config _config; State _state; std::atomic<bool> _acceptWrite; @@ -55,15 +55,15 @@ private: void notifyDiskMemUsage(const Guard &guard, DiskMemUsageState state); public: - DiskMemUsageFilter(uint64_t physicalMememory_in); + DiskMemUsageFilter(const HwInfo &hwInfo); ~DiskMemUsageFilter(); void setMemoryStats(vespalib::ProcessMemoryStats memoryStats_in); - void setDiskStats(space_info diskStats_in); + void setDiskUsedSize(uint64_t diskUsedSizeBytes); void setConfig(Config config); vespalib::ProcessMemoryStats getMemoryStats() const; - space_info getDiskStats() const; + uint64_t getDiskUsedSize() const; Config getConfig() const; - uint64_t getPhysicalMemory() const { return _physicalMemory; } + const HwInfo &getHwInfo() const { return _hwInfo; } double getMemoryUsedRatio() const; double getDiskUsedRatio() const; bool acceptWriteOperation() const override; diff --git a/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_sampler.cpp b/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_sampler.cpp index bacf80e69a6..ddbede13880 100644 --- a/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_sampler.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_sampler.cpp @@ -3,30 +3,16 @@ #include "disk_mem_usage_sampler.h" #include <vespa/vespalib/util/timer.h> #include <vespa/vespalib/util/lambdatask.h> +#include <experimental/filesystem> #include <unistd.h> using vespalib::makeLambdaTask; namespace proton { -namespace { - -uint64_t getPhysicalMemoryBytes() -{ - // TODO: Temporal workaround for Docker nodes. Remove when this is part of proton.cfg instead. - if (const char *memoryEnv = std::getenv("VESPA_TOTAL_MEMORY_MB")) { - uint64_t physicalMemoryMB = atoll(memoryEnv); - return physicalMemoryMB * 1024u * 1024u; - } else { - return sysconf(_SC_PHYS_PAGES) * sysconf(_SC_PAGESIZE); - } -} - -} // namespace proton:<anonymous> - DiskMemUsageSampler::DiskMemUsageSampler(const std::string &path_in, const Config &config) - : _filter(getPhysicalMemoryBytes()), + : _filter(config.hwInfo), _path(path_in), _sampleInterval(60.0), _periodicTimer() @@ -43,8 +29,8 @@ void DiskMemUsageSampler::setConfig(const Config &config) { _periodicTimer.reset(); - _filter.setConfig(config._filterConfig); - _sampleInterval = config._sampleInterval; + _filter.setConfig(config.filterConfig); + _sampleInterval = config.sampleInterval; sampleUsage(); _periodicTimer = std::make_unique<vespalib::Timer>(); _periodicTimer->scheduleAtFixedRate(makeLambdaTask([this]() @@ -59,10 +45,43 @@ DiskMemUsageSampler::sampleUsage() sampleDiskUsage(); } +namespace { + +namespace fs = std::experimental::filesystem; + +uint64_t +sampleDiskUsageOnFileSystem(const fs::path &path, const HwInfo::Disk &disk) +{ + auto space_info = fs::space(path); + uint64_t result = (space_info.capacity - space_info.available); + if (result > disk.sizeBytes()) { + return disk.sizeBytes(); + } + return result; +} + +uint64_t +sampleDiskUsageInDirectory(const fs::path &path) +{ + uint64_t result = 0; + for (const auto &elem : fs::recursive_directory_iterator(path, + fs::directory_options::skip_permission_denied)) { + if (fs::is_regular_file(elem.path()) && !fs::is_symlink(elem.path())) { + result += fs::file_size(elem.path()); + } + } + return result; +} + +} + void DiskMemUsageSampler::sampleDiskUsage() { - _filter.setDiskStats(std::experimental::filesystem::space(_path)); + const auto &disk = _filter.getHwInfo().disk(); + _filter.setDiskUsedSize(disk.shared() ? + sampleDiskUsageInDirectory(_path) : + sampleDiskUsageOnFileSystem(_path, disk)); } void diff --git a/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_sampler.h b/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_sampler.h index 198f111f052..4ed48613f6a 100644 --- a/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_sampler.h +++ b/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_sampler.h @@ -22,19 +22,24 @@ class DiskMemUsageSampler { void sampleMemoryUsage(); public: struct Config { - DiskMemUsageFilter::Config _filterConfig; - double _sampleInterval; - public: + DiskMemUsageFilter::Config filterConfig; + double sampleInterval; + HwInfo hwInfo; + Config() - : _filterConfig(), - _sampleInterval(60.0) + : filterConfig(), + sampleInterval(60.0), + hwInfo() { } - Config(double memoryLimit_in, double diskLimit_in, - double sampleInterval_in) - : _filterConfig(memoryLimit_in, diskLimit_in), - _sampleInterval(sampleInterval_in) + Config(double memoryLimit_in, + double diskLimit_in, + double sampleInterval_in, + const HwInfo &hwInfo_in) + : filterConfig(memoryLimit_in, diskLimit_in), + sampleInterval(sampleInterval_in), + hwInfo(hwInfo_in) { } }; diff --git a/searchcore/src/vespa/searchcore/proton/server/proton.cpp b/searchcore/src/vespa/searchcore/proton/server/proton.cpp index d95b0fd44d1..664808acbc0 100644 --- a/searchcore/src/vespa/searchcore/proton/server/proton.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/proton.cpp @@ -32,6 +32,7 @@ #include <vespa/searchlib/expression/forcelink.hpp> #include <vespa/log/log.h> + LOG_SETUP(".proton.server.proton"); using document::DocumentTypeRepo; @@ -75,12 +76,13 @@ setFS4Compression(const ProtonConfig & proton) } DiskMemUsageSampler::Config -diskMemUsageSamplerConfig(const ProtonConfig &proton) +diskMemUsageSamplerConfig(const ProtonConfig &proton, const HwInfo &hwInfo) { return DiskMemUsageSampler::Config( proton.writefilter.memorylimit, proton.writefilter.disklimit, - proton.writefilter.sampleinterval); + proton.writefilter.sampleinterval, + hwInfo); } } @@ -223,17 +225,22 @@ Proton::init(const BootstrapConfig::SP & configSnapshot) { assert( _initStarted && ! _initComplete ); const ProtonConfig &protonConfig = configSnapshot->getProtonConfig(); - const auto &samplerCfgArgs = protonConfig.hwinfo.disk; - HwInfoSampler::Config samplerCfg(samplerCfgArgs.writespeed, - samplerCfgArgs.slowwritespeedlimit, - samplerCfgArgs.samplewritesize); + const auto &hwDiskCfg = protonConfig.hwinfo.disk; + const auto &hwMemoryCfg = protonConfig.hwinfo.memory; + // TODO: Forward disk size when performance impact of disk usage sampling is verified + HwInfoSampler::Config samplerCfg(0, + hwDiskCfg.writespeed, + hwDiskCfg.slowwritespeedlimit, + hwDiskCfg.samplewritesize, + hwDiskCfg.shared, + hwMemoryCfg.size); _hwInfoSampler = std::make_unique<HwInfoSampler>(protonConfig.basedir, samplerCfg); _hwInfo = _hwInfoSampler->hwInfo(); setFS4Compression(protonConfig); _diskMemUsageSampler = std::make_unique<DiskMemUsageSampler> (protonConfig.basedir, - diskMemUsageSamplerConfig(protonConfig)); + diskMemUsageSamplerConfig(protonConfig, _hwInfo)); _metricsEngine.reset(new MetricsEngine()); _metricsEngine->addMetricsHook(_metricsHook); @@ -341,7 +348,7 @@ Proton::applyConfig(const BootstrapConfig::SP & configSnapshot) protonConfig.search.memory.limiter.minhits); const DocumentTypeRepo::SP repo = configSnapshot->getDocumentTypeRepoSP(); - _diskMemUsageSampler->setConfig(diskMemUsageSamplerConfig(protonConfig)); + _diskMemUsageSampler->setConfig(diskMemUsageSamplerConfig(protonConfig, _hwInfo)); if (_memoryFlushConfigUpdater) { _memoryFlushConfigUpdater->setConfig(protonConfig.flush.memory); _flushEngine->kick(); diff --git a/searchcore/src/vespa/searchcore/proton/server/resource_usage_explorer.cpp b/searchcore/src/vespa/searchcore/proton/server/resource_usage_explorer.cpp index afe9f6b85d3..33b37649ed2 100644 --- a/searchcore/src/vespa/searchcore/proton/server/resource_usage_explorer.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/resource_usage_explorer.cpp @@ -9,11 +9,10 @@ using namespace vespalib::slime; namespace proton { void -convertDiskStatsToSlime(const DiskMemUsageFilter::space_info &stats, Cursor &object) +convertDiskStatsToSlime(const HwInfo &hwInfo, uint64_t diskUsedSizeBytes, Cursor &object) { - object.setLong("capacity", stats.capacity); - object.setLong("free", stats.free); - object.setLong("available", stats.available); + object.setLong("capacity", hwInfo.disk().sizeBytes()); + object.setLong("used", diskUsedSizeBytes); } void @@ -39,12 +38,12 @@ ResourceUsageExplorer::get_state(const vespalib::slime::Inserter &inserter, bool Cursor &disk = object.setObject("disk"); disk.setDouble("usedRatio", _usageFilter.getDiskUsedRatio()); disk.setDouble("usedLimit", config._diskLimit); - convertDiskStatsToSlime(_usageFilter.getDiskStats(), disk.setObject("stats")); + convertDiskStatsToSlime(_usageFilter.getHwInfo(), _usageFilter.getDiskUsedSize(), disk.setObject("stats")); Cursor &memory = object.setObject("memory"); memory.setDouble("usedRatio", _usageFilter.getMemoryUsedRatio()); memory.setDouble("usedLimit", config._memoryLimit); - memory.setLong("physicalMemory", _usageFilter.getPhysicalMemory()); + memory.setLong("physicalMemory", _usageFilter.getHwInfo().memory().sizeBytes()); convertMemoryStatsToSlime(_usageFilter.getMemoryStats(), memory.setObject("stats")); } else { object.setDouble("disk", _usageFilter.getDiskUsedRatio()); diff --git a/searchlib/CMakeLists.txt b/searchlib/CMakeLists.txt index 787ca6ed008..d77ec346cef 100644 --- a/searchlib/CMakeLists.txt +++ b/searchlib/CMakeLists.txt @@ -225,3 +225,10 @@ vespa_define_module( src/tests/util/statefile src/tests/vespa-fileheader-inspect ) + +install_java_artifact(searchlib) +install_fat_java_artifact(searchlib) + +vespa_install_script(src/main/sh/vespa-gbdt-converter bin) +vespa_install_script(src/main/sh/vespa-treenet-converter bin) + diff --git a/searchlib/src/tests/features/prod_features_attributematch.cpp b/searchlib/src/tests/features/prod_features_attributematch.cpp index 8dcceec9a22..4ddb3170efe 100644 --- a/searchlib/src/tests/features/prod_features_attributematch.cpp +++ b/searchlib/src/tests/features/prod_features_attributematch.cpp @@ -18,6 +18,7 @@ using AVC = search::attribute::Config; using AVBT = search::attribute::BasicType; using AVCT = search::attribute::CollectionType; using CollectionType = FieldInfo::CollectionType; +using DataType = FieldInfo::DataType; void Test::testAttributeMatch() @@ -303,4 +304,10 @@ Test::testAttributeMatch() addScore("attributeMatch(wint).fieldCompleteness", 0); ASSERT_TRUE(ft.execute(exp)); } + { // tensor attribute is not allowed + FtFeatureTest ft(_factory, "attributeMatch(tensor)"); + ft.getIndexEnv().getBuilder().addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, DataType::TENSOR, "tensor"); + ASSERT_TRUE(ft.getQueryEnv().getBuilder().addAttributeNode("tensor") != nullptr); + ASSERT_TRUE(!ft.setup()); + } } diff --git a/searchlib/src/tests/fef/parameter/parameter_test.cpp b/searchlib/src/tests/fef/parameter/parameter_test.cpp index 5fa0633f56e..2cff534d289 100644 --- a/searchlib/src/tests/fef/parameter/parameter_test.cpp +++ b/searchlib/src/tests/fef/parameter/parameter_test.cpp @@ -9,6 +9,7 @@ LOG_SETUP("parameter_test"); using namespace search::fef::test; using CollectionType = search::fef::FieldInfo::CollectionType; +using DataType = search::fef::FieldInfo::DataType; namespace search { namespace fef { @@ -135,6 +136,7 @@ ParameterTest::testValidator() IndexEnvironmentBuilder builder(env); builder.addField(FieldType::INDEX, CollectionType::SINGLE, "foo") .addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "bar") + .addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, DataType::TENSOR, "tbar") .addField(FieldType::INDEX, CollectionType::ARRAY, "afoo") .addField(FieldType::INDEX, CollectionType::WEIGHTEDSET, "wfoo") .addField(FieldType::INDEX, CollectionType::SINGLE, "hybrid"); @@ -156,6 +158,8 @@ ParameterTest::testValidator() EXPECT_TRUE(validate(env, SL().add("baz"), PDS().desc().feature())); EXPECT_TRUE(validate(env, SL().add("123"), PDS().desc().number())); EXPECT_TRUE(validate(env, SL().add("baz"), PDS().desc().string())); + EXPECT_TRUE(validate(env, SL().add("tbar"), PDS().desc().attributeField(ParameterCollection::ANY))); + EXPECT_TRUE(validate(env, SL().add("tbar"), PDS().desc().attribute(ParameterCollection::ANY))); // first fail but second pass EXPECT_TRUE(validate(env, SL().add("baz"), PDS().desc().field().desc().string())); @@ -180,6 +184,8 @@ ParameterTest::testValidator() EXPECT_FALSE(validate(env, SL().add("hybrid"), PDS().desc().attributeField(ParameterCollection::ANY))); EXPECT_FALSE(validate(env, SL().add("12a"), PDS().desc().number())); EXPECT_FALSE(validate(env, SL().add("a12"), PDS().desc().number())); + EXPECT_FALSE(validate(env, SL().add("tbar"), PDS().desc().attributeField(ParameterDataTypeSet::normalTypeSet(), ParameterCollection::ANY))); + EXPECT_FALSE(validate(env, SL().add("tbar"), PDS().desc().attribute(ParameterDataTypeSet::normalTypeSet(), ParameterCollection::ANY))); // test repeat PDS d1 = PDS().desc().field().repeat(); diff --git a/searchlib/src/vespa/searchlib/config/CMakeLists.txt b/searchlib/src/vespa/searchlib/config/CMakeLists.txt index 2f34d228f60..571cd3ad9f1 100644 --- a/searchlib/src/vespa/searchlib/config/CMakeLists.txt +++ b/searchlib/src/vespa/searchlib/config/CMakeLists.txt @@ -4,4 +4,4 @@ vespa_add_library(searchlib_sconfig OBJECT DEPENDS ) vespa_generate_config(searchlib_sconfig translogserver.def) -install(FILES translogserver.def RENAME searchlib.translogserver.def DESTINATION var/db/vespa/config_server/serverdb/classes) +install_config_definition(translogserver.def searchlib.translogserver.def) diff --git a/searchlib/src/vespa/searchlib/features/agefeature.cpp b/searchlib/src/vespa/searchlib/features/agefeature.cpp index 82227344a7f..258648408f8 100644 --- a/searchlib/src/vespa/searchlib/features/agefeature.cpp +++ b/searchlib/src/vespa/searchlib/features/agefeature.cpp @@ -76,5 +76,11 @@ AgeBlueprint::createExecutor(const search::fef::IQueryEnvironment &env, vespalib return stash.create<AgeExecutor>(attribute); } +fef::ParameterDescriptions +AgeBlueprint::getDescriptions() const +{ + return fef::ParameterDescriptions().desc().attribute(fef::ParameterDataTypeSet::normalTypeSet(), fef::ParameterCollection::ANY); +} + } } diff --git a/searchlib/src/vespa/searchlib/features/agefeature.h b/searchlib/src/vespa/searchlib/features/agefeature.h index de89edd49b1..99898af910f 100644 --- a/searchlib/src/vespa/searchlib/features/agefeature.h +++ b/searchlib/src/vespa/searchlib/features/agefeature.h @@ -37,9 +37,7 @@ public: void visitDumpFeatures(const fef::IIndexEnvironment &env, fef::IDumpFeatureVisitor &) const override; fef::Blueprint::UP createInstance() const override; fef::FeatureExecutor &createExecutor(const fef::IQueryEnvironment &env, vespalib::Stash &stash) const override; - fef::ParameterDescriptions getDescriptions() const override { - return fef::ParameterDescriptions().desc().attribute(fef::ParameterCollection::ANY); - } + fef::ParameterDescriptions getDescriptions() const override; bool setup(const fef::IIndexEnvironment & env, const fef::ParameterList & params) override; }; diff --git a/searchlib/src/vespa/searchlib/features/attributefeature.cpp b/searchlib/src/vespa/searchlib/features/attributefeature.cpp index 029971b3eeb..5f03cda1869 100644 --- a/searchlib/src/vespa/searchlib/features/attributefeature.cpp +++ b/searchlib/src/vespa/searchlib/features/attributefeature.cpp @@ -426,5 +426,14 @@ AttributeBlueprint::createExecutor(const search::fef::IQueryEnvironment &env, ve } } +fef::ParameterDescriptions +AttributeBlueprint::getDescriptions() const +{ + auto dataTypeSet = fef::ParameterDataTypeSet::normalOrTensorTypeSet(); + return fef::ParameterDescriptions(). + desc().attribute(dataTypeSet, fef::ParameterCollection::ANY). + desc().attribute(dataTypeSet, fef::ParameterCollection::ANY).string(); +} + } // namespace features } // namespace search diff --git a/searchlib/src/vespa/searchlib/features/attributefeature.h b/searchlib/src/vespa/searchlib/features/attributefeature.h index 18ec54e14c2..2d206fb738e 100644 --- a/searchlib/src/vespa/searchlib/features/attributefeature.h +++ b/searchlib/src/vespa/searchlib/features/attributefeature.h @@ -28,12 +28,7 @@ public: fef::Blueprint::UP createInstance() const override; fef::FeatureExecutor &createExecutor(const fef::IQueryEnvironment &env, vespalib::Stash &stash) const override; - fef::ParameterDescriptions getDescriptions() const override{ - return fef::ParameterDescriptions(). - desc().attribute(fef::ParameterCollection::ANY). - desc().attribute(fef::ParameterCollection::ANY).string(); - } - + fef::ParameterDescriptions getDescriptions() const override; bool setup(const fef::IIndexEnvironment & env, const fef::ParameterList & params) override; }; diff --git a/searchlib/src/vespa/searchlib/features/attributematchfeature.cpp b/searchlib/src/vespa/searchlib/features/attributematchfeature.cpp index 2ff53951d8b..27c7b77ec26 100644 --- a/searchlib/src/vespa/searchlib/features/attributematchfeature.cpp +++ b/searchlib/src/vespa/searchlib/features/attributematchfeature.cpp @@ -6,6 +6,7 @@ #include <vespa/searchlib/fef/featurenamebuilder.h> #include <vespa/searchlib/fef/fieldinfo.h> #include <vespa/searchlib/fef/properties.h> +#include <vespa/searchlib/fef/parameterdescriptions.h> #include <vespa/searchcommon/attribute/attributecontent.h> #include <vespa/log/log.h> @@ -271,7 +272,8 @@ AttributeMatchBlueprint::visitDumpFeatures(const IIndexEnvironment &env, { for (uint32_t i = 0; i < env.getNumFields(); ++i) { const FieldInfo * field = env.getField(i); - if (field->type() == FieldType::ATTRIBUTE) { + if (field->type() == FieldType::ATTRIBUTE && + ParameterDataTypeSet::normalTypeSet().allowedType(field->get_data_type())) { FeatureNameBuilder fnb; fnb.baseName(getBaseName()).parameter(field->name()); visitor.visitDumpFeature(fnb.buildName()); diff --git a/searchlib/src/vespa/searchlib/features/attributematchfeature.h b/searchlib/src/vespa/searchlib/features/attributematchfeature.h index 707572abf9e..3cdc6e322a3 100644 --- a/searchlib/src/vespa/searchlib/features/attributematchfeature.h +++ b/searchlib/src/vespa/searchlib/features/attributematchfeature.h @@ -98,7 +98,7 @@ public: void visitDumpFeatures(const fef::IIndexEnvironment & env, fef::IDumpFeatureVisitor & visitor) const override; fef::Blueprint::UP createInstance() const override; fef::ParameterDescriptions getDescriptions() const override { - return fef::ParameterDescriptions().desc().attributeField(fef::ParameterCollection::ANY); + return fef::ParameterDescriptions().desc().attributeField(fef::ParameterDataTypeSet::normalTypeSet(), fef::ParameterCollection::ANY); } bool setup(const fef::IIndexEnvironment & env, const fef::ParameterList & params) override; diff --git a/searchlib/src/vespa/searchlib/features/debug_attribute_wait.cpp b/searchlib/src/vespa/searchlib/features/debug_attribute_wait.cpp index 7c42df8d9bb..20de658f7f6 100644 --- a/searchlib/src/vespa/searchlib/features/debug_attribute_wait.cpp +++ b/searchlib/src/vespa/searchlib/features/debug_attribute_wait.cpp @@ -103,6 +103,12 @@ DebugAttributeWaitBlueprint::createExecutor(const IQueryEnvironment &env, vespal return stash.create<DebugAttributeWaitExecutor>(env, attribute, _params); } +fef::ParameterDescriptions +DebugAttributeWaitBlueprint::getDescriptions() const +{ + return fef::ParameterDescriptions().desc().attribute(fef::ParameterDataTypeSet::normalTypeSet(), fef::ParameterCollection::ANY).number(); +} + //----------------------------------------------------------------------------- } // namespace features diff --git a/searchlib/src/vespa/searchlib/features/debug_attribute_wait.h b/searchlib/src/vespa/searchlib/features/debug_attribute_wait.h index fbe027f96dc..fdb23a3f374 100644 --- a/searchlib/src/vespa/searchlib/features/debug_attribute_wait.h +++ b/searchlib/src/vespa/searchlib/features/debug_attribute_wait.h @@ -28,9 +28,7 @@ public: DebugAttributeWaitBlueprint(); void visitDumpFeatures(const fef::IIndexEnvironment & env, fef::IDumpFeatureVisitor & visitor) const override; fef::Blueprint::UP createInstance() const override; - fef::ParameterDescriptions getDescriptions() const override { - return fef::ParameterDescriptions().desc().attribute(fef::ParameterCollection::ANY).number(); - } + fef::ParameterDescriptions getDescriptions() const override; bool setup(const fef::IIndexEnvironment &env, const fef::ParameterList ¶ms) override; fef::FeatureExecutor &createExecutor(const fef::IQueryEnvironment &env, vespalib::Stash &stash) const override; }; diff --git a/searchlib/src/vespa/searchlib/features/dotproductfeature.cpp b/searchlib/src/vespa/searchlib/features/dotproductfeature.cpp index cd84eadf536..cbc8a5158a2 100644 --- a/searchlib/src/vespa/searchlib/features/dotproductfeature.cpp +++ b/searchlib/src/vespa/searchlib/features/dotproductfeature.cpp @@ -293,7 +293,7 @@ DotProductBlueprint::setup(const IIndexEnvironment & env, const ParameterList & ParameterDescriptions DotProductBlueprint::getDescriptions() const { - return ParameterDescriptions().desc().attribute(ParameterCollection::ANY).string(); + return ParameterDescriptions().desc().attribute(ParameterDataTypeSet::normalTypeSet(), ParameterCollection::ANY).string(); } Blueprint::UP diff --git a/searchlib/src/vespa/searchlib/features/freshnessfeature.cpp b/searchlib/src/vespa/searchlib/features/freshnessfeature.cpp index 8b484820fb5..11ae8305e16 100644 --- a/searchlib/src/vespa/searchlib/features/freshnessfeature.cpp +++ b/searchlib/src/vespa/searchlib/features/freshnessfeature.cpp @@ -89,6 +89,12 @@ FreshnessBlueprint::createInstance() const return Blueprint::UP(new FreshnessBlueprint()); } +fef::ParameterDescriptions +FreshnessBlueprint::getDescriptions() const +{ + return fef::ParameterDescriptions().desc().attribute(fef::ParameterDataTypeSet::normalTypeSet(), fef::ParameterCollection::ANY); +} + FeatureExecutor & FreshnessBlueprint::createExecutor(const IQueryEnvironment &, vespalib::Stash &stash) const { diff --git a/searchlib/src/vespa/searchlib/features/freshnessfeature.h b/searchlib/src/vespa/searchlib/features/freshnessfeature.h index a3a893ab3fc..e156cad53ed 100644 --- a/searchlib/src/vespa/searchlib/features/freshnessfeature.h +++ b/searchlib/src/vespa/searchlib/features/freshnessfeature.h @@ -37,9 +37,7 @@ public: void visitDumpFeatures(const fef::IIndexEnvironment & env, fef::IDumpFeatureVisitor & visitor) const override; fef::Blueprint::UP createInstance() const override; - fef::ParameterDescriptions getDescriptions() const override { - return fef::ParameterDescriptions().desc().attribute(fef::ParameterCollection::ANY); - } + fef::ParameterDescriptions getDescriptions() const override; bool setup(const fef::IIndexEnvironment & env, const fef::ParameterList & params) override; fef::FeatureExecutor &createExecutor(const fef::IQueryEnvironment &env, vespalib::Stash &stash) const override; }; diff --git a/searchlib/src/vespa/searchlib/features/internal_max_reduce_prod_join_feature.cpp b/searchlib/src/vespa/searchlib/features/internal_max_reduce_prod_join_feature.cpp index d07654c0a21..92120b925eb 100644 --- a/searchlib/src/vespa/searchlib/features/internal_max_reduce_prod_join_feature.cpp +++ b/searchlib/src/vespa/searchlib/features/internal_max_reduce_prod_join_feature.cpp @@ -130,32 +130,12 @@ InternalMaxReduceProdJoinBlueprint::createInstance() const ParameterDescriptions InternalMaxReduceProdJoinBlueprint::getDescriptions() const { - return ParameterDescriptions().desc().attribute(ParameterCollection::ARRAY).string(); -} - -bool supportedAttributeType(Parameter param) { - const FieldInfo *attributeInfo = param.asField(); - if (attributeInfo == nullptr) { - return false; - } - if (attributeInfo->collection() != FieldInfo::CollectionType::ARRAY) { - return false; - } - if (attributeInfo->get_data_type() == FieldInfo::DataType::INT64) { - return true; - } - if (attributeInfo->get_data_type() == FieldInfo::DataType::INT32) { - return true; - } - return false; + return ParameterDescriptions().desc().attribute(ParameterDataTypeSet::int32OrInt64TypeSet(), ParameterCollection::ARRAY).string(); } bool InternalMaxReduceProdJoinBlueprint::setup(const IIndexEnvironment &env, const ParameterList ¶ms) { - if (!supportedAttributeType(params[0])) { - return false; - } _attribute = params[0].getValue(); _query = params[1].getValue(); describeOutput("scalar", "Internal executor for optimized execution of reduce(join(A,Q,f(x,y)(x*y)),max)"); diff --git a/searchlib/src/vespa/searchlib/features/nativeattributematchfeature.cpp b/searchlib/src/vespa/searchlib/features/nativeattributematchfeature.cpp index 1e6423f9de8..5dda159f629 100644 --- a/searchlib/src/vespa/searchlib/features/nativeattributematchfeature.cpp +++ b/searchlib/src/vespa/searchlib/features/nativeattributematchfeature.cpp @@ -118,6 +118,12 @@ NativeAttributeMatchBlueprint::createInstance() const return Blueprint::UP(new NativeAttributeMatchBlueprint()); } +fef::ParameterDescriptions +NativeAttributeMatchBlueprint::getDescriptions() const +{ + return fef::ParameterDescriptions().desc().attribute(fef::ParameterDataTypeSet::normalTypeSet(), fef::ParameterCollection::ANY).repeat(); +} + bool NativeAttributeMatchBlueprint::setup(const IIndexEnvironment & env, const ParameterList & params) diff --git a/searchlib/src/vespa/searchlib/features/nativeattributematchfeature.h b/searchlib/src/vespa/searchlib/features/nativeattributematchfeature.h index e47cbed0344..e2c9dc7aef1 100644 --- a/searchlib/src/vespa/searchlib/features/nativeattributematchfeature.h +++ b/searchlib/src/vespa/searchlib/features/nativeattributematchfeature.h @@ -94,9 +94,7 @@ public: void visitDumpFeatures(const fef::IIndexEnvironment & env, fef::IDumpFeatureVisitor & visitor) const override; fef::Blueprint::UP createInstance() const override; - fef::ParameterDescriptions getDescriptions() const override { - return fef::ParameterDescriptions().desc().attribute(fef::ParameterCollection::ANY).repeat(); - } + fef::ParameterDescriptions getDescriptions() const override; bool setup(const fef::IIndexEnvironment & env, const fef::ParameterList & params) override; fef::FeatureExecutor &createExecutor(const fef::IQueryEnvironment &env, vespalib::Stash &stash) const override; diff --git a/searchlib/src/vespa/searchlib/fef/parameterdescriptions.h b/searchlib/src/vespa/searchlib/fef/parameterdescriptions.h index d0e29cfa318..4c5d2c785cb 100644 --- a/searchlib/src/vespa/searchlib/fef/parameterdescriptions.h +++ b/searchlib/src/vespa/searchlib/fef/parameterdescriptions.h @@ -2,6 +2,7 @@ #pragma once +#include <vespa/searchcommon/common/datatype.h> #include <vector> #include <cstddef> @@ -37,16 +38,77 @@ struct ParameterCollection { }; }; +/* + * A set of accepted data types for a parameter. + */ +class ParameterDataTypeSet +{ +public: + using DataType = search::index::schema::DataType; + +private: + uint32_t _typeMask; + + static uint32_t asMask(DataType dataType) { + return (1u << static_cast<unsigned int>(dataType)); + } + static uint32_t normalTypesMask() { + return (asMask(DataType::UINT1) | + asMask(DataType::UINT2) | + asMask(DataType::UINT4) | + asMask(DataType::INT8) | + asMask(DataType::INT16) | + asMask(DataType::INT32) | + asMask(DataType::INT64) | + asMask(DataType::FLOAT) | + asMask(DataType::DOUBLE) | + asMask(DataType::STRING) | + asMask(DataType::RAW)); + } + static uint32_t allTypesMask() { + return (normalTypesMask() | + asMask(DataType::BOOLEANTREE) | + asMask(DataType::TENSOR) | + asMask(DataType::REFERENCE)); + } + ParameterDataTypeSet(uint32_t typeMask) + : _typeMask(typeMask) + { + } +public: + ParameterDataTypeSet() + : ParameterDataTypeSet(allTypesMask()) + { + } + static ParameterDataTypeSet normalTypeSet() { + return ParameterDataTypeSet(normalTypesMask()); + } + static ParameterDataTypeSet int32OrInt64TypeSet() { + return ParameterDataTypeSet(asMask(DataType::INT32) | asMask(DataType::INT64)); + } + static ParameterDataTypeSet normalOrTensorTypeSet() { + return ParameterDataTypeSet(normalTypesMask() | asMask(DataType::TENSOR)); + } + bool allowedType(DataType dataType) const { + return ((asMask(dataType) & _typeMask) != 0); + } +}; + /** * The description of a single parameter within a single * ParameterDescription object. **/ struct ParamDescItem { ParameterType::Enum type; + ParameterDataTypeSet dataTypeSet; ParameterCollection::Enum collection; ParamDescItem(ParameterType::Enum t, ParameterCollection::Enum c) - : type(t), collection(c) {} + : type(t), dataTypeSet(), collection(c) {} + ParamDescItem(ParameterType::Enum t, + ParameterDataTypeSet dts, + ParameterCollection::Enum c) + : type(t), dataTypeSet(dts), collection(c) {} }; /** @@ -101,6 +163,9 @@ private: Description & getCurrent() { return _descriptions.back(); } void addParameter(const ParamDescItem ¶m); + void addParameter(ParameterType::Enum type, ParameterDataTypeSet dataTypeSet, ParameterCollection::Enum collection) { + addParameter(ParamDescItem(type, dataTypeSet, collection)); + } void addParameter(ParameterType::Enum type, ParameterCollection::Enum collection) { addParameter(ParamDescItem(type, collection)); } @@ -141,6 +206,10 @@ public: addParameter(ParameterType::ATTRIBUTE_FIELD, collection); return *this; } + ParameterDescriptions & attributeField(ParameterDataTypeSet dataTypeSet, ParameterCollection::Enum collection) { + addParameter(ParameterType::ATTRIBUTE_FIELD, dataTypeSet, collection); + return *this; + } /** * Adds an attribute parameter to the current description. */ @@ -148,6 +217,10 @@ public: addParameter(ParameterType::ATTRIBUTE, collection); return *this; } + ParameterDescriptions & attribute(ParameterDataTypeSet dataTypeSet, ParameterCollection::Enum collection) { + addParameter(ParameterType::ATTRIBUTE, dataTypeSet, collection); + return *this; + } /** * Adds a feature parameter to the current description. */ diff --git a/searchlib/src/vespa/searchlib/fef/parametervalidator.cpp b/searchlib/src/vespa/searchlib/fef/parametervalidator.cpp index 62c9efc6739..dda5ec0b719 100644 --- a/searchlib/src/vespa/searchlib/fef/parametervalidator.cpp +++ b/searchlib/src/vespa/searchlib/fef/parametervalidator.cpp @@ -25,6 +25,10 @@ bool checkCollectionType(ParameterCollection::Enum accept, CollectionType actual return false; } +bool checkDataType(ParameterDataTypeSet accept, search::index::schema::DataType actual) { + return accept.allowedType(actual); +} + class ValidateException { public: @@ -50,7 +54,9 @@ ParameterValidator::Result & ParameterValidator::Result::operator=(const Result ParameterValidator::Result::~Result() { } void -ParameterValidator::validateField(ParameterType::Enum type, ParameterCollection::Enum collection, +ParameterValidator::validateField(ParameterType::Enum type, + ParameterDataTypeSet dataTypeSet, + ParameterCollection::Enum collection, size_t i, Result & result) { const FieldInfo * field = _indexEnv.getFieldByName(_params[i]); @@ -74,6 +80,10 @@ ParameterValidator::validateField(ParameterType::Enum type, ParameterCollection: i, _params[i].c_str())); } } + if (!checkDataType(dataTypeSet, field->get_data_type())) { + throw ValidateException(make_string("Param[%zu]: field '%s' has inappropriate data type", + i, _params[i].c_str())); + } if (!checkCollectionType(collection, field->collection())) { throw ValidateException(make_string("Param[%zu]: field '%s' has inappropriate collection type", i, _params[i].c_str())); @@ -116,7 +126,7 @@ ParameterValidator::validate(const ParameterDescriptions::Description & desc) case ParameterType::INDEX_FIELD: case ParameterType::ATTRIBUTE_FIELD: case ParameterType::ATTRIBUTE: - validateField(type, param.collection, i, result); + validateField(type, param.dataTypeSet, param.collection, i, result); break; case ParameterType::NUMBER: validateNumber(type, i, result); diff --git a/searchlib/src/vespa/searchlib/fef/parametervalidator.h b/searchlib/src/vespa/searchlib/fef/parametervalidator.h index 25dc5296988..6bde59641da 100644 --- a/searchlib/src/vespa/searchlib/fef/parametervalidator.h +++ b/searchlib/src/vespa/searchlib/fef/parametervalidator.h @@ -59,7 +59,7 @@ private: const StringVector & _params; const ParameterDescriptions & _descs; - void validateField(ParameterType::Enum type, ParameterCollection::Enum collection, + void validateField(ParameterType::Enum type, ParameterDataTypeSet dataTypeSet, ParameterCollection::Enum collection, size_t i, Result & result); void validateNumber(ParameterType::Enum type, size_t i, Result & result); Result validate(const ParameterDescriptions::Description & desc); diff --git a/searchsummary/src/vespa/searchsummary/config/CMakeLists.txt b/searchsummary/src/vespa/searchsummary/config/CMakeLists.txt index 7f6dc5c4cdd..a01938d8ad4 100644 --- a/searchsummary/src/vespa/searchsummary/config/CMakeLists.txt +++ b/searchsummary/src/vespa/searchsummary/config/CMakeLists.txt @@ -4,4 +4,4 @@ vespa_add_library(searchsummary_config OBJECT DEPENDS ) vespa_generate_config(searchsummary_config juniperrc.def) -install(FILES juniperrc.def RENAME vespa.config.search.summary.juniperrc.def DESTINATION var/db/vespa/config_server/serverdb/classes) +install_config_definition(juniperrc.def vespa.config.search.summary.juniperrc.def) diff --git a/service-monitor/pom.xml b/service-monitor/pom.xml index 8d8518f2b20..13602c60cf8 100644 --- a/service-monitor/pom.xml +++ b/service-monitor/pom.xml @@ -54,6 +54,18 @@ <scope>compile</scope> </dependency> <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>config-model-api</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>config-provisioning</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> + <dependency> <groupId>org.json4s</groupId> <artifactId>json4s-native_${scala.major-version}</artifactId> <scope>test</scope> diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/ConfigServerApplication.java b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/ConfigServerApplication.java new file mode 100644 index 00000000000..5cffcec82b8 --- /dev/null +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/ConfigServerApplication.java @@ -0,0 +1,52 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.service.monitor; + +import com.yahoo.vespa.applicationmodel.ApplicationInstance; +import com.yahoo.vespa.applicationmodel.ApplicationInstanceId; +import com.yahoo.vespa.applicationmodel.ClusterId; +import com.yahoo.vespa.applicationmodel.ConfigId; +import com.yahoo.vespa.applicationmodel.HostName; +import com.yahoo.vespa.applicationmodel.ServiceCluster; +import com.yahoo.vespa.applicationmodel.ServiceInstance; +import com.yahoo.vespa.applicationmodel.ServiceType; +import com.yahoo.vespa.applicationmodel.TenantId; + +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +/** + * A service/application model of the config server with health status. + */ +public class ConfigServerApplication { + public static final ClusterId CLUSTER_ID = new ClusterId("zone-config-servers"); + public static final ServiceType SERVICE_TYPE = new ServiceType("configserver"); + public static final TenantId TENANT_ID = new TenantId("hosted-vespa"); + public static final ApplicationInstanceId APPLICATION_INSTANCE_ID = new ApplicationInstanceId("zone-config-servers"); + public static final String CONFIG_ID_PREFIX = "configid."; + + ApplicationInstance<ServiceMonitorStatus> toApplicationInstance(List<String> hostnames) { + Set<ServiceInstance<ServiceMonitorStatus>> serviceInstances = hostnames.stream() + .map(hostname -> new ServiceInstance<>( + new ConfigId(CONFIG_ID_PREFIX + hostname), + new HostName(hostname), + ServiceMonitorStatus.NOT_CHECKED)) + .collect(Collectors.toSet()); + + ServiceCluster<ServiceMonitorStatus> serviceCluster = new ServiceCluster<>( + CLUSTER_ID, + SERVICE_TYPE, + serviceInstances); + + Set<ServiceCluster<ServiceMonitorStatus>> serviceClusters = + Stream.of(serviceCluster).collect(Collectors.toSet()); + + ApplicationInstance<ServiceMonitorStatus> applicationInstance = new ApplicationInstance<>( + TENANT_ID, + APPLICATION_INSTANCE_ID, + serviceClusters); + + return applicationInstance; + } +} diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/ModelGenerator.java b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/ModelGenerator.java new file mode 100644 index 00000000000..ef418f99d47 --- /dev/null +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/ModelGenerator.java @@ -0,0 +1,119 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.service.monitor; + +import com.yahoo.config.model.api.ApplicationInfo; +import com.yahoo.config.model.api.HostInfo; +import com.yahoo.config.model.api.ServiceInfo; +import com.yahoo.config.model.api.SuperModel; +import com.yahoo.config.provision.Zone; +import com.yahoo.vespa.applicationmodel.ApplicationInstance; +import com.yahoo.vespa.applicationmodel.ApplicationInstanceId; +import com.yahoo.vespa.applicationmodel.ApplicationInstanceReference; +import com.yahoo.vespa.applicationmodel.ClusterId; +import com.yahoo.vespa.applicationmodel.ConfigId; +import com.yahoo.vespa.applicationmodel.HostName; +import com.yahoo.vespa.applicationmodel.ServiceCluster; +import com.yahoo.vespa.applicationmodel.ServiceClusterKey; +import com.yahoo.vespa.applicationmodel.ServiceInstance; +import com.yahoo.vespa.applicationmodel.ServiceType; +import com.yahoo.vespa.applicationmodel.TenantId; + +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +/** + * Util to convert SuperModel to ServiceModel and application model classes + */ +public class ModelGenerator { + public static final String CLUSTER_ID_PROPERTY_NAME = "clustername"; + + ServiceModel toServiceModel( + SuperModel superModel, + Zone zone, + List<String> configServerHosts, + SlobrokMonitor2 slobrokMonitor) { + Map<ApplicationInstanceReference, ApplicationInstance<ServiceMonitorStatus>> applicationInstances = new HashMap<>(); + + for (ApplicationInfo applicationInfo : superModel.getAllApplicationInfos()) { + ApplicationInstance<ServiceMonitorStatus> applicationInstance = toApplicationInstance( + applicationInfo, + zone, + slobrokMonitor); + applicationInstances.put(applicationInstance.reference(), applicationInstance); + } + + // The config server is part of the service model (but not super model) + ConfigServerApplication configServerApplication = new ConfigServerApplication(); + ApplicationInstance<ServiceMonitorStatus> configServerApplicationInstance = + configServerApplication.toApplicationInstance(configServerHosts); + applicationInstances.put(configServerApplicationInstance.reference(), configServerApplicationInstance); + + return new ServiceModel(applicationInstances); + } + + ApplicationInstance<ServiceMonitorStatus> toApplicationInstance( + ApplicationInfo applicationInfo, + Zone zone, + SlobrokMonitor2 slobrokMonitor) { + Map<ServiceClusterKey, Set<ServiceInstance<ServiceMonitorStatus>>> groupedServiceInstances = new HashMap<>(); + + for (HostInfo host : applicationInfo.getModel().getHosts()) { + HostName hostName = new HostName(host.getHostname()); + for (ServiceInfo serviceInfo : host.getServices()) { + ServiceClusterKey serviceClusterKey = toServiceClusterKey(serviceInfo); + ServiceInstance<ServiceMonitorStatus> serviceInstance = + toServiceInstance(serviceInfo, hostName, slobrokMonitor); + + if (!groupedServiceInstances.containsKey(serviceClusterKey)) { + groupedServiceInstances.put(serviceClusterKey, new HashSet<>()); + } + groupedServiceInstances.get(serviceClusterKey).add(serviceInstance); + } + } + + Set<ServiceCluster<ServiceMonitorStatus>> serviceClusters = groupedServiceInstances.entrySet().stream() + .map(entry -> new ServiceCluster<>( + entry.getKey().clusterId(), + entry.getKey().serviceType(), + entry.getValue())) + .collect(Collectors.toSet()); + + ApplicationInstance<ServiceMonitorStatus> applicationInstance = new ApplicationInstance<>( + new TenantId(applicationInfo.getApplicationId().tenant().toString()), + toApplicationInstanceId(applicationInfo, zone), + serviceClusters); + + return applicationInstance; + } + + ServiceClusterKey toServiceClusterKey(ServiceInfo serviceInfo) { + ClusterId clusterId = new ClusterId(serviceInfo.getProperty(CLUSTER_ID_PROPERTY_NAME).orElse("")); + ServiceType serviceType = toServiceType(serviceInfo); + return new ServiceClusterKey(clusterId, serviceType); + } + + ServiceInstance<ServiceMonitorStatus> toServiceInstance( + ServiceInfo serviceInfo, + HostName hostName, + SlobrokMonitor2 slobrokMonitor) { + ConfigId configId = new ConfigId(serviceInfo.getConfigId()); + ServiceMonitorStatus serviceStatus = slobrokMonitor.getStatus(toServiceType(serviceInfo), configId); + return new ServiceInstance<>(configId, hostName,serviceStatus); + } + + ApplicationInstanceId toApplicationInstanceId(ApplicationInfo applicationInfo, Zone zone) { + return new ApplicationInstanceId(String.format("%s:%s:%s:%s", + applicationInfo.getApplicationId().application().value(), + zone.environment().value(), + zone.region().value(), + applicationInfo.getApplicationId().instance().value())); + } + + ServiceType toServiceType(ServiceInfo serviceInfo) { + return new ServiceType(serviceInfo.getServiceType()); + } +} diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/ServiceModel.java b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/ServiceModel.java new file mode 100644 index 00000000000..b39af0238c5 --- /dev/null +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/ServiceModel.java @@ -0,0 +1,34 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.service.monitor; + +import com.yahoo.vespa.applicationmodel.ApplicationInstance; +import com.yahoo.vespa.applicationmodel.ApplicationInstanceReference; + +import java.util.Collections; +import java.util.Map; +import java.util.Optional; + +/** + * The ServiceModel is almost a mirror of the SuperModel, except that it + * also gives ServiceMonitorStatus on each service, and there may be + * artificial applications like the config server "application". + */ +// @Immutable +public class ServiceModel { + private final Map<ApplicationInstanceReference, + ApplicationInstance<ServiceMonitorStatus>> applications; + + ServiceModel(Map<ApplicationInstanceReference, + ApplicationInstance<ServiceMonitorStatus>> applications) { + this.applications = Collections.unmodifiableMap(applications); + } + + Map<ApplicationInstanceReference, + ApplicationInstance<ServiceMonitorStatus>> getAllApplicationInstances() { + return applications; + } + + Optional<ApplicationInstance<ServiceMonitorStatus>> getApplicationInstance(ApplicationInstanceReference reference) { + return Optional.ofNullable(applications.get(reference)); + } +} diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/ServiceMonitorImpl.java b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/ServiceMonitorImpl.java new file mode 100644 index 00000000000..3468644169d --- /dev/null +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/ServiceMonitorImpl.java @@ -0,0 +1,44 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.service.monitor; + +import com.google.inject.Inject; +import com.yahoo.cloud.config.ConfigserverConfig; +import com.yahoo.config.model.api.SuperModelProvider; +import com.yahoo.config.provision.Zone; +import com.yahoo.vespa.applicationmodel.ApplicationInstance; +import com.yahoo.vespa.applicationmodel.ApplicationInstanceReference; + +import java.util.List; +import java.util.Map; +import java.util.logging.Logger; +import java.util.stream.Collectors; + +public class ServiceMonitorImpl implements ServiceMonitor { + private static final Logger logger = Logger.getLogger(ServiceMonitorImpl.class.getName()); + + private final Zone zone; + private final List<String> configServerHostnames; + private final SlobrokMonitor2 slobrokMonitor = new SlobrokMonitor2(); + private final SuperModelListenerImpl superModelListener = + new SuperModelListenerImpl(slobrokMonitor); + + @Inject + public ServiceMonitorImpl(SuperModelProvider superModelProvider, + ConfigserverConfig configserverConfig) { + this.zone = superModelProvider.getZone(); + this.configServerHostnames = configserverConfig.zookeeperserver().stream() + .map(server -> server.hostname()) + .collect(Collectors.toList()); + superModelListener.start(superModelProvider); + } + + @Override + public Map<ApplicationInstanceReference, + ApplicationInstance<ServiceMonitorStatus>> queryStatusOfAllApplicationInstances() { + // If we ever need to optimize this method, then consider reusing ServiceModel snapshots + // for up to X ms. + ServiceModel serviceModel = + superModelListener.createServiceModelSnapshot(zone, configServerHostnames); + return serviceModel.getAllApplicationInstances(); + } +} diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/SlobrokMonitor2.java b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/SlobrokMonitor2.java new file mode 100644 index 00000000000..39a59826cf0 --- /dev/null +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/SlobrokMonitor2.java @@ -0,0 +1,121 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.service.monitor; + +import com.yahoo.config.model.api.ApplicationInfo; +import com.yahoo.config.model.api.HostInfo; +import com.yahoo.config.model.api.PortInfo; +import com.yahoo.config.model.api.ServiceInfo; +import com.yahoo.config.model.api.SuperModel; +import com.yahoo.jrt.Spec; +import com.yahoo.jrt.Supervisor; +import com.yahoo.jrt.Transport; +import com.yahoo.jrt.slobrok.api.Mirror; +import com.yahoo.jrt.slobrok.api.SlobrokList; +import com.yahoo.log.LogLevel; +import com.yahoo.vespa.applicationmodel.ConfigId; +import com.yahoo.vespa.applicationmodel.ServiceType; + +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import java.util.logging.Logger; + +public class SlobrokMonitor2 implements AutoCloseable { + public static final String SLOBROK_RPC_PORT_TAG = "rpc"; + + private static final Logger log = Logger.getLogger(SlobrokMonitor2.class.getName()); + + private final SlobrokList slobrokList; + private final Mirror mirror; + + SlobrokMonitor2() { + this(new SlobrokList()); + } + + // Package-private for testing. + SlobrokMonitor2(SlobrokList slobrokList, Mirror mirror) { + this.slobrokList = slobrokList; + this.mirror = mirror; + } + + private SlobrokMonitor2(SlobrokList slobrokList) { + this(slobrokList, new Mirror(new Supervisor(new Transport()), slobrokList)); + } + + void updateSlobrokList(SuperModel superModel) { + // If we ever need to optimize this method, then we should make this class + // have a Map<ApplicationId, List<String>>, mapping each application to + // its list of specs. Then, whenever a single application is activated or removed, + // only modify that List<String>. + + List<String> slobrokSpecs = new ArrayList<>(); + + for (ApplicationInfo application : superModel.getAllApplicationInfos()) { + for (HostInfo host : application.getModel().getHosts()) { + for (ServiceInfo service : host.getServices()) { + for (PortInfo port : service.getPorts()) { + if (port.getTags().contains(SLOBROK_RPC_PORT_TAG)) { + Spec spec = new Spec(host.getHostname(), port.getPort()); + slobrokSpecs.add(spec.toString()); + } + } + } + } + } + + slobrokList.setup(slobrokSpecs.toArray(new String[0])); + } + + ServiceMonitorStatus getStatus(ServiceType serviceType, ConfigId configId) { + Optional<String> slobrokServiceName = lookup(serviceType, configId); + if (slobrokServiceName.isPresent()) { + if (mirror.lookup(slobrokServiceName.get()).length != 0) { + return ServiceMonitorStatus.UP; + } else { + return ServiceMonitorStatus.DOWN; + } + } else { + return ServiceMonitorStatus.NOT_CHECKED; + } + } + + @Override + public void close() { + mirror.shutdown(); + } + + // Package-private for testing + Optional<String> lookup(ServiceType serviceType, ConfigId configId) { + switch (serviceType.s()) { + case "adminserver": + case "config-sentinel": + case "configproxy": + case "configserver": + case "filedistributorservice": + case "logd": + case "logserver": + case "metricsproxy": + case "slobrok": + case "transactionlogserver": + return Optional.empty(); + + case "topleveldispatch": + return Optional.of(configId.s()); + + case "qrserver": + case "container": + case "docprocservice": + case "container-clustercontroller": + return Optional.of("vespa/service/" + configId.s()); + + case "searchnode": //TODO: handle only as storagenode instead of both as searchnode/storagenode + return Optional.of(configId.s() + "/realtimecontroller"); + case "distributor": + case "storagenode": + return Optional.of("storage/cluster." + configId.s()); + default: + log.log(LogLevel.DEBUG, "Unknown service type " + serviceType.s() + " with config id " + configId.s()); + return Optional.empty(); + } + } +} diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/SuperModelListenerImpl.java b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/SuperModelListenerImpl.java new file mode 100644 index 00000000000..8beb90c382a --- /dev/null +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/SuperModelListenerImpl.java @@ -0,0 +1,67 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.service.monitor; + +import com.yahoo.config.model.api.SuperModel; +import com.yahoo.config.model.api.SuperModelListener; +import com.yahoo.config.model.api.SuperModelProvider; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.Zone; + +import java.util.List; +import java.util.logging.Logger; + +public class SuperModelListenerImpl implements SuperModelListener { + private static final Logger logger = Logger.getLogger(SuperModelListenerImpl.class.getName()); + + // Guard for updating superModel and slobrokMonitor exclusively and atomically: + // - superModel and slobrokMonitor must be updated in combination (exclusively and atomically) + // - Anyone may take a snapshot of superModel for reading purposes, hence volatile. + private final Object monitor = new Object(); + private final SlobrokMonitor2 slobrokMonitor; + private volatile SuperModel superModel; + + SuperModelListenerImpl(SlobrokMonitor2 slobrokMonitor) { + this.slobrokMonitor = slobrokMonitor; + } + + void start(SuperModelProvider superModelProvider) { + synchronized (monitor) { + // This snapshot() call needs to be within the synchronized block, + // since applicationActivated()/applicationRemoved() may be called + // asynchronously even before snapshot() returns. + SuperModel snapshot = superModelProvider.snapshot(this); + exclusiveUpdate(snapshot); + } + } + + @Override + public void applicationActivated(SuperModel superModel, ApplicationId applicationId) { + synchronized (monitor) { + exclusiveUpdate(superModel); + } + } + + @Override + public void applicationRemoved(SuperModel superModel, ApplicationId id) { + synchronized (monitor) { + exclusiveUpdate(superModel); + } + } + + ServiceModel createServiceModelSnapshot(Zone zone, List<String> configServerHostnames) { + // Save a snapshot of volatile this.superModel outside of synchronized block. + SuperModel superModelSnapshot = this.superModel; + + ModelGenerator modelGenerator = new ModelGenerator(); + return modelGenerator.toServiceModel( + superModelSnapshot, + zone, + configServerHostnames, + slobrokMonitor); + } + + private void exclusiveUpdate(SuperModel superModel) { + this.superModel = superModel; + slobrokMonitor.updateSlobrokList(superModel); + } +}
\ No newline at end of file diff --git a/service-monitor/src/test/java/com/yahoo/vespa/service/monitor/ConfigServerApplicationTest.java b/service-monitor/src/test/java/com/yahoo/vespa/service/monitor/ConfigServerApplicationTest.java new file mode 100644 index 00000000000..ec91507c846 --- /dev/null +++ b/service-monitor/src/test/java/com/yahoo/vespa/service/monitor/ConfigServerApplicationTest.java @@ -0,0 +1,61 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.service.monitor; + +import com.yahoo.vespa.applicationmodel.ApplicationInstance; +import org.junit.Test; + +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +public class ConfigServerApplicationTest { + private static final String configServer1 = "cfg1.yahoo.com"; + private static final String configServer2 = "cfg2.yahoo.com"; + private static final String configServer3 = "cfg3.yahoo.com"; + private static final List<String> configServerList = Stream.of( + configServer1, + configServer2, + configServer3).collect(Collectors.toList()); + + @Test + public void toApplicationInstance() throws Exception { + ConfigServerApplication application = new ConfigServerApplication(); + ApplicationInstance<ServiceMonitorStatus> applicationInstance = + application.toApplicationInstance(configServerList); + + // Backward compatibility check + assertEquals( + SlobrokAndConfigIntersector.configServerApplicationInstanceId(), + applicationInstance.applicationInstanceId()); + assertEquals( + SlobrokAndConfigIntersector.syntheticHostedVespaTenantId(), + applicationInstance.tenantId()); + + assertEquals( + ConfigServerApplication.TENANT_ID.toString() + + ":" + ConfigServerApplication.APPLICATION_INSTANCE_ID, + applicationInstance.reference().toString()); + + assertEquals( + ConfigServerApplication.CLUSTER_ID, + applicationInstance.serviceClusters().iterator().next().clusterId()); + + assertEquals( + ServiceMonitorStatus.NOT_CHECKED, + applicationInstance + .serviceClusters().iterator().next() + .serviceInstances().iterator().next() + .serviceStatus()); + + assertTrue(configServerList.contains( + applicationInstance + .serviceClusters().iterator().next() + .serviceInstances().iterator().next() + .hostName() + .toString())); + } + +}
\ No newline at end of file diff --git a/service-monitor/src/test/java/com/yahoo/vespa/service/monitor/ExampleModel.java b/service-monitor/src/test/java/com/yahoo/vespa/service/monitor/ExampleModel.java new file mode 100644 index 00000000000..d9766d65d3f --- /dev/null +++ b/service-monitor/src/test/java/com/yahoo/vespa/service/monitor/ExampleModel.java @@ -0,0 +1,65 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.service.monitor; + +import com.yahoo.config.model.api.ApplicationInfo; +import com.yahoo.config.model.api.HostInfo; +import com.yahoo.config.model.api.Model; +import com.yahoo.config.model.api.PortInfo; +import com.yahoo.config.model.api.ServiceInfo; +import com.yahoo.config.model.api.SuperModel; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.ApplicationName; +import com.yahoo.config.provision.InstanceName; +import com.yahoo.config.provision.TenantName; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class ExampleModel { + + static final String CLUSTER_ID = "cluster-id"; + static final String SERVICE_NAME = "service-name"; + static final String SERVICE_TYPE = "service-type"; + static final String CONFIG_ID = "config-id"; + static final String TENANT = "tenant"; + static final String APPLICATION_NAME = "application"; + public static final String INSTANCE_NAME = "default"; + + static SuperModel createExampleSuperModelWithOneRpcPort(String hostname, int rpcPort) { + Set<String> tags = Stream.of(SlobrokMonitor2.SLOBROK_RPC_PORT_TAG, "footag") + .collect(Collectors.toSet()); + Map<String, String> properties = new HashMap<>(); + properties.put(ModelGenerator.CLUSTER_ID_PROPERTY_NAME, CLUSTER_ID); + Set<PortInfo> portInfos = Stream.of(new PortInfo(rpcPort, tags)).collect(Collectors.toSet()); + ServiceInfo serviceInfo = new ServiceInfo( + SERVICE_NAME, + SERVICE_TYPE, + portInfos, + properties, + CONFIG_ID, + hostname); + List<ServiceInfo> serviceInfos = Stream.of(serviceInfo).collect(Collectors.toList()); + HostInfo hostInfo = new HostInfo(hostname, serviceInfos); + List<HostInfo> hostInfos = Stream.of(hostInfo).collect(Collectors.toList()); + + TenantName tenantName = TenantName.from(TENANT); + ApplicationName applicationName = ApplicationName.from(APPLICATION_NAME); + InstanceName instanceName = InstanceName.from(INSTANCE_NAME); + ApplicationId applicationId = ApplicationId.from(tenantName, applicationName, instanceName); + Model model = mock(Model.class); + when(model.getHosts()).thenReturn(hostInfos); + ApplicationInfo applicationInfo = new ApplicationInfo(applicationId, 1l, model); + + Map<TenantName, Map<ApplicationId, ApplicationInfo>> applicationInfos = new HashMap<>(); + applicationInfos.put(tenantName, new HashMap<>()); + applicationInfos.get(tenantName).put(applicationId, applicationInfo); + return new SuperModel(applicationInfos); + } +} diff --git a/service-monitor/src/test/java/com/yahoo/vespa/service/monitor/ModelGeneratorTest.java b/service-monitor/src/test/java/com/yahoo/vespa/service/monitor/ModelGeneratorTest.java new file mode 100644 index 00000000000..e54fdc51a3a --- /dev/null +++ b/service-monitor/src/test/java/com/yahoo/vespa/service/monitor/ModelGeneratorTest.java @@ -0,0 +1,107 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.service.monitor; + +import com.yahoo.config.model.api.SuperModel; +import com.yahoo.config.provision.Environment; +import com.yahoo.config.provision.RegionName; +import com.yahoo.config.provision.Zone; +import com.yahoo.vespa.applicationmodel.ApplicationInstance; +import com.yahoo.vespa.applicationmodel.ApplicationInstanceReference; +import com.yahoo.vespa.applicationmodel.ServiceCluster; +import com.yahoo.vespa.applicationmodel.ServiceInstance; +import org.junit.Test; + +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static org.junit.Assert.assertEquals; +import static org.mockito.Matchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class ModelGeneratorTest { + private final String ENVIRONMENT = "prod"; + private final String REGION = "us-west-1"; + private final String HOSTNAME = "hostname"; + private final int PORT = 2; + + @Test + public void toApplicationModel() throws Exception { + SuperModel superModel = + ExampleModel.createExampleSuperModelWithOneRpcPort(HOSTNAME, PORT); + ModelGenerator modelGenerator = new ModelGenerator(); + + Zone zone = new Zone(Environment.from(ENVIRONMENT), RegionName.from(REGION)); + + List<String> configServerHosts = Stream.of("cfg1", "cfg2", "cfg3") + .collect(Collectors.toList()); + + SlobrokMonitor2 slobrokMonitor = mock(SlobrokMonitor2.class); + when(slobrokMonitor.getStatus(any(), any())).thenReturn(ServiceMonitorStatus.UP); + + ServiceModel serviceModel = + modelGenerator.toServiceModel( + superModel, + zone, + configServerHosts, + slobrokMonitor); + + Map<ApplicationInstanceReference, + ApplicationInstance<ServiceMonitorStatus>> applicationInstances = + serviceModel.getAllApplicationInstances(); + + assertEquals(2, applicationInstances.size()); + + Iterator<Map.Entry<ApplicationInstanceReference, + ApplicationInstance<ServiceMonitorStatus>>> iterator = + applicationInstances.entrySet().iterator(); + + ApplicationInstance<ServiceMonitorStatus> applicationInstance1 = iterator.next().getValue(); + ApplicationInstance<ServiceMonitorStatus> applicationInstance2 = iterator.next().getValue(); + + if (applicationInstance1.applicationInstanceId().equals( + ConfigServerApplication.APPLICATION_INSTANCE_ID)) { + verifyConfigServerApplication(applicationInstance1); + verifyOtherApplication(applicationInstance2); + } else { + verifyConfigServerApplication(applicationInstance2); + verifyOtherApplication(applicationInstance1); + } + } + + private void verifyOtherApplication(ApplicationInstance<ServiceMonitorStatus> applicationInstance) { + assertEquals(String.format("%s:%s:%s:%s:%s", + ExampleModel.TENANT, + ExampleModel.APPLICATION_NAME, + ENVIRONMENT, + REGION, + ExampleModel.INSTANCE_NAME), + applicationInstance.reference().toString()); + + assertEquals(ExampleModel.TENANT, applicationInstance.tenantId().toString()); + Set<ServiceCluster<ServiceMonitorStatus>> serviceClusters = + applicationInstance.serviceClusters(); + assertEquals(1, serviceClusters.size()); + ServiceCluster<ServiceMonitorStatus> serviceCluster = serviceClusters.iterator().next(); + assertEquals(ExampleModel.CLUSTER_ID, serviceCluster.clusterId().toString()); + assertEquals(ExampleModel.SERVICE_TYPE, serviceCluster.serviceType().toString()); + Set<ServiceInstance<ServiceMonitorStatus>> serviceInstances = + serviceCluster.serviceInstances(); + assertEquals(1, serviceClusters.size()); + ServiceInstance<ServiceMonitorStatus> serviceInstance = serviceInstances.iterator().next(); + assertEquals(HOSTNAME, serviceInstance.hostName().toString()); + assertEquals(ExampleModel.CONFIG_ID, serviceInstance.configId().toString()); + assertEquals(ServiceMonitorStatus.UP, serviceInstance.serviceStatus()); + } + + private void verifyConfigServerApplication( + ApplicationInstance<ServiceMonitorStatus> applicationInstance) { + assertEquals(ConfigServerApplication.APPLICATION_INSTANCE_ID, + applicationInstance.applicationInstanceId()); + } + +}
\ No newline at end of file diff --git a/service-monitor/src/test/java/com/yahoo/vespa/service/monitor/SlobrokMonitor2Test.java b/service-monitor/src/test/java/com/yahoo/vespa/service/monitor/SlobrokMonitor2Test.java new file mode 100644 index 00000000000..242112694eb --- /dev/null +++ b/service-monitor/src/test/java/com/yahoo/vespa/service/monitor/SlobrokMonitor2Test.java @@ -0,0 +1,58 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.service.monitor; + +import com.yahoo.config.model.api.SuperModel; +import com.yahoo.jrt.slobrok.api.Mirror; +import com.yahoo.jrt.slobrok.api.SlobrokList; +import com.yahoo.vespa.applicationmodel.ConfigId; +import com.yahoo.vespa.applicationmodel.ServiceType; +import org.junit.Test; + +import java.util.Optional; + +import static org.junit.Assert.assertEquals; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +public class SlobrokMonitor2Test { + private final SlobrokList slobrokList = mock(SlobrokList.class); + private final Mirror mirror = mock(Mirror.class); + private SlobrokMonitor2 slobrokMonitor = new SlobrokMonitor2(slobrokList, mirror); + + @Test + public void testLookup() { + assertEquals( + Optional.of("config.id"), + lookup("topleveldispatch", "config.id")); + + assertEquals( + Optional.empty(), + lookup("adminserver", "config.id")); + } + + private Optional<String> lookup(String serviceType, String configId) { + return slobrokMonitor.lookup(new ServiceType(serviceType), new ConfigId(configId)); + } + + @Test + public void testGetStatus() { + ServiceType serviceType = new ServiceType("topleveldispatch"); + ConfigId configId = new ConfigId("config.id"); + when(mirror.lookup("config.id")).thenReturn(new Mirror.Entry[1]); + assertEquals(ServiceMonitorStatus.UP, slobrokMonitor.getStatus(serviceType, configId)); + } + + @Test + public void testUpdateSlobrokList() { + final String hostname = "hostname"; + final int port = 1; + + SuperModel superModel = ExampleModel.createExampleSuperModelWithOneRpcPort(hostname, port); + slobrokMonitor.updateSlobrokList(superModel); + + String[] expectedSpecs = new String[] {"tcp/" + hostname + ":" + port}; + verify(slobrokList).setup(expectedSpecs); + } + +}
\ No newline at end of file diff --git a/service-monitor/src/test/java/com/yahoo/vespa/service/monitor/SuperModelListenerImplTest.java b/service-monitor/src/test/java/com/yahoo/vespa/service/monitor/SuperModelListenerImplTest.java new file mode 100644 index 00000000000..07e84b22d72 --- /dev/null +++ b/service-monitor/src/test/java/com/yahoo/vespa/service/monitor/SuperModelListenerImplTest.java @@ -0,0 +1,29 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package com.yahoo.vespa.service.monitor; + +import com.yahoo.config.model.api.SuperModel; +import com.yahoo.config.provision.ApplicationId; +import org.junit.Test; + +import static org.mockito.Mockito.doNothing; +import static org.mockito.Mockito.mock; + +public class SuperModelListenerImplTest { + private final SlobrokMonitor2 slobrokMonitor = mock(SlobrokMonitor2.class); + private final SuperModel superModel = mock(SuperModel.class); + private final ApplicationId applicationId = ApplicationId.defaultId(); + private final SuperModelListenerImpl listener = new SuperModelListenerImpl(slobrokMonitor); + + @Test + public void testActivateApplication() { + listener.applicationActivated(superModel, applicationId); + doNothing().when(slobrokMonitor).updateSlobrokList(superModel); + } + + @Test + public void testRemoveApplication() { + listener.applicationRemoved(superModel, applicationId); + doNothing().when(slobrokMonitor).updateSlobrokList(superModel); + } +}
\ No newline at end of file diff --git a/simplemetrics/CMakeLists.txt b/simplemetrics/CMakeLists.txt new file mode 100644 index 00000000000..c145fbb2ec7 --- /dev/null +++ b/simplemetrics/CMakeLists.txt @@ -0,0 +1,4 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +install_fat_java_artifact(simplemetrics) + +install_config_definition(src/main/resources/configdefinitions/manager.def metrics.manager.def) diff --git a/standalone-container/CMakeLists.txt b/standalone-container/CMakeLists.txt new file mode 100644 index 00000000000..83c58e09945 --- /dev/null +++ b/standalone-container/CMakeLists.txt @@ -0,0 +1,2 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +install_fat_java_artifact(standalone-container) diff --git a/statistics/CMakeLists.txt b/statistics/CMakeLists.txt new file mode 100644 index 00000000000..3b187b72d6e --- /dev/null +++ b/statistics/CMakeLists.txt @@ -0,0 +1,2 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +install_config_definition(src/main/resources/configdefinitions/statistics.def container.statistics.def) diff --git a/storage/src/vespa/storage/bucketdb/CMakeLists.txt b/storage/src/vespa/storage/bucketdb/CMakeLists.txt index 5c818631d54..6e3a0c2e986 100644 --- a/storage/src/vespa/storage/bucketdb/CMakeLists.txt +++ b/storage/src/vespa/storage/bucketdb/CMakeLists.txt @@ -17,6 +17,6 @@ vespa_add_library(storage_bucketdb OBJECT storage_storageconfig ) vespa_generate_config(storage_bucketdb stor-bucketdb.def) -install(FILES stor-bucketdb.def RENAME vespa.config.content.core.stor-bucketdb.def DESTINATION var/db/vespa/config_server/serverdb/classes) +install_config_definition(stor-bucketdb.def vespa.config.content.core.stor-bucketdb.def) vespa_generate_config(storage_bucketdb stor-bucket-init.def) -install(FILES stor-bucket-init.def RENAME vespa.config.content.core.stor-bucket-init.def DESTINATION var/db/vespa/config_server/serverdb/classes) +install_config_definition(stor-bucket-init.def vespa.config.content.core.stor-bucket-init.def) diff --git a/storage/src/vespa/storage/config/CMakeLists.txt b/storage/src/vespa/storage/config/CMakeLists.txt index 0f3f392f324..4a20d510043 100644 --- a/storage/src/vespa/storage/config/CMakeLists.txt +++ b/storage/src/vespa/storage/config/CMakeLists.txt @@ -5,26 +5,26 @@ vespa_add_library(storage_storageconfig OBJECT DEPENDS ) vespa_generate_config(storage_storageconfig stor-communicationmanager.def) -install(FILES stor-communicationmanager.def RENAME vespa.config.content.core.stor-communicationmanager.def DESTINATION var/db/vespa/config_server/serverdb/classes) +install_config_definition(stor-communicationmanager.def vespa.config.content.core.stor-communicationmanager.def) vespa_generate_config(storage_storageconfig stor-distributormanager.def) -install(FILES stor-distributormanager.def RENAME vespa.config.content.core.stor-distributormanager.def DESTINATION var/db/vespa/config_server/serverdb/classes) +install_config_definition(stor-distributormanager.def vespa.config.content.core.stor-distributormanager.def) vespa_generate_config(storage_storageconfig stor-server.def) -install(FILES stor-server.def RENAME vespa.config.content.core.stor-server.def DESTINATION var/db/vespa/config_server/serverdb/classes) +install_config_definition(stor-server.def vespa.config.content.core.stor-server.def) vespa_generate_config(storage_storageconfig stor-status.def) -install(FILES stor-status.def RENAME vespa.config.content.core.stor-status.def DESTINATION var/db/vespa/config_server/serverdb/classes) +install_config_definition(stor-status.def vespa.config.content.core.stor-status.def) vespa_generate_config(storage_storageconfig stor-messageforwarder.def) -install(FILES stor-messageforwarder.def RENAME vespa.config.content.core.stor-messageforwarder.def DESTINATION var/db/vespa/config_server/serverdb/classes) +install_config_definition(stor-messageforwarder.def vespa.config.content.core.stor-messageforwarder.def) vespa_generate_config(storage_storageconfig stor-opslogger.def) -install(FILES stor-opslogger.def RENAME vespa.config.content.core.stor-opslogger.def DESTINATION var/db/vespa/config_server/serverdb/classes) +install_config_definition(stor-opslogger.def vespa.config.content.core.stor-opslogger.def) vespa_generate_config(storage_storageconfig stor-visitordispatcher.def) -install(FILES stor-visitordispatcher.def RENAME vespa.config.content.core.stor-visitordispatcher.def DESTINATION var/db/vespa/config_server/serverdb/classes) +install_config_definition(stor-visitordispatcher.def vespa.config.content.core.stor-visitordispatcher.def) vespa_generate_config(storage_storageconfig stor-integritychecker.def) -install(FILES stor-integritychecker.def RENAME vespa.config.content.core.stor-integritychecker.def DESTINATION var/db/vespa/config_server/serverdb/classes) +install_config_definition(stor-integritychecker.def vespa.config.content.core.stor-integritychecker.def) vespa_generate_config(storage_storageconfig stor-bucketmover.def) -install(FILES stor-bucketmover.def RENAME vespa.config.content.core.stor-bucketmover.def DESTINATION var/db/vespa/config_server/serverdb/classes) +install_config_definition(stor-bucketmover.def vespa.config.content.core.stor-bucketmover.def) vespa_generate_config(storage_storageconfig stor-bouncer.def) -install(FILES stor-bouncer.def RENAME vespa.config.content.core.stor-bouncer.def DESTINATION var/db/vespa/config_server/serverdb/classes) +install_config_definition(stor-bouncer.def vespa.config.content.core.stor-bouncer.def) vespa_generate_config(storage_storageconfig stor-prioritymapping.def) -install(FILES stor-prioritymapping.def RENAME vespa.config.content.core.stor-prioritymapping.def DESTINATION var/db/vespa/config_server/serverdb/classes) +install_config_definition(stor-prioritymapping.def vespa.config.content.core.stor-prioritymapping.def) vespa_generate_config(storage_storageconfig rpc-provider.def) -install(FILES rpc-provider.def RENAME vespa.config.content.core.rpc-provider.def DESTINATION var/db/vespa/config_server/serverdb/classes) +install_config_definition(rpc-provider.def vespa.config.content.core.rpc-provider.def) diff --git a/storage/src/vespa/storage/visiting/CMakeLists.txt b/storage/src/vespa/storage/visiting/CMakeLists.txt index ee10bbd58ab..6d93d96114d 100644 --- a/storage/src/vespa/storage/visiting/CMakeLists.txt +++ b/storage/src/vespa/storage/visiting/CMakeLists.txt @@ -17,4 +17,4 @@ vespa_add_library(storage_visitor OBJECT storage_storageconfig ) vespa_generate_config(storage_visitor stor-visitor.def) -install(FILES stor-visitor.def RENAME vespa.config.content.core.stor-visitor.def DESTINATION var/db/vespa/config_server/serverdb/classes) +install_config_definition(stor-visitor.def vespa.config.content.core.stor-visitor.def) diff --git a/storageserver/src/apps/storaged/CMakeLists.txt b/storageserver/src/apps/storaged/CMakeLists.txt index b971ce44339..2c5c837d3cd 100644 --- a/storageserver/src/apps/storaged/CMakeLists.txt +++ b/storageserver/src/apps/storaged/CMakeLists.txt @@ -9,3 +9,5 @@ vespa_add_executable(storageserver_storaged_app storageserver_storageapp searchlib_searchlib_uca ) + +install(CODE "execute_process(COMMAND ln -snf vespa-storaged-bin $ENV{DESTDIR}/${CMAKE_INSTALL_PREFIX}/sbin/vespa-distributord-bin)") diff --git a/vespa-http-client/CMakeLists.txt b/vespa-http-client/CMakeLists.txt new file mode 100644 index 00000000000..511b4a4c985 --- /dev/null +++ b/vespa-http-client/CMakeLists.txt @@ -0,0 +1,2 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +install_fat_java_artifact(vespa-http-client) diff --git a/vespa_jersey2/CMakeLists.txt b/vespa_jersey2/CMakeLists.txt new file mode 100644 index 00000000000..f4f6c44202c --- /dev/null +++ b/vespa_jersey2/CMakeLists.txt @@ -0,0 +1,2 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +install_java_artifact_dependencies(vespa_jersey2) diff --git a/vespabase/CMakeLists.txt b/vespabase/CMakeLists.txt index e658a959d84..ea007b130f9 100644 --- a/vespabase/CMakeLists.txt +++ b/vespabase/CMakeLists.txt @@ -31,3 +31,7 @@ configure_file(src/vespa-configserver.service.in src/vespa-configserver.service install(FILES ${CMAKE_CURRENT_BINARY_DIR}/src/vespa.service ${CMAKE_CURRENT_BINARY_DIR}/src/vespa-configserver.service DESTINATION etc/systemd/system) install(FILES src/Defaults.pm DESTINATION lib/perl5/site_perl/Yahoo/Vespa) + +configure_file(conf/default-env.txt.in conf/default-env.txt @ONLY) +install(FILES ${CMAKE_CURRENT_BINARY_DIR}/conf/default-env.txt DESTINATION conf/vespa) + diff --git a/vespabase/conf/default-env.txt.in b/vespabase/conf/default-env.txt.in new file mode 100644 index 00000000000..4855ac9f571 --- /dev/null +++ b/vespabase/conf/default-env.txt.in @@ -0,0 +1,2 @@ +fallback VESPA_HOME @CMAKE_INSTALL_PREFIX@ +override VESPA_USER vespa diff --git a/vespabase/src/rhel-prestart.sh b/vespabase/src/rhel-prestart.sh index c00557243a7..37fc5e98533 100755 --- a/vespabase/src/rhel-prestart.sh +++ b/vespabase/src/rhel-prestart.sh @@ -97,6 +97,7 @@ fixdir ${VESPA_USER} wheel 755 var/db/vespa/config_server/serverdb/application fixdir ${VESPA_USER} wheel 755 var/db/vespa/index fixdir ${VESPA_USER} wheel 755 var/db/vespa/logcontrol fixdir ${VESPA_USER} wheel 755 var/db/vespa/search +fixdir ${VESPA_USER} wheel 755 var/jdisc_core fixdir ${VESPA_USER} wheel 755 var/vespa/bundlecache fixdir ${VESPA_USER} wheel 755 var/vespa/bundlecache/configserver fixdir ${VESPA_USER} wheel 755 var/vespa/cache/config/ diff --git a/vespaclient-container-plugin/CMakeLists.txt b/vespaclient-container-plugin/CMakeLists.txt new file mode 100644 index 00000000000..4c8a8647b23 --- /dev/null +++ b/vespaclient-container-plugin/CMakeLists.txt @@ -0,0 +1,2 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +install_fat_java_artifact(vespaclient-container-plugin) diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/OperationHandler.java b/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/OperationHandler.java index 3e16bced996..e7ed9ce10db 100644 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/OperationHandler.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/OperationHandler.java @@ -8,11 +8,11 @@ import java.util.Optional; /** * Abstract the backend stuff for the REST API, such as retrieving or updating documents. * - * @author dybis + * @author Haakon Dybdahl */ public interface OperationHandler { - class VisitResult{ + class VisitResult { public final Optional<String> token; public final String documentsAsJsonList; @@ -23,7 +23,19 @@ public interface OperationHandler { } } - VisitResult visit(RestUri restUri, String documentSelection, Optional<String> cluster, Optional<String> continuation) throws RestApiException; + class VisitOptions { + public final Optional<String> cluster; + public final Optional<String> continuation; + public final Optional<Integer> wantedDocumentCount; + + public VisitOptions(Optional<String> cluster, Optional<String> continuation, Optional<Integer> wantedDocumentCount) { + this.cluster = cluster; + this.continuation = continuation; + this.wantedDocumentCount = wantedDocumentCount; + } + } + + VisitResult visit(RestUri restUri, String documentSelection, VisitOptions options) throws RestApiException; void put(RestUri restUri, VespaXMLFeedReader.Operation data, Optional<String> route) throws RestApiException; diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/OperationHandlerImpl.java b/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/OperationHandlerImpl.java index 482a39c60e5..46678ea67e3 100644 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/OperationHandlerImpl.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/OperationHandlerImpl.java @@ -49,6 +49,7 @@ public class OperationHandlerImpl implements OperationHandler { } public static final int VISIT_TIMEOUT_MS = 120000; + public static final int WANTED_DOCUMENT_COUNT_UPPER_BOUND = 1000; // Approximates the max default size of a bucket private final DocumentAccess documentAccess; private final DocumentApiMetrics metricsHelper; private final ClusterEnumerator clusterEnumerator; @@ -109,13 +110,8 @@ public class OperationHandlerImpl implements OperationHandler { } @Override - public VisitResult visit( - RestUri restUri, - String documentSelection, - Optional<String> cluster, - Optional<String> continuation) throws RestApiException { - - VisitorParameters visitorParameters = createVisitorParameters(restUri, documentSelection, cluster, continuation); + public VisitResult visit(RestUri restUri, String documentSelection, VisitOptions options) throws RestApiException { + VisitorParameters visitorParameters = createVisitorParameters(restUri, documentSelection, options); VisitorControlHandler visitorControlHandler = new VisitorControlHandler(); visitorParameters.setControlHandler(visitorControlHandler); @@ -326,13 +322,13 @@ public class OperationHandlerImpl implements OperationHandler { private VisitorParameters createVisitorParameters( RestUri restUri, String documentSelection, - Optional<String> clusterName, - Optional<String> continuation) + VisitOptions options) throws RestApiException { StringBuilder selection = new StringBuilder(); if (! documentSelection.isEmpty()) { + // TODO shouldn't selection be wrapped in () itself ? selection.append("(").append(documentSelection).append(" and "); } selection.append(restUri.getDocumentType()).append(" and (id.namespace=='").append(restUri.getNamespace()).append("')"); @@ -346,24 +342,26 @@ public class OperationHandlerImpl implements OperationHandler { params.setMaxBucketsPerVisitor(1); params.setMaxPending(32); params.setMaxFirstPassHits(1); - params.setMaxTotalHits(1); + params.setMaxTotalHits(options.wantedDocumentCount + .map(n -> Math.min(Math.max(n, 1), WANTED_DOCUMENT_COUNT_UPPER_BOUND)) + .orElse(1)); params.setThrottlePolicy(new StaticThrottlePolicy().setMaxPendingCount(1)); params.setToTimestamp(0L); params.setFromTimestamp(0L); params.setSessionTimeoutMs(VISIT_TIMEOUT_MS); - params.visitInconsistentBuckets(true); + params.visitInconsistentBuckets(true); // TODO document this as part of consistency doc params.setVisitorOrdering(VisitorOrdering.ASCENDING); - params.setRoute(resolveClusterRoute(clusterName)); + params.setRoute(resolveClusterRoute(options.cluster)); params.setTraceLevel(0); params.setPriority(DocumentProtocol.Priority.NORMAL_4); params.setVisitRemoves(false); - if (continuation.isPresent()) { + if (options.continuation.isPresent()) { try { - params.setResumeToken(ContinuationHit.getToken(continuation.get())); + params.setResumeToken(ContinuationHit.getToken(options.continuation.get())); } catch (Exception e) { throw new RestApiException(Response.createErrorResponse(500, ExceptionUtils.getStackTrace(e), restUri, RestUri.apiErrorCodes.UNSPECIFIED)); } diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/RestApi.java b/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/RestApi.java index 1f1eca9674b..5e0fea8ab7d 100644 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/RestApi.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/RestApi.java @@ -38,7 +38,7 @@ import java.util.concurrent.atomic.AtomicInteger; /** * API for handling single operation on a document and visiting. * - * @author dybis + * @author Haakon Dybdahl */ public class RestApi extends LoggingRequestHandler { @@ -52,6 +52,7 @@ public class RestApi extends LoggingRequestHandler { private static final String SELECTION = "selection"; private static final String CLUSTER = "cluster"; private static final String CONTINUATION = "continuation"; + private static final String WANTED_DOCUMENT_COUNT = "wantedDocumentCount"; private static final String APPLICATION_JSON = "application/json"; private final OperationHandler operationHandler; private SingleDocumentParser singleDocumentParser; @@ -96,19 +97,33 @@ public class RestApi extends LoggingRequestHandler { this.singleDocumentParser = new SingleDocumentParser(docTypeManager); } - // Returns null if invalid value. - private Optional<Boolean> parseBoolean(String parameter, HttpRequest request) { + private static Optional<String> requestProperty(String parameter, HttpRequest request) { final String property = request.getProperty(parameter); if (property != null && ! property.isEmpty()) { - switch (property) { - case "true" : return Optional.of(true); - case "false": return Optional.of(false); - default : return null; - } + return Optional.of(property); } return Optional.empty(); } + private static boolean parseBooleanStrict(String value) { + if ("true".equalsIgnoreCase(value)) { + return true; + } else if ("false".equalsIgnoreCase(value)) { + return false; + } + throw new IllegalArgumentException(String.format("Value not convertible to bool: '%s'", value)); + } + + private static Optional<Boolean> parseBoolean(String parameter, HttpRequest request) { + Optional<String> property = requestProperty(parameter, request); + return property.map(RestApi::parseBooleanStrict); + } + + private static Optional<Integer> parseInteger(String parameter, HttpRequest request) throws NumberFormatException { + Optional<String> property = requestProperty(parameter, request); + return property.map(Integer::parseInt); + } + @Override public HttpResponse handle(HttpRequest request) { try { @@ -134,8 +149,10 @@ public class RestApi extends LoggingRequestHandler { return Response.createErrorResponse(500, "Exception while parsing URI: " + e2.getMessage(), RestUri.apiErrorCodes.URL_PARSING); } - Optional<Boolean> create = parseBoolean(CREATE_PARAMETER_NAME, request); - if (create == null) { + final Optional<Boolean> create; + try { + create = parseBoolean(CREATE_PARAMETER_NAME, request); + } catch (IllegalArgumentException e) { return Response.createErrorResponse(403, "Non valid value for 'create' parameter, must be empty, true, or " + "false: " + request.getProperty(CREATE_PARAMETER_NAME), RestUri.apiErrorCodes.INVALID_CREATE_VALUE); } @@ -184,9 +201,7 @@ public class RestApi extends LoggingRequestHandler { if (condition != null && ! condition.isEmpty()) { operationUpdate.getDocumentUpdate().setCondition(new TestAndSetCondition(condition)); } - if (create.isPresent()) { - operationUpdate.getDocumentUpdate().setCreateIfNonExistent(create.get()); - } + create.ifPresent(c -> operationUpdate.getDocumentUpdate().setCreateIfNonExistent(c)); return operationUpdate; } @@ -214,11 +229,16 @@ public class RestApi extends LoggingRequestHandler { } }; } + + private static HttpResponse createInvalidParameterResponse(String parameter, String explanation) { + return Response.createErrorResponse(403, String.format("Invalid '%s' value. %s", parameter, explanation), RestUri.apiErrorCodes.UNSPECIFIED); + } private HttpResponse handleVisit(RestUri restUri, HttpRequest request) throws RestApiException { String documentSelection = Optional.ofNullable(request.getProperty(SELECTION)).orElse(""); if (restUri.getGroup().isPresent() && ! restUri.getGroup().get().value.isEmpty()) { if (! documentSelection.isEmpty()) { + // TODO why is this restriction in place? Document selection allows composition of location predicate and other expressions return Response.createErrorResponse( 400, "Visiting does not support setting value for group/value in combination with expression, try using only expression parameter instead.", @@ -234,11 +254,17 @@ public class RestApi extends LoggingRequestHandler { } Optional<String> cluster = Optional.ofNullable(request.getProperty(CLUSTER)); Optional<String> continuation = Optional.ofNullable(request.getProperty(CONTINUATION)); - final OperationHandler.VisitResult visit = operationHandler.visit(restUri, documentSelection, cluster, continuation); - final ObjectNode resultNode = mapper.createObjectNode(); - if (visit.token.isPresent()) { - resultNode.put(CONTINUATION, visit.token.get()); + Optional<Integer> wantedDocumentCount; + try { + wantedDocumentCount = parseInteger(WANTED_DOCUMENT_COUNT, request); + } catch (IllegalArgumentException e) { + return createInvalidParameterResponse(WANTED_DOCUMENT_COUNT, "Expected integer"); } + + final OperationHandler.VisitOptions options = new OperationHandler.VisitOptions(cluster, continuation, wantedDocumentCount); + final OperationHandler.VisitResult visit = operationHandler.visit(restUri, documentSelection, options); + final ObjectNode resultNode = mapper.createObjectNode(); + visit.token.ifPresent(t -> resultNode.put(CONTINUATION, t)); resultNode.putArray(DOCUMENTS).addPOJO(visit.documentsAsJsonList); resultNode.put(PATH_NAME, restUri.getRawPath()); diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/OperationHandlerImplTest.java b/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/OperationHandlerImplTest.java index 796c6d23deb..5735e84f3fe 100644 --- a/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/OperationHandlerImplTest.java +++ b/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/OperationHandlerImplTest.java @@ -91,7 +91,7 @@ public class OperationHandlerImplTest { return new String( stream.toByteArray()); } - private class OperationHandlerImplFixture { + private static class OperationHandlerImplFixture { DocumentAccess documentAccess = mock(DocumentAccess.class); AtomicReference<VisitorParameters> assignedParameters = new AtomicReference<>(); VisitorControlHandler.CompletionCode completionCode = VisitorControlHandler.CompletionCode.SUCCESS; @@ -123,6 +123,14 @@ public class OperationHandlerImplTest { return new RestUri(new URI("http://localhost/document/v1/namespace/document-type/docid/")); } + private static OperationHandler.VisitOptions visitOptionsWithWantedDocumentCount(int wantedDocumentCount) { + return new OperationHandler.VisitOptions(Optional.empty(), Optional.empty(), Optional.of(wantedDocumentCount)); + } + + private static OperationHandler.VisitOptions emptyVisitOptions() { + return new OperationHandler.VisitOptions(Optional.empty(), Optional.empty(), Optional.empty()); + } + @Test public void timeout_without_buckets_visited_throws_timeout_error() throws Exception { OperationHandlerImplFixture fixture = new OperationHandlerImplFixture(); @@ -131,7 +139,7 @@ public class OperationHandlerImplTest { // RestApiException hides its guts internally, so cannot trivially use @Rule directly to check for error category try { OperationHandlerImpl handler = fixture.createHandler(); - handler.visit(dummyVisitUri(), "", Optional.empty(), Optional.empty()); + handler.visit(dummyVisitUri(), "", emptyVisitOptions()); } catch (RestApiException e) { assertThat(e.getResponse().getStatus(), is(500)); assertThat(renderRestApiExceptionAsString(e), containsString("Timed out")); @@ -145,7 +153,7 @@ public class OperationHandlerImplTest { fixture.bucketsVisited = 1; OperationHandlerImpl handler = fixture.createHandler(); - handler.visit(dummyVisitUri(), "", Optional.empty(), Optional.empty()); + handler.visit(dummyVisitUri(), "", emptyVisitOptions()); } @Test @@ -153,8 +161,50 @@ public class OperationHandlerImplTest { OperationHandlerImplFixture fixture = new OperationHandlerImplFixture(); OperationHandlerImpl handler = fixture.createHandler(); - handler.visit(dummyVisitUri(), "", Optional.empty(), Optional.empty()); + handler.visit(dummyVisitUri(), "", emptyVisitOptions()); assertThat(fixture.assignedParameters.get().getSessionTimeoutMs(), is((long)OperationHandlerImpl.VISIT_TIMEOUT_MS)); } + + private static VisitorParameters generatedParametersFromVisitOptions(OperationHandler.VisitOptions options) throws Exception { + OperationHandlerImplFixture fixture = new OperationHandlerImplFixture(); + OperationHandlerImpl handler = fixture.createHandler(); + + handler.visit(dummyVisitUri(), "", options); + return fixture.assignedParameters.get(); + } + + @Test + public void provided_wanted_document_count_is_propagated_to_visitor_parameters() throws Exception { + VisitorParameters params = generatedParametersFromVisitOptions(visitOptionsWithWantedDocumentCount(123)); + assertThat(params.getMaxTotalHits(), is((long)123)); + } + + @Test + public void wanted_document_count_is_1_unless_specified() throws Exception { + VisitorParameters params = generatedParametersFromVisitOptions(emptyVisitOptions()); + assertThat(params.getMaxTotalHits(), is((long)1)); + } + + @Test + public void too_low_wanted_document_count_is_bounded_to_1() throws Exception { + VisitorParameters params = generatedParametersFromVisitOptions(visitOptionsWithWantedDocumentCount(-1)); + assertThat(params.getMaxTotalHits(), is((long)1)); + + params = generatedParametersFromVisitOptions(visitOptionsWithWantedDocumentCount(Integer.MIN_VALUE)); + assertThat(params.getMaxTotalHits(), is((long)1)); + + params = generatedParametersFromVisitOptions(visitOptionsWithWantedDocumentCount(0)); + assertThat(params.getMaxTotalHits(), is((long)1)); + } + + @Test + public void too_high_wanted_document_count_is_bounded_to_upper_bound() throws Exception { + VisitorParameters params = generatedParametersFromVisitOptions(visitOptionsWithWantedDocumentCount(OperationHandlerImpl.WANTED_DOCUMENT_COUNT_UPPER_BOUND + 1)); + assertThat(params.getMaxTotalHits(), is((long)OperationHandlerImpl.WANTED_DOCUMENT_COUNT_UPPER_BOUND)); + + params = generatedParametersFromVisitOptions(visitOptionsWithWantedDocumentCount(Integer.MAX_VALUE)); + assertThat(params.getMaxTotalHits(), is((long)OperationHandlerImpl.WANTED_DOCUMENT_COUNT_UPPER_BOUND)); + } + } diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/resource/MockedOperationHandler.java b/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/resource/MockedOperationHandler.java index 97f45c4062a..f353013232f 100644 --- a/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/resource/MockedOperationHandler.java +++ b/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/resource/MockedOperationHandler.java @@ -18,9 +18,11 @@ public class MockedOperationHandler implements OperationHandler { int deleteCount = 0; @Override - public VisitResult visit(RestUri restUri, String documentSelection, Optional<String> cluster, Optional<String> continuation) throws RestApiException { - return new VisitResult(Optional.of("token"), "List of json docs, cont token " + continuation.map(a->a).orElse("not set") + ", doc selection: '" - + documentSelection + "'"); + public VisitResult visit(RestUri restUri, String documentSelection, VisitOptions options) throws RestApiException { + return new VisitResult(Optional.of("token"), "List of json docs, cont token " + + options.continuation.orElse("not set") + ", doc selection: '" + + documentSelection + "'" + + options.wantedDocumentCount.map(n -> String.format(", min docs returned: %d", n)).orElse("")); } @Override diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/resource/RestApiTest.java b/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/resource/RestApiTest.java index 10ae80a5d03..91390e3a0d8 100644 --- a/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/resource/RestApiTest.java +++ b/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/resource/RestApiTest.java @@ -270,6 +270,7 @@ public class RestApiTest { assertThat(rest, containsString(visit_response_part3)); } + // TODO why is this a limitation? String visit_test_bad_uri = "/document/v1/namespace/document-type/group/abc?continuation=abc&selection=foo"; String visit_test_bad_response = "Visiting does not support setting value for group/value in combination with expression"; @@ -294,6 +295,22 @@ public class RestApiTest { assertThat(rest, containsString(visit_test_response_selection_rewrite)); } + @Test + public void wanted_document_count_returned_parameter_is_propagated() throws IOException { + Request request = new Request(String.format("http://localhost:%s/document/v1/namespace/document-type/docid/?wantedDocumentCount=321", getFirstListenPort())); + HttpGet get = new HttpGet(request.getUri()); + String rest = doRest(get); + assertThat(rest, containsString("min docs returned: 321")); + } + + @Test + public void wanted_document_count_parameter_returns_error_response() throws IOException { + Request request = new Request(String.format("http://localhost:%s/document/v1/namespace/document-type/docid/?wantedDocumentCount=aardvark", getFirstListenPort())); + HttpGet get = new HttpGet(request.getUri()); + String rest = doRest(get); + assertThat(rest, containsString("Invalid 'wantedDocumentCount' value. Expected integer")); + } + private String doRest(HttpRequestBase request) throws IOException { HttpClient client = HttpClientBuilder.create().build(); HttpResponse response = client.execute(request); diff --git a/vespaclient-core/CMakeLists.txt b/vespaclient-core/CMakeLists.txt new file mode 100644 index 00000000000..facea0b447d --- /dev/null +++ b/vespaclient-core/CMakeLists.txt @@ -0,0 +1,3 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +install_config_definition(src/main/resources/configdefinitions/feeder.def vespaclient.config.feeder.def) +install_config_definition(src/main/resources/configdefinitions/spooler.def vespa.config.content.spooler.spooler.def) diff --git a/vespaclient-java/CMakeLists.txt b/vespaclient-java/CMakeLists.txt new file mode 100644 index 00000000000..b9240adee8a --- /dev/null +++ b/vespaclient-java/CMakeLists.txt @@ -0,0 +1,12 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +install_fat_java_artifact(vespaclient-java) + +vespa_install_script(src/main/sh/vespa-document-statistics.sh vespa-document-statistics bin) +vespa_install_script(src/main/sh/vespa-stat.sh vespa-stat bin) +vespa_install_script(src/main/sh/vespa-query-profile-dump-tool.sh vespa-query-profile-dump-tool bin) +vespa_install_script(src/main/sh/vespa-summary-benchmark.sh vespa-summary-benchmark bin) +vespa_install_script(src/main/sh/vespa-destination.sh vespa-destination bin) +vespa_install_script(src/main/sh/vespa-feeder.sh vespa-feeder bin) +vespa_install_script(src/main/sh/vespa-get.sh vespa-get bin) +vespa_install_script(src/main/sh/vespa-visit.sh vespa-visit bin) +vespa_install_script(src/main/sh/vespa-visit-target.sh vespa-visit-target bin) diff --git a/vespajlib/CMakeLists.txt b/vespajlib/CMakeLists.txt new file mode 100644 index 00000000000..7235535ace8 --- /dev/null +++ b/vespajlib/CMakeLists.txt @@ -0,0 +1,2 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +install_java_artifact(vespajlib) diff --git a/vsm/src/vespa/vsm/config/CMakeLists.txt b/vsm/src/vespa/vsm/config/CMakeLists.txt index ea65d8c8fb4..e3bd2db68e2 100644 --- a/vsm/src/vespa/vsm/config/CMakeLists.txt +++ b/vsm/src/vespa/vsm/config/CMakeLists.txt @@ -4,8 +4,8 @@ vespa_add_library(vsm_vconfig OBJECT DEPENDS ) vespa_generate_config(vsm_vconfig vsmfields.def) -install(FILES vsmfields.def RENAME vespa.config.search.vsm.vsmfields.def DESTINATION var/db/vespa/config_server/serverdb/classes) +install_config_definition(vsmfields.def vespa.config.search.vsm.vsmfields.def) vespa_generate_config(vsm_vconfig vsm.def) -install(FILES vsm.def RENAME vespa.config.search.vsm.vsm.def DESTINATION var/db/vespa/config_server/serverdb/classes) +install_config_definition(vsm.def vespa.config.search.vsm.vsm.def) vespa_generate_config(vsm_vconfig vsmsummary.def) -install(FILES vsmsummary.def RENAME vespa.config.search.vsm.vsmsummary.def DESTINATION var/db/vespa/config_server/serverdb/classes) +install_config_definition(vsmsummary.def vespa.config.search.vsm.vsmsummary.def) diff --git a/zkfacade/CMakeLists.txt b/zkfacade/CMakeLists.txt new file mode 100644 index 00000000000..6610356c0fd --- /dev/null +++ b/zkfacade/CMakeLists.txt @@ -0,0 +1,2 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +install_fat_java_artifact(zkfacade) |