diff options
146 files changed, 1073 insertions, 709 deletions
diff --git a/CMakeLists.txt b/CMakeLists.txt index 413d232c00d..f7752d5915e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -16,7 +16,6 @@ find_package(JNI REQUIRED) include(functions.cmake) include(build_settings.cmake) -include(install_java.cmake) # Enable CTest unit testing enable_testing() @@ -24,12 +23,36 @@ enable_testing() # Include vespa config definitions in every target include_directories(BEFORE ${CMAKE_BINARY_DIR}/configdefinitions/src) +add_subdirectory(application-preprocessor) +add_subdirectory(chain) +add_subdirectory(component) +add_subdirectory(config-bundle) +add_subdirectory(config-model) +add_subdirectory(config-model-api) +add_subdirectory(config-provisioning) +add_subdirectory(config-proxy) add_subdirectory(config) +add_subdirectory(config-model-fat) add_subdirectory(configd) add_subdirectory(configdefinitions) add_subdirectory(configserver) add_subdirectory(configutil) +add_subdirectory(container-accesslogging) +add_subdirectory(container-core) +add_subdirectory(container-di) +add_subdirectory(container-disc) +add_subdirectory(container-jersey2) +add_subdirectory(container-messagebus) +add_subdirectory(container-search) +add_subdirectory(container-search-and-docproc) +add_subdirectory(clustercontroller-apps) +add_subdirectory(clustercontroller-apputil) +add_subdirectory(clustercontroller-utils) +add_subdirectory(clustercontroller-core) add_subdirectory(defaults) +add_subdirectory(docker-api) +add_subdirectory(docproc) +add_subdirectory(docprocs) add_subdirectory(document) add_subdirectory(documentapi) add_subdirectory(eval) @@ -41,14 +64,20 @@ add_subdirectory(filedistribution) add_subdirectory(fnet) add_subdirectory(frtstream) add_subdirectory(fsa) +add_subdirectory(jdisc_core) +add_subdirectory(jdisc_http_service) +add_subdirectory(jdisc_jetty) add_subdirectory(jrt_test) add_subdirectory(juniper) add_subdirectory(logd) +add_subdirectory(logserver) add_subdirectory(lowercasing_test) add_subdirectory(memfilepersistence) add_subdirectory(messagebus) add_subdirectory(messagebus_test) add_subdirectory(metrics) +add_subdirectory(node-repository) +add_subdirectory(orchestrator) add_subdirectory(persistence) add_subdirectory(persistencetypes) add_subdirectory(searchcommon) @@ -56,25 +85,32 @@ add_subdirectory(searchcore) add_subdirectory(searchcorespi) add_subdirectory(searchlib) add_subdirectory(searchsummary) +add_subdirectory(simplemetrics) add_subdirectory(slobrok) add_subdirectory(staging_vespalib) +add_subdirectory(standalone-container) add_subdirectory(storage) add_subdirectory(storageapi) add_subdirectory(storageframework) add_subdirectory(storageserver) +add_subdirectory(statistics) add_subdirectory(streamingvisitors) add_subdirectory(vbench) add_subdirectory(vdslib) add_subdirectory(vdstestlib) +add_subdirectory(vespa-http-client) +add_subdirectory(vespa_jersey2) add_subdirectory(vespabase) add_subdirectory(vespaclient) +add_subdirectory(vespaclient-core) +add_subdirectory(vespaclient-container-plugin) +add_subdirectory(vespaclient-java) +add_subdirectory(vespajlib) add_subdirectory(vespalib) add_subdirectory(vespalog) add_subdirectory(vespamalloc) add_subdirectory(vsm) -# Note: Change when cmake gets proper post-install support. -# Post installation steps are run from dist subdirectory which needs to be the last add_subdirectory(...) call in this file. -add_subdirectory(dist) +add_subdirectory(zkfacade) # Create module targets with name ${MODULE}+module depending on every target defined within that module __create_module_targets(TARGETS "module") diff --git a/application-preprocessor/CMakeLists.txt b/application-preprocessor/CMakeLists.txt new file mode 100644 index 00000000000..e40fd4a6736 --- /dev/null +++ b/application-preprocessor/CMakeLists.txt @@ -0,0 +1,4 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +install_fat_java_artifact(application-preprocessor) +vespa_install_script(src/main/sh/vespa-preprocess-application bin) + diff --git a/dist/CMakeLists.txt b/chain/CMakeLists.txt index d06074074df..3b5b5fd2c99 100644 --- a/dist/CMakeLists.txt +++ b/chain/CMakeLists.txt @@ -1,2 +1,2 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -install(CODE "execute_process(COMMAND ${CMAKE_CURRENT_LIST_DIR}/post_install.sh ${CMAKE_INSTALL_PREFIX})") +install_config_definition(src/main/resources/configdefinitions/chains.def container.core.chains.def) diff --git a/clustercontroller-apps/CMakeLists.txt b/clustercontroller-apps/CMakeLists.txt new file mode 100644 index 00000000000..f59ffbfa7bf --- /dev/null +++ b/clustercontroller-apps/CMakeLists.txt @@ -0,0 +1,2 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +install_fat_java_artifact(clustercontroller-apps) diff --git a/clustercontroller-apputil/CMakeLists.txt b/clustercontroller-apputil/CMakeLists.txt new file mode 100644 index 00000000000..bdfb3ab3ed7 --- /dev/null +++ b/clustercontroller-apputil/CMakeLists.txt @@ -0,0 +1,2 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +install_fat_java_artifact(clustercontroller-apputil) diff --git a/clustercontroller-core/CMakeLists.txt b/clustercontroller-core/CMakeLists.txt new file mode 100644 index 00000000000..6754e893009 --- /dev/null +++ b/clustercontroller-core/CMakeLists.txt @@ -0,0 +1,2 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +install_fat_java_artifact(clustercontroller-core) diff --git a/clustercontroller-utils/CMakeLists.txt b/clustercontroller-utils/CMakeLists.txt new file mode 100644 index 00000000000..250a8e7e693 --- /dev/null +++ b/clustercontroller-utils/CMakeLists.txt @@ -0,0 +1,2 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +install_fat_java_artifact(clustercontroller-utils) diff --git a/component/CMakeLists.txt b/component/CMakeLists.txt new file mode 100644 index 00000000000..87d0a4989ba --- /dev/null +++ b/component/CMakeLists.txt @@ -0,0 +1,2 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +install_fat_java_artifact(component) diff --git a/config-bundle/CMakeLists.txt b/config-bundle/CMakeLists.txt new file mode 100644 index 00000000000..8d4878920a4 --- /dev/null +++ b/config-bundle/CMakeLists.txt @@ -0,0 +1,2 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +install_fat_java_artifact(config-bundle) diff --git a/config-model-api/CMakeLists.txt b/config-model-api/CMakeLists.txt new file mode 100644 index 00000000000..f69e5242e2e --- /dev/null +++ b/config-model-api/CMakeLists.txt @@ -0,0 +1,2 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +install_fat_java_artifact(config-model-api) diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/SuperModel.java b/config-model-api/src/main/java/com/yahoo/config/model/api/SuperModel.java new file mode 100644 index 00000000000..d03824120d8 --- /dev/null +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/SuperModel.java @@ -0,0 +1,91 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.model.api; + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.TenantName; + +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.stream.Collectors; + +public class SuperModel { + private final Map<TenantName, Map<ApplicationId, ApplicationInfo>> models; + + public SuperModel() { + this.models = Collections.emptyMap(); + } + + public SuperModel(Map<TenantName, Map<ApplicationId, ApplicationInfo>> models) { + this.models = models; + } + + /** + * Do NOT mutate the returned map. + * TODO: Make the returned map immutable (and type to Map<ApplicationId, ApplicationInfo>) + */ + public Map<TenantName, Map<ApplicationId, ApplicationInfo>> getAllModels() { + return models; + } + + public List<ApplicationInfo> getAllApplicationInfos() { + return models.values().stream().flatMap(entry -> entry.values().stream()).collect(Collectors.toList()); + } + + public Optional<ApplicationInfo> getApplicationInfo(ApplicationId applicationId) { + Map<ApplicationId, ApplicationInfo> tenantInfo = models.get(applicationId.tenant()); + if (tenantInfo == null) { + return Optional.empty(); + } + + ApplicationInfo applicationInfo = tenantInfo.get(applicationId); + if (applicationInfo == null) { + return Optional.empty(); + } + + return Optional.of(applicationInfo); + } + + public SuperModel cloneAndSetApplication(ApplicationInfo application) { + TenantName tenant = application.getApplicationId().tenant(); + Map<TenantName, Map<ApplicationId, ApplicationInfo>> newModels = cloneModels(models); + if (!newModels.containsKey(tenant)) { + // New application has been activated + newModels.put(tenant, new LinkedHashMap<>()); + } else { + // Application has been redeployed + } + + newModels.get(tenant).put(application.getApplicationId(), application); + + return new SuperModel(newModels); + } + + public SuperModel cloneAndRemoveApplication(ApplicationId applicationId) { + Map<TenantName, Map<ApplicationId, ApplicationInfo>> newModels = cloneModels(models); + if (newModels.containsKey(applicationId.tenant())) { + newModels.get(applicationId.tenant()).remove(applicationId); + if (newModels.get(applicationId.tenant()).isEmpty()) { + newModels.remove(applicationId.tenant()); + } + } + + return new SuperModel(newModels); + } + + private static Map<TenantName, Map<ApplicationId, ApplicationInfo>> cloneModels( + Map<TenantName, Map<ApplicationId, ApplicationInfo>> models) { + Map<TenantName, Map<ApplicationId, ApplicationInfo>> newModels = new LinkedHashMap<>(); + for (Map.Entry<TenantName, Map<ApplicationId, ApplicationInfo>> entry : models.entrySet()) { + Map<ApplicationId, ApplicationInfo> appMap = new LinkedHashMap<>(); + newModels.put(entry.getKey(), appMap); + for (Map.Entry<ApplicationId, ApplicationInfo> appEntry : entry.getValue().entrySet()) { + appMap.put(appEntry.getKey(), appEntry.getValue()); + } + } + + return newModels; + } +} diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/SuperModelListener.java b/config-model-api/src/main/java/com/yahoo/config/model/api/SuperModelListener.java index 16580c9e9f6..043e23902b2 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/SuperModelListener.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/SuperModelListener.java @@ -11,10 +11,10 @@ public interface SuperModelListener { * Application has been activated: Either deployed the first time, * internally redeployed, or externally triggered redeploy. */ - void applicationActivated(ApplicationInfo application); + void applicationActivated(SuperModel superModel, ApplicationId id); /** * Application has been removed. */ - void applicationRemoved(ApplicationId id); + void applicationRemoved(SuperModel superModel, ApplicationId id); } diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/SuperModelProvider.java b/config-model-api/src/main/java/com/yahoo/config/model/api/SuperModelProvider.java index 42437b20b83..4c7f15143ec 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/SuperModelProvider.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/SuperModelProvider.java @@ -1,13 +1,18 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.config.model.api; -import java.util.List; +import com.yahoo.config.provision.Zone; public interface SuperModelProvider { + SuperModel getSuperModel(); + /** - * Returns all applications in the SuperModel. All changes to the SuperModel + * Returns the current SuperModel. All changes to the SuperModel * following that snapshot will be published to the listener. Warning: The listener * methods may have been invoked before (or concurrently with) this method returning. */ - List<ApplicationInfo> snapshot(SuperModelListener listener); + SuperModel snapshot(SuperModelListener listener); + + // TODO: Remove - clients of SuperModel should get zone from elsewhere. + Zone getZone(); } diff --git a/config-model-fat/CMakeLists.txt b/config-model-fat/CMakeLists.txt new file mode 100644 index 00000000000..1e2364556dc --- /dev/null +++ b/config-model-fat/CMakeLists.txt @@ -0,0 +1,5 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +install_java_artifact(config-model-fat) + +install(FILES src/main/resources/config-models.xml + DESTINATION conf/configserver-app) diff --git a/config-model/CMakeLists.txt b/config-model/CMakeLists.txt new file mode 100644 index 00000000000..274ab8a763b --- /dev/null +++ b/config-model/CMakeLists.txt @@ -0,0 +1,10 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +install_fat_java_artifact(config-model) + +vespa_install_script(src/main/perl/vespa-deploy bin) +vespa_install_script(src/main/perl/vespa-expand-config.pl bin) +vespa_install_script(src/main/perl/vespa-replicate-log-stream bin) +vespa_install_script(src/main/sh/vespa-validate-application bin) + +install(DIRECTORY src/main/resources/schema DESTINATION share/vespa PATTERN ".gitignore" EXCLUDE) +install(DIRECTORY src/main/resources/schema DESTINATION share/vespa/schema/version/6.x PATTERN ".gitignore" EXCLUDE) diff --git a/config-model/src/main/java/com/yahoo/vespa/model/AbstractService.java b/config-model/src/main/java/com/yahoo/vespa/model/AbstractService.java index 38e417aca9e..6d4945f23ad 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/AbstractService.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/AbstractService.java @@ -45,7 +45,7 @@ public abstract class AbstractService extends AbstractConfigProducer<AbstractCon /** The optional PRELOAD libraries for this Service. */ // Please keep non-null, as passed to command line in service startup - private String preload = Defaults.getDefaults().underVespaHome("lib64/vespa/malloc/libvespamallocd.so"); + private String preload = Defaults.getDefaults().underVespaHome("lib64/vespa/malloc/libvespamalloc.so"); // If larger or equal to 0 it mean that explicit mmaps shall not be included in coredump. private long mmapNoCoreLimit = -1l; diff --git a/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java b/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java index 98f599769c0..63d5d37598b 100644 --- a/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java +++ b/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java @@ -110,9 +110,9 @@ public class ModelProvisioningTest { assertThat(model.getContainerClusters().get("mydisc").getContainers().get(0).getJvmArgs(), is("")); assertThat(model.getContainerClusters().get("mydisc").getContainers().get(1).getJvmArgs(), is("")); assertThat(model.getContainerClusters().get("mydisc").getContainers().get(2).getJvmArgs(), is("")); - assertThat(model.getContainerClusters().get("mydisc").getContainers().get(0).getPreLoad(), is(getDefaults().underVespaHome("lib64/vespa/malloc/libvespamallocd.so"))); - assertThat(model.getContainerClusters().get("mydisc").getContainers().get(1).getPreLoad(), is(getDefaults().underVespaHome("lib64/vespa/malloc/libvespamallocd.so"))); - assertThat(model.getContainerClusters().get("mydisc").getContainers().get(2).getPreLoad(), is(getDefaults().underVespaHome("lib64/vespa/malloc/libvespamallocd.so"))); + assertThat(model.getContainerClusters().get("mydisc").getContainers().get(0).getPreLoad(), is(getDefaults().underVespaHome("lib64/vespa/malloc/libvespamalloc.so"))); + assertThat(model.getContainerClusters().get("mydisc").getContainers().get(1).getPreLoad(), is(getDefaults().underVespaHome("lib64/vespa/malloc/libvespamalloc.so"))); + assertThat(model.getContainerClusters().get("mydisc").getContainers().get(2).getPreLoad(), is(getDefaults().underVespaHome("lib64/vespa/malloc/libvespamalloc.so"))); assertThat(model.getContainerClusters().get("mydisc").getMemoryPercentage(), is(Optional.empty())); assertThat(model.getContainerClusters().get("mydisc2").getContainers().get(0).getJvmArgs(), is("-verbosegc")); diff --git a/config-provisioning/CMakeLists.txt b/config-provisioning/CMakeLists.txt new file mode 100644 index 00000000000..829ba87fab8 --- /dev/null +++ b/config-provisioning/CMakeLists.txt @@ -0,0 +1,3 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +install_fat_java_artifact(config-provisioning) +install_config_definition(src/main/resources/configdefinitions/flavors.def config.provisioning.flavors.def) diff --git a/config-proxy/CMakeLists.txt b/config-proxy/CMakeLists.txt new file mode 100644 index 00000000000..a87f10573be --- /dev/null +++ b/config-proxy/CMakeLists.txt @@ -0,0 +1,6 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +install_fat_java_artifact(config-proxy) + +vespa_install_script(src/main/sh/vespa-config-ctl.sh vespa-config-ctl bin) +vespa_install_script(src/main/sh/vespa-config-loadtester.sh vespa-config-loadtester bin) +vespa_install_script(src/main/sh/vespa-config-verification.sh vespa-config-verification bin) diff --git a/config/src/main/java/com/yahoo/config/subscription/impl/JRTConfigRequester.java b/config/src/main/java/com/yahoo/config/subscription/impl/JRTConfigRequester.java index 47bb7052c97..7d830484edd 100644 --- a/config/src/main/java/com/yahoo/config/subscription/impl/JRTConfigRequester.java +++ b/config/src/main/java/com/yahoo/config/subscription/impl/JRTConfigRequester.java @@ -41,8 +41,8 @@ public class JRTConfigRequester implements RequestWaiter { private int fatalFailures = 0; // independent of transientFailures private int transientFailures = 0; // independent of fatalFailures private final ScheduledThreadPoolExecutor scheduler = new ScheduledThreadPoolExecutor(1, new JRTSourceThreadFactory()); - private Instant suspendWarningLogged; - private Instant noApplicationWarningLogged; + private Instant suspendWarningLogged = Instant.MIN; + private Instant noApplicationWarningLogged = Instant.MIN; private static final Duration delayBetweenWarnings = Duration.ofSeconds(60); private final ConnectionPool connectionPool; static final float randomFraction = 0.2f; diff --git a/configdefinitions/CMakeLists.txt b/configdefinitions/CMakeLists.txt index d8f89e04cc1..ee78759254a 100644 --- a/configdefinitions/CMakeLists.txt +++ b/configdefinitions/CMakeLists.txt @@ -8,3 +8,5 @@ vespa_define_module( LIBS src/vespa ) + +install_fat_java_artifact(configdefinitions) diff --git a/configdefinitions/src/vespa/CMakeLists.txt b/configdefinitions/src/vespa/CMakeLists.txt index 016739f4594..4ed4dc06d41 100644 --- a/configdefinitions/src/vespa/CMakeLists.txt +++ b/configdefinitions/src/vespa/CMakeLists.txt @@ -5,66 +5,66 @@ vespa_add_library(configdefinitions DEPENDS ) vespa_generate_config(configdefinitions application-id.def) -install(FILES application-id.def RENAME cloud.config.application-id.def DESTINATION var/db/vespa/config_server/serverdb/classes) +install_config_definition(application-id.def cloud.config.application-id.def) vespa_generate_config(configdefinitions attributes.def) -install(FILES attributes.def RENAME vespa.config.search.attributes.def DESTINATION var/db/vespa/config_server/serverdb/classes) +install_config_definition(attributes.def vespa.config.search.attributes.def) vespa_generate_config(configdefinitions cluster-info.def) -install(FILES cluster-info.def RENAME cloud.config.cluster-info.def DESTINATION var/db/vespa/config_server/serverdb/classes) +install_config_definition(cluster-info.def cloud.config.cluster-info.def) vespa_generate_config(configdefinitions cluster-list.def) -install(FILES cluster-list.def RENAME cloud.config.cluster-list.def DESTINATION var/db/vespa/config_server/serverdb/classes) +install_config_definition(cluster-list.def cloud.config.cluster-list.def) vespa_generate_config(configdefinitions configserver.def) -install(FILES configserver.def RENAME cloud.config.configserver.def DESTINATION var/db/vespa/config_server/serverdb/classes) +install_config_definition(configserver.def cloud.config.configserver.def) vespa_generate_config(configdefinitions dispatch.def) -install(FILES dispatch.def RENAME vespa.config.search.dispatch.def DESTINATION var/db/vespa/config_server/serverdb/classes) +install_config_definition(dispatch.def vespa.config.search.dispatch.def) vespa_generate_config(configdefinitions fleetcontroller.def) -install(FILES fleetcontroller.def RENAME vespa.config.content.fleetcontroller.def DESTINATION var/db/vespa/config_server/serverdb/classes) +install_config_definition(fleetcontroller.def vespa.config.content.fleetcontroller.def) vespa_generate_config(configdefinitions ilscripts.def) -install(FILES ilscripts.def RENAME vespa.configdefinition.ilscripts.def DESTINATION var/db/vespa/config_server/serverdb/classes) +install_config_definition(ilscripts.def vespa.configdefinition.ilscripts.def) vespa_generate_config(configdefinitions imported-fields.def) -install(FILES imported-fields.def RENAME vespa.config.search.imported-fields.def DESTINATION var/db/vespa/config_server/serverdb/classes) +install_config_definition(imported-fields.def vespa.config.search.imported-fields.def) vespa_generate_config(configdefinitions indexschema.def) -install(FILES indexschema.def RENAME vespa.config.search.indexschema.def DESTINATION var/db/vespa/config_server/serverdb/classes) +install_config_definition(indexschema.def vespa.config.search.indexschema.def) vespa_generate_config(configdefinitions lb-services.def) -install(FILES lb-services.def RENAME cloud.config.lb-services.def DESTINATION var/db/vespa/config_server/serverdb/classes) +install_config_definition(lb-services.def cloud.config.lb-services.def) vespa_generate_config(configdefinitions load-type.def) -install(FILES load-type.def RENAME vespa.config.content.load-type.def DESTINATION var/db/vespa/config_server/serverdb/classes) +install_config_definition(load-type.def vespa.config.content.load-type.def) vespa_generate_config(configdefinitions messagetyperouteselectorpolicy.def) -install(FILES messagetyperouteselectorpolicy.def RENAME vespa.config.content.messagetyperouteselectorpolicy.def DESTINATION var/db/vespa/config_server/serverdb/classes) +install_config_definition(messagetyperouteselectorpolicy.def vespa.config.content.messagetyperouteselectorpolicy.def) vespa_generate_config(configdefinitions model.def) -install(FILES model.def RENAME cloud.config.model.def DESTINATION var/db/vespa/config_server/serverdb/classes) +install_config_definition(model.def cloud.config.model.def) vespa_generate_config(configdefinitions orchestrator.def) -install(FILES orchestrator.def RENAME vespa.orchestrator.config.orchestrator.def DESTINATION var/db/vespa/config_server/serverdb/classes) +install_config_definition(orchestrator.def vespa.orchestrator.config.orchestrator.def) vespa_generate_config(configdefinitions persistence.def) -install(FILES persistence.def RENAME vespa.config.content.persistence.def DESTINATION var/db/vespa/config_server/serverdb/classes) +install_config_definition(persistence.def vespa.config.content.persistence.def) vespa_generate_config(configdefinitions rank-profiles.def) -install(FILES rank-profiles.def RENAME vespa.config.search.rank-profiles.def DESTINATION var/db/vespa/config_server/serverdb/classes) +install_config_definition(rank-profiles.def vespa.config.search.rank-profiles.def) vespa_generate_config(configdefinitions routing.def) -install(FILES routing.def RENAME cloud.config.routing.def DESTINATION var/db/vespa/config_server/serverdb/classes) +install_config_definition(routing.def cloud.config.routing.def) vespa_generate_config(configdefinitions routing-provider.def) -install(FILES routing-provider.def RENAME cloud.config.routing-provider.def DESTINATION var/db/vespa/config_server/serverdb/classes) +install_config_definition(routing-provider.def cloud.config.routing-provider.def) vespa_generate_config(configdefinitions sentinel.def) -install(FILES sentinel.def RENAME cloud.config.sentinel.def DESTINATION var/db/vespa/config_server/serverdb/classes) +install_config_definition(sentinel.def cloud.config.sentinel.def) vespa_generate_config(configdefinitions slobroks.def) -install(FILES slobroks.def RENAME cloud.config.slobroks.def DESTINATION var/db/vespa/config_server/serverdb/classes) +install_config_definition(slobroks.def cloud.config.slobroks.def) vespa_generate_config(configdefinitions specialtokens.def) -install(FILES specialtokens.def RENAME vespa.configdefinition.specialtokens.def DESTINATION var/db/vespa/config_server/serverdb/classes) +install_config_definition(specialtokens.def vespa.configdefinition.specialtokens.def) vespa_generate_config(configdefinitions stor-devices.def) -install(FILES stor-devices.def RENAME vespa.config.storage.stor-devices.def DESTINATION var/db/vespa/config_server/serverdb/classes) +install_config_definition(stor-devices.def vespa.config.storage.stor-devices.def) vespa_generate_config(configdefinitions stor-distribution.def) -install(FILES stor-distribution.def RENAME vespa.config.content.stor-distribution.def DESTINATION var/db/vespa/config_server/serverdb/classes) +install_config_definition(stor-distribution.def vespa.config.content.stor-distribution.def) vespa_generate_config(configdefinitions stor-filestor.def) -install(FILES stor-filestor.def RENAME vespa.config.content.stor-filestor.def DESTINATION var/db/vespa/config_server/serverdb/classes) +install_config_definition(stor-filestor.def vespa.config.content.stor-filestor.def) vespa_generate_config(configdefinitions stor-memfilepersistence.def) -install(FILES stor-memfilepersistence.def RENAME vespa.config.storage.stor-memfilepersistence.def DESTINATION var/db/vespa/config_server/serverdb/classes) +install_config_definition(stor-memfilepersistence.def vespa.config.storage.stor-memfilepersistence.def) vespa_generate_config(configdefinitions summary.def) -install(FILES summary.def RENAME vespa.config.search.summary.def DESTINATION var/db/vespa/config_server/serverdb/classes) +install_config_definition(summary.def vespa.config.search.summary.def) vespa_generate_config(configdefinitions summarymap.def) -install(FILES summarymap.def RENAME vespa.config.search.summarymap.def DESTINATION var/db/vespa/config_server/serverdb/classes) +install_config_definition(summarymap.def vespa.config.search.summarymap.def) vespa_generate_config(configdefinitions upgrading.def) -install(FILES upgrading.def RENAME vespa.config.content.upgrading.def DESTINATION var/db/vespa/config_server/serverdb/classes) +install_config_definition(upgrading.def vespa.config.content.upgrading.def) vespa_generate_config(configdefinitions ymon.def) -install(FILES ymon.def RENAME cloud.config.ymon.def DESTINATION var/db/vespa/config_server/serverdb/classes) +install_config_definition(ymon.def cloud.config.ymon.def) vespa_generate_config(configdefinitions zookeeper-server.def) -install(FILES zookeeper-server.def RENAME cloud.config.zookeeper-server.def DESTINATION var/db/vespa/config_server/serverdb/classes) +install_config_definition(zookeeper-server.def cloud.config.zookeeper-server.def) vespa_generate_config(configdefinitions zookeepers.def) -install(FILES zookeepers.def RENAME cloud.config.zookeepers.def DESTINATION var/db/vespa/config_server/serverdb/classes) +install_config_definition(zookeepers.def cloud.config.zookeepers.def) diff --git a/configserver/CMakeLists.txt b/configserver/CMakeLists.txt index e54a93b3d28..8134fde2105 100644 --- a/configserver/CMakeLists.txt +++ b/configserver/CMakeLists.txt @@ -1,9 +1,21 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +install_fat_java_artifact(configserver) + vespa_install_script(src/main/sh/vespa-configserver-remove-state bin) vespa_install_script(src/main/sh/start-filedistribution libexec/vespa) vespa_install_script(src/main/sh/ping-configserver libexec/vespa) vespa_install_script(src/main/sh/start-configserver libexec/vespa) vespa_install_script(src/main/sh/start-logd libexec/vespa) vespa_install_script(src/main/sh/stop-configserver libexec/vespa) + install(DIRECTORY src/main/resources/logd DESTINATION conf) install(DIRECTORY src/main/resources/configserver-app DESTINATION conf) + +install(CODE "execute_process(COMMAND mkdir -p $ENV{DESTDIR}/${CMAKE_INSTALL_PREFIX}/conf/configserver-app/components)") +install(CODE "execute_process(COMMAND mkdir -p $ENV{DESTDIR}/${CMAKE_INSTALL_PREFIX}/conf/configserver-app/config-models)") +install(CODE "execute_process(COMMAND ln -snf ${CMAKE_INSTALL_PREFIX}/lib/jars/config-model-fat.jar $ENV{DESTDIR}/${CMAKE_INSTALL_PREFIX}/conf/configserver-app/components/config-model-fat.jar)") +install(CODE "execute_process(COMMAND ln -snf ${CMAKE_INSTALL_PREFIX}/lib/jars/configserver-jar-with-dependencies.jar $ENV{DESTDIR}/${CMAKE_INSTALL_PREFIX}/conf/configserver-app/components/configserver.jar)") +install(CODE "execute_process(COMMAND ln -snf ${CMAKE_INSTALL_PREFIX}/lib/jars/orchestrator-jar-with-dependencies.jar $ENV{DESTDIR}/${CMAKE_INSTALL_PREFIX}/conf/configserver-app/components/orchestrator.jar)") +install(CODE "execute_process(COMMAND ln -snf ${CMAKE_INSTALL_PREFIX}/lib/jars/node-repository-jar-with-dependencies.jar $ENV{DESTDIR}/${CMAKE_INSTALL_PREFIX}/conf/configserver-app/components/node-repository.jar)") +install(CODE "execute_process(COMMAND ln -snf ${CMAKE_INSTALL_PREFIX}/lib/jars/zkfacade-jar-with-dependencies.jar $ENV{DESTDIR}/${CMAKE_INSTALL_PREFIX}/conf/configserver-app/components/zkfacade.jar)") +install(CODE "execute_process(COMMAND ln -snf ${CMAKE_INSTALL_PREFIX}/conf/configserver-app/components $ENV{DESTDIR}/${CMAKE_INSTALL_PREFIX}/lib/jars/config-models)") diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelController.java b/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelController.java index 1532c05a56b..8e865f96db3 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelController.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelController.java @@ -13,7 +13,7 @@ import com.yahoo.vespa.config.GetConfigRequest; import com.yahoo.vespa.config.buildergen.ConfigDefinition; import com.yahoo.vespa.config.protocol.ConfigResponse; import com.yahoo.vespa.config.protocol.DefContent; -import com.yahoo.vespa.config.server.model.SuperModel; +import com.yahoo.vespa.config.server.model.SuperModelConfigProvider; import com.yahoo.vespa.config.server.rpc.ConfigResponseFactory; import java.io.IOException; @@ -28,12 +28,12 @@ import java.io.StringReader; */ public class SuperModelController { - private final SuperModel model; + private final SuperModelConfigProvider model; private final long generation; private final ConfigDefinitionRepo configDefinitionRepo; private final ConfigResponseFactory responseFactory; - public SuperModelController(SuperModel model, ConfigDefinitionRepo configDefinitionRepo, long generation, ConfigResponseFactory responseFactory) { + public SuperModelController(SuperModelConfigProvider model, ConfigDefinitionRepo configDefinitionRepo, long generation, ConfigResponseFactory responseFactory) { this.model = model; this.configDefinitionRepo = configDefinitionRepo; this.generation = generation; @@ -68,7 +68,7 @@ public class SuperModelController { } } - public SuperModel getSuperModel() { + public SuperModelConfigProvider getSuperModel() { return model; } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelManager.java b/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelManager.java index 06f7ffa66c4..0ee95ab7e08 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelManager.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelManager.java @@ -5,6 +5,7 @@ package com.yahoo.vespa.config.server; import com.google.inject.Inject; import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.config.model.api.ApplicationInfo; +import com.yahoo.config.model.api.SuperModel; import com.yahoo.config.model.api.SuperModelListener; import com.yahoo.config.model.api.SuperModelProvider; import com.yahoo.config.provision.ApplicationId; @@ -12,18 +13,13 @@ import com.yahoo.config.provision.NodeFlavors; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.Zone; import com.yahoo.vespa.config.GenerationCounter; -import com.yahoo.vespa.config.server.application.Application; import com.yahoo.vespa.config.server.application.ApplicationSet; -import com.yahoo.vespa.config.server.model.SuperModel; +import com.yahoo.vespa.config.server.model.SuperModelConfigProvider; import java.time.Instant; import java.util.ArrayList; -import java.util.HashMap; -import java.util.LinkedHashMap; import java.util.List; -import java.util.Map; import java.util.Optional; -import java.util.stream.Collectors; /** * Provides a SuperModel - a model of all application instances, and makes it stays @@ -31,7 +27,7 @@ import java.util.stream.Collectors; */ public class SuperModelManager implements SuperModelProvider { private final Zone zone; - private SuperModel superModel; // Guarded by 'this' monitor + private SuperModelConfigProvider superModelConfigProvider; // Guarded by 'this' monitor private final List<SuperModelListener> listeners = new ArrayList<>(); // Guarded by 'this' monitor // Generation of the super model @@ -46,11 +42,16 @@ public class SuperModelManager implements SuperModelProvider { this.zone = new Zone(configserverConfig, nodeFlavors); this.generationCounter = generationCounter; this.masterGeneration = configserverConfig.masterGeneration(); - makeNewSuperModel(new HashMap<>()); + makeNewSuperModelConfigProvider(new SuperModel()); } + @Override public synchronized SuperModel getSuperModel() { - return superModel; + return superModelConfigProvider.getSuperModel(); + } + + public synchronized SuperModelConfigProvider getSuperModelConfigProvider() { + return superModelConfigProvider; } public synchronized long getGeneration() { @@ -58,60 +59,41 @@ public class SuperModelManager implements SuperModelProvider { } @Override - public synchronized List<ApplicationInfo> snapshot(SuperModelListener listener) { + public synchronized SuperModel snapshot(SuperModelListener listener) { listeners.add(listener); - return superModel.applicationModels().values().stream() - .flatMap(applications -> applications.values().stream()) - .map(Application::toApplicationInfo) - .collect(Collectors.toList()); + return superModelConfigProvider.getSuperModel(); } - public synchronized void configActivated(TenantName tenant, ApplicationSet applicationSet) { - Map<TenantName, Map<ApplicationId, Application>> newModels = createModelCopy(); - if (!newModels.containsKey(tenant)) { - // New application has been activated - newModels.put(tenant, new LinkedHashMap<>()); - } else { - // Application has been redeployed - } + @Override + public Zone getZone() { + return zone; + } + public synchronized void configActivated(TenantName tenant, ApplicationSet applicationSet) { // TODO: Should supermodel care about multiple versions? - Application application = applicationSet.getForVersionOrLatest(Optional.empty(), Instant.now()); - newModels.get(tenant).put(applicationSet.getId(), application); - - makeNewSuperModel(newModels); - listeners.stream().forEach(listener -> listener.applicationActivated(application.toApplicationInfo())); + ApplicationInfo applicationInfo = applicationSet + .getForVersionOrLatest(Optional.empty(), Instant.now()) + .toApplicationInfo(); + + SuperModel newSuperModel = this.superModelConfigProvider + .getSuperModel() + .cloneAndSetApplication(applicationInfo); + makeNewSuperModelConfigProvider(newSuperModel); + listeners.stream().forEach(listener -> + listener.applicationActivated(newSuperModel, applicationInfo.getApplicationId())); } public synchronized void applicationRemoved(ApplicationId applicationId) { - Map<TenantName, Map<ApplicationId, Application>> newModels = createModelCopy(); - if (newModels.containsKey(applicationId.tenant())) { - newModels.get(applicationId.tenant()).remove(applicationId); - if (newModels.get(applicationId.tenant()).isEmpty()) { - newModels.remove(applicationId.tenant()); - } - } - - makeNewSuperModel(newModels); - listeners.stream().forEach(listener -> listener.applicationRemoved(applicationId)); + SuperModel newSuperModel = this.superModelConfigProvider + .getSuperModel() + .cloneAndRemoveApplication(applicationId); + makeNewSuperModelConfigProvider(newSuperModel); + listeners.stream().forEach(listener -> + listener.applicationRemoved(newSuperModel, applicationId)); } - private void makeNewSuperModel(Map<TenantName, Map<ApplicationId, Application>> newModels) { + private void makeNewSuperModelConfigProvider(SuperModel newSuperModel) { generation = masterGeneration + generationCounter.get(); - superModel = new SuperModel(newModels, zone); - } - - private Map<TenantName, Map<ApplicationId, Application>> createModelCopy() { - Map<TenantName, Map<ApplicationId, Application>> currentModels = superModel.applicationModels(); - Map<TenantName, Map<ApplicationId, Application>> newModels = new LinkedHashMap<>(); - for (Map.Entry<TenantName, Map<ApplicationId, Application>> entry : currentModels.entrySet()) { - Map<ApplicationId, Application> appMap = new LinkedHashMap<>(); - newModels.put(entry.getKey(), appMap); - for (Map.Entry<ApplicationId, Application> appEntry : entry.getValue().entrySet()) { - appMap.put(appEntry.getKey(), appEntry.getValue()); - } - } - - return newModels; + superModelConfigProvider = new SuperModelConfigProvider(newSuperModel, zone); } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelRequestHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelRequestHandler.java index 95f16a7c1e7..8db7ee9ffc3 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelRequestHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelRequestHandler.java @@ -5,14 +5,11 @@ import com.google.inject.Inject; import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.config.ConfigInstance; import com.yahoo.config.model.api.ConfigDefinitionRepo; -import com.yahoo.config.provision.NodeFlavors; import com.yahoo.config.provision.Version; import com.yahoo.log.LogLevel; import com.yahoo.vespa.config.ConfigKey; -import com.yahoo.vespa.config.GenerationCounter; import com.yahoo.vespa.config.GetConfigRequest; import com.yahoo.vespa.config.protocol.ConfigResponse; -import com.yahoo.vespa.config.server.application.Application; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.TenantName; import com.yahoo.vespa.config.server.application.ApplicationSet; @@ -20,8 +17,6 @@ import com.yahoo.vespa.config.server.rpc.ConfigResponseFactory; import com.yahoo.vespa.config.server.rpc.ConfigResponseFactoryFactory; import java.io.IOException; -import java.util.LinkedHashMap; -import java.util.Map; import java.util.Optional; import java.util.Set; @@ -72,7 +67,7 @@ public class SuperModelRequestHandler implements RequestHandler { private void updateHandler() { handler = new SuperModelController( - superModelManager.getSuperModel(), + superModelManager.getSuperModelConfigProvider(), configDefinitionRepo, superModelManager.getGeneration(), responseFactory); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/model/LbServicesProducer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/model/LbServicesProducer.java index f9665262d76..1806414f510 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/model/LbServicesProducer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/model/LbServicesProducer.java @@ -1,20 +1,21 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.model; +import com.google.common.base.Joiner; +import com.yahoo.cloud.config.LbServicesConfig; +import com.yahoo.config.model.api.ApplicationInfo; +import com.yahoo.config.model.api.HostInfo; +import com.yahoo.config.model.api.ServiceInfo; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.TenantName; +import com.yahoo.config.provision.Zone; + import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.stream.Collectors; import java.util.stream.Stream; -import com.google.common.base.Joiner; -import com.yahoo.config.model.api.HostInfo; -import com.yahoo.config.model.api.ServiceInfo; -import com.yahoo.cloud.config.LbServicesConfig; -import com.yahoo.config.provision.Zone; -import com.yahoo.vespa.config.server.application.Application; -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.TenantName; /** * Produces lb-services cfg @@ -24,10 +25,10 @@ import com.yahoo.config.provision.TenantName; */ public class LbServicesProducer implements LbServicesConfig.Producer { - private final Map<TenantName, Map<ApplicationId, Application>> models; + private final Map<TenantName, Map<ApplicationId, ApplicationInfo>> models; private final Zone zone; - public LbServicesProducer(Map<TenantName, Map<ApplicationId, Application>> models, Zone zone) { + public LbServicesProducer(Map<TenantName, Map<ApplicationId, ApplicationInfo>> models, Zone zone) { this.models = models; this.zone = zone; } @@ -41,7 +42,7 @@ public class LbServicesProducer implements LbServicesConfig.Producer { }); } - private LbServicesConfig.Tenants.Builder getTenantConfig(Map<ApplicationId, Application> apps) { + private LbServicesConfig.Tenants.Builder getTenantConfig(Map<ApplicationId, ApplicationInfo> apps) { LbServicesConfig.Tenants.Builder tb = new LbServicesConfig.Tenants.Builder(); apps.keySet().stream() .sorted() @@ -55,7 +56,7 @@ public class LbServicesProducer implements LbServicesConfig.Producer { return applicationId.application().value() + ":" + zone.environment().value() + ":" + zone.region().value() + ":" + applicationId.instance().value(); } - private LbServicesConfig.Tenants.Applications.Builder getAppConfig(Application app) { + private LbServicesConfig.Tenants.Applications.Builder getAppConfig(ApplicationInfo app) { LbServicesConfig.Tenants.Applications.Builder ab = new LbServicesConfig.Tenants.Applications.Builder(); ab.activeRotation(getActiveRotation(app)); app.getModel().getHosts().stream() @@ -66,7 +67,7 @@ public class LbServicesProducer implements LbServicesConfig.Producer { return ab; } - private boolean getActiveRotation(Application app) { + private boolean getActiveRotation(ApplicationInfo app) { boolean activeRotation = false; for (HostInfo hostInfo : app.getModel().getHosts()) { final Optional<ServiceInfo> container = hostInfo.getServices().stream().filter( diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/model/RoutingProducer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/model/RoutingProducer.java index c8d94235abc..850b3f99d16 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/model/RoutingProducer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/model/RoutingProducer.java @@ -2,11 +2,11 @@ package com.yahoo.vespa.config.server.model; import com.yahoo.cloud.config.RoutingConfig; +import com.yahoo.config.model.api.ApplicationInfo; import com.yahoo.config.model.api.HostInfo; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ApplicationName; import com.yahoo.config.provision.TenantName; -import com.yahoo.vespa.config.server.application.Application; import com.yahoo.vespa.config.server.tenant.Tenants; import java.util.Map; @@ -20,20 +20,22 @@ import java.util.Map; public class RoutingProducer implements RoutingConfig.Producer { static final ApplicationName ROUTING_APPLICATION = ApplicationName.from("routing"); - private final Map<TenantName, Map<ApplicationId, Application>> models; + private final Map<TenantName, Map<ApplicationId, ApplicationInfo>> models; - public RoutingProducer(Map<TenantName, Map<ApplicationId, Application>> models) { + public RoutingProducer(Map<TenantName, Map<ApplicationId, ApplicationInfo>> models) { this.models = models; } @Override public void getConfig(RoutingConfig.Builder builder) { - for (Map<ApplicationId, Application> model : models.values()) { - model.values().stream().filter(application -> isHostedVespaRoutingApplication(application.getId())).forEach(application -> { - for (HostInfo host : application.getModel().getHosts()) { - builder.hosts(host.getHostname()); - } - }); + for (Map<ApplicationId, ApplicationInfo> model : models.values()) { + model.values().stream() + .filter(application -> isHostedVespaRoutingApplication(application.getApplicationId())) + .forEach(application -> { + for (HostInfo host : application.getModel().getHosts()) { + builder.hosts(host.getHostname()); + } + }); } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/model/SuperModel.java b/configserver/src/main/java/com/yahoo/vespa/config/server/model/SuperModelConfigProvider.java index eb41373aab8..e087ef64ae3 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/model/SuperModel.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/model/SuperModelConfigProvider.java @@ -5,15 +5,15 @@ import com.yahoo.cloud.config.LbServicesConfig; import com.yahoo.cloud.config.RoutingConfig; import com.yahoo.config.ConfigInstance; import com.yahoo.config.ConfigurationRuntimeException; +import com.yahoo.config.model.api.ApplicationInfo; +import com.yahoo.config.model.api.SuperModel; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.Zone; import com.yahoo.vespa.config.ConfigKey; import com.yahoo.vespa.config.ConfigPayload; -import com.yahoo.vespa.config.server.application.Application; import java.util.Collections; -import java.util.LinkedHashMap; import java.util.Map; /** @@ -22,16 +22,20 @@ import java.util.Map; * @author vegardh * @since 5.9 */ -public class SuperModel implements LbServicesConfig.Producer, RoutingConfig.Producer { +public class SuperModelConfigProvider implements LbServicesConfig.Producer, RoutingConfig.Producer { - private final Map<TenantName, Map<ApplicationId, Application>> models; + private final SuperModel superModel; private final LbServicesProducer lbProd; private final RoutingProducer zoneProd; - public SuperModel(Map<TenantName, Map<ApplicationId, Application>> models, Zone zone) { - this.models = models; - this.lbProd = new LbServicesProducer(Collections.unmodifiableMap(models), zone); - this.zoneProd = new RoutingProducer(Collections.unmodifiableMap(models)); + public SuperModelConfigProvider(SuperModel superModel, Zone zone) { + this.superModel = superModel; + this.lbProd = new LbServicesProducer(Collections.unmodifiableMap(superModel.getAllModels()), zone); + this.zoneProd = new RoutingProducer(Collections.unmodifiableMap(superModel.getAllModels())); + } + + public SuperModel getSuperModel() { + return superModel; } public ConfigPayload getConfig(ConfigKey<?> configKey) { @@ -49,7 +53,7 @@ public class SuperModel implements LbServicesConfig.Producer, RoutingConfig.Prod } } - public Map<TenantName, Map<ApplicationId, Application>> applicationModels() { return models; } + public Map<TenantName, Map<ApplicationId, ApplicationInfo>> applicationModels() { return superModel.getAllModels(); } @Override public void getConfig(LbServicesConfig.Builder builder) { @@ -65,14 +69,14 @@ public class SuperModel implements LbServicesConfig.Producer, RoutingConfig.Prod ApplicationId applicationId, String configId) { TenantName tenant = applicationId.tenant(); - if (!models.containsKey(tenant)) { + if (!superModel.getAllModels().containsKey(tenant)) { throw new IllegalArgumentException("Tenant " + tenant + " not found"); } - Map<ApplicationId, Application> applications = models.get(tenant); + Map<ApplicationId, ApplicationInfo> applications = superModel.getAllModels().get(tenant); if (!applications.containsKey(applicationId)) { throw new IllegalArgumentException("Application id " + applicationId + " not found"); } - Application application = applications.get(applicationId); + ApplicationInfo application = applications.get(applicationId); ConfigKey<CONFIGTYPE> key = new ConfigKey<>(configClass, configId); ConfigPayload payload = application.getModel().getConfig(key, null); return payload.toInstance(configClass, configId); diff --git a/configserver/src/main/sh/start-configserver b/configserver/src/main/sh/start-configserver index 43bb6b2ba83..3b168b14c41 100755 --- a/configserver/src/main/sh/start-configserver +++ b/configserver/src/main/sh/start-configserver @@ -80,6 +80,7 @@ fixddir () { chmod 755 $1 } +fixddir ${VESPA_HOME}/conf/zookeeper fixfile ${VESPA_HOME}/conf/zookeeper/zookeeper.cfg fixfile ${VESPA_HOME}/var/zookeeper/myid fixddir ${VESPA_HOME}/var/zookeeper/version-2 diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/SuperModelControllerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/SuperModelControllerTest.java index e4835f47e48..c10d5123ea7 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/SuperModelControllerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/SuperModelControllerTest.java @@ -2,22 +2,26 @@ package com.yahoo.vespa.config.server; import com.yahoo.cloud.config.LbServicesConfig; +import com.yahoo.cloud.config.LbServicesConfig.Tenants.Applications; +import com.yahoo.config.model.api.ApplicationInfo; +import com.yahoo.config.model.api.SuperModel; import com.yahoo.config.model.application.provider.FilesApplicationPackage; -import com.yahoo.config.provision.*; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.ApplicationName; +import com.yahoo.config.provision.InstanceName; +import com.yahoo.config.provision.TenantName; +import com.yahoo.config.provision.Version; +import com.yahoo.config.provision.Zone; import com.yahoo.jrt.Request; import com.yahoo.vespa.config.ConfigKey; -import com.yahoo.cloud.config.LbServicesConfig.Tenants.Applications; import com.yahoo.vespa.config.protocol.CompressionType; import com.yahoo.vespa.config.protocol.DefContent; import com.yahoo.vespa.config.protocol.JRTClientConfigRequestV3; import com.yahoo.vespa.config.protocol.JRTServerConfigRequestV3; import com.yahoo.vespa.config.protocol.Trace; -import com.yahoo.vespa.config.server.application.Application; -import com.yahoo.vespa.config.server.model.SuperModel; -import com.yahoo.vespa.config.server.monitoring.MetricUpdater; +import com.yahoo.vespa.config.server.model.SuperModelConfigProvider; import com.yahoo.vespa.config.server.rpc.UncompressedConfigResponseFactory; import com.yahoo.vespa.model.VespaModel; - import org.junit.Before; import org.junit.Test; import org.xml.sax.SAXException; @@ -43,13 +47,14 @@ public class SuperModelControllerTest { @Before public void setupHandler() throws IOException, SAXException { - Map<TenantName, Map<ApplicationId, Application>> models = new LinkedHashMap<>(); + Map<TenantName, Map<ApplicationId, ApplicationInfo>> models = new LinkedHashMap<>(); models.put(TenantName.from("a"), new LinkedHashMap<>()); File testApp = new File("src/test/resources/deploy/app"); ApplicationId app = ApplicationId.from(TenantName.from("a"), ApplicationName.from("foo"), InstanceName.defaultName()); - models.get(app.tenant()).put(app, new Application(new VespaModel(FilesApplicationPackage.fromFile(testApp)), new ServerCache(), 4l, Version.fromIntValues(1, 2, 3), MetricUpdater.createTestUpdater(), app)); - handler = new SuperModelController(new SuperModel(models, Zone.defaultZone()), new TestConfigDefinitionRepo(), 2, new UncompressedConfigResponseFactory()); + models.get(app.tenant()).put(app, new ApplicationInfo(app, 4l, new VespaModel(FilesApplicationPackage.fromFile(testApp)))); + SuperModel superModel = new SuperModel(models); + handler = new SuperModelController(new SuperModelConfigProvider(superModel, Zone.defaultZone()), new TestConfigDefinitionRepo(), 2, new UncompressedConfigResponseFactory()); } @Test @@ -77,7 +82,7 @@ public class SuperModelControllerTest { @Test public void test_lb_config_multiple_apps() throws IOException, SAXException { - Map<TenantName, Map<ApplicationId, Application>> models = new LinkedHashMap<>(); + Map<TenantName, Map<ApplicationId, ApplicationInfo>> models = new LinkedHashMap<>(); models.put(TenantName.from("t1"), new LinkedHashMap<>()); models.put(TenantName.from("t2"), new LinkedHashMap<>()); File testApp1 = new File("src/test/resources/deploy/app"); @@ -86,13 +91,13 @@ public class SuperModelControllerTest { // TODO must fix equals, hashCode on Tenant Version vespaVersion = Version.fromIntValues(1, 2, 3); models.get(TenantName.from("t1")).put(applicationId("mysimpleapp"), - new Application(new VespaModel(FilesApplicationPackage.fromFile(testApp1)), new ServerCache(), 4l, vespaVersion, MetricUpdater.createTestUpdater(), applicationId("mysimpleapp"))); + new ApplicationInfo(applicationId("mysimpleapp"), 4l, new VespaModel(FilesApplicationPackage.fromFile(testApp1)))); models.get(TenantName.from("t1")).put(applicationId("myadvancedapp"), - new Application(new VespaModel(FilesApplicationPackage.fromFile(testApp2)), new ServerCache(), 4l, vespaVersion, MetricUpdater.createTestUpdater(), applicationId("myadvancedapp"))); + new ApplicationInfo(applicationId("myadvancedapp"), 4l, new VespaModel(FilesApplicationPackage.fromFile(testApp2)))); models.get(TenantName.from("t2")).put(applicationId("minetooadvancedapp"), - new Application(new VespaModel(FilesApplicationPackage.fromFile(testApp3)), new ServerCache(), 4l, vespaVersion, MetricUpdater.createTestUpdater(), applicationId("minetooadvancedapp"))); - - SuperModelController han = new SuperModelController(new SuperModel(models, Zone.defaultZone()), new TestConfigDefinitionRepo(), 2, new UncompressedConfigResponseFactory()); + new ApplicationInfo(applicationId("minetooadvancedapp"), 4l, new VespaModel(FilesApplicationPackage.fromFile(testApp3)))); + SuperModel superModel = new SuperModel(models); + SuperModelController han = new SuperModelController(new SuperModelConfigProvider(superModel, Zone.defaultZone()), new TestConfigDefinitionRepo(), 2, new UncompressedConfigResponseFactory()); LbServicesConfig.Builder lb = new LbServicesConfig.Builder(); han.getSuperModel().getConfig(lb); LbServicesConfig lbc = new LbServicesConfig(lb); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/SuperModelRequestHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/SuperModelRequestHandlerTest.java index 4c746eba64e..bc07ac7d79c 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/SuperModelRequestHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/SuperModelRequestHandlerTest.java @@ -61,7 +61,7 @@ public class SuperModelRequestHandlerTest { assertThat(controller.getHandler().getGeneration(), is(gen)); // Test that a new app is used when there already exist an application with the same id ApplicationId appId = new ApplicationId.Builder().tenant(tenantA).applicationName("foo").build(); - assertThat(((TestApplication) controller.getHandler().getSuperModel().applicationModels().get(tenantA).get(appId)).version, is(2l)); + assertThat(controller.getHandler().getSuperModel().applicationModels().get(tenantA).get(appId).getGeneration(), is(4l)); gen = counter.increment(); controller.reloadConfig(tenantA, createApp(tenantA, "bar", 2l, 3)); assertThat(controller.getHandler().getGeneration(), is(gen)); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/model/LbServicesProducerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/model/LbServicesProducerTest.java index 4d7f33e6ef9..474b93f6972 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/model/LbServicesProducerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/model/LbServicesProducerTest.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.config.server.model; import com.yahoo.cloud.config.LbServicesConfig; import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.model.NullConfigModelRegistry; +import com.yahoo.config.model.api.ApplicationInfo; import com.yahoo.config.model.api.Model; import com.yahoo.config.model.deploy.DeployProperties; import com.yahoo.config.model.deploy.DeployState; @@ -13,18 +14,20 @@ import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.Rotation; import com.yahoo.config.provision.TenantName; -import com.yahoo.config.provision.Version; import com.yahoo.config.provision.Zone; import com.yahoo.vespa.config.ConfigPayload; -import com.yahoo.vespa.config.server.ServerCache; -import com.yahoo.vespa.config.server.application.Application; -import com.yahoo.vespa.config.server.monitoring.MetricUpdater; import com.yahoo.vespa.model.VespaModel; import org.junit.Test; import org.xml.sax.SAXException; import java.io.IOException; -import java.util.*; +import java.util.ArrayList; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Random; +import java.util.Set; import static org.hamcrest.Matchers.is; import static org.junit.Assert.assertFalse; @@ -43,7 +46,7 @@ public class LbServicesProducerTest { @Test public void testDeterministicGetConfig() throws IOException, SAXException { - Map<TenantName, Map<ApplicationId, Application>> testModel = createTestModel(new DeployState.Builder().rotations(rotations)); + Map<TenantName, Map<ApplicationId, ApplicationInfo>> testModel = createTestModel(new DeployState.Builder().rotations(rotations)); LbServicesConfig last = null; for (int i = 0; i < 100; i++) { testModel = randomizeTenant(testModel, i); @@ -57,7 +60,7 @@ public class LbServicesProducerTest { @Test public void testConfigAliases() throws IOException, SAXException { - Map<TenantName, Map<ApplicationId, Application>> testModel = createTestModel(new DeployState.Builder()); + Map<TenantName, Map<ApplicationId, ApplicationInfo>> testModel = createTestModel(new DeployState.Builder()); LbServicesConfig conf = getLbServicesConfig(Zone.defaultZone(), testModel); final LbServicesConfig.Tenants.Applications.Hosts.Services services = conf.tenants("foo").applications("foo:prod:default:default").hosts("foo.foo.yahoo.com").services("qrserver"); assertThat(services.servicealiases().size(), is(1)); @@ -85,12 +88,12 @@ public class LbServicesProducerTest { private LbServicesConfig createModelAndGetLbServicesConfig(RegionName regionName) throws IOException, SAXException { final Zone zone = new Zone(Environment.prod, regionName); - Map<TenantName, Map<ApplicationId, Application>> testModel = createTestModel(new DeployState.Builder(). + Map<TenantName, Map<ApplicationId, ApplicationInfo>> testModel = createTestModel(new DeployState.Builder(). properties(new DeployProperties.Builder().zone(zone).build())); return getLbServicesConfig(new Zone(Environment.prod, regionName), testModel); } - private LbServicesConfig getLbServicesConfig(Zone zone, Map<TenantName, Map<ApplicationId, Application>> testModel) { + private LbServicesConfig getLbServicesConfig(Zone zone, Map<TenantName, Map<ApplicationId, ApplicationInfo>> testModel) { LbServicesProducer producer = new LbServicesProducer(testModel, zone); LbServicesConfig.Builder builder = new LbServicesConfig.Builder(); producer.getConfig(builder); @@ -99,7 +102,7 @@ public class LbServicesProducerTest { @Test public void testConfigAliasesWithRotations() throws IOException, SAXException { - Map<TenantName, Map<ApplicationId, Application>> testModel = createTestModel(new DeployState.Builder().rotations(rotations)); + Map<TenantName, Map<ApplicationId, ApplicationInfo>> testModel = createTestModel(new DeployState.Builder().rotations(rotations)); RegionName regionName = RegionName.from("us-east-1"); LbServicesConfig conf = getLbServicesConfig(new Zone(Environment.prod, regionName), testModel); final LbServicesConfig.Tenants.Applications.Hosts.Services services = conf.tenants("foo").applications("foo:prod:" + regionName.value() + ":default").hosts("foo.foo.yahoo.com").services("qrserver"); @@ -113,8 +116,8 @@ public class LbServicesProducerTest { assertThat(services.endpointaliases(3), is(rotation2)); } - private Map<TenantName, Map<ApplicationId, Application>> randomizeTenant(Map<TenantName, Map<ApplicationId, Application>> testModel, int seed) { - Map<TenantName, Map<ApplicationId, Application>> randomizedTenants = new LinkedHashMap<>(); + private Map<TenantName, Map<ApplicationId, ApplicationInfo>> randomizeTenant(Map<TenantName, Map<ApplicationId, ApplicationInfo>> testModel, int seed) { + Map<TenantName, Map<ApplicationId, ApplicationInfo>> randomizedTenants = new LinkedHashMap<>(); List<TenantName> keys = new ArrayList<>(testModel.keySet()); Collections.shuffle(keys, new Random(seed)); for (TenantName key : keys) { @@ -123,8 +126,8 @@ public class LbServicesProducerTest { return randomizedTenants; } - private Map<ApplicationId, Application> randomizeApplications(Map<ApplicationId, Application> applicationIdApplicationMap, int seed) { - Map<ApplicationId, Application> randomizedApplications = new LinkedHashMap<>(); + private Map<ApplicationId, ApplicationInfo> randomizeApplications(Map<ApplicationId, ApplicationInfo> applicationIdApplicationMap, int seed) { + Map<ApplicationId, ApplicationInfo> randomizedApplications = new LinkedHashMap<>(); List<ApplicationId> keys = new ArrayList<>(applicationIdApplicationMap.keySet()); Collections.shuffle(keys, new Random(seed)); for (ApplicationId key : keys) { @@ -133,8 +136,8 @@ public class LbServicesProducerTest { return randomizedApplications; } - private Map<TenantName, Map<ApplicationId, Application>> createTestModel(DeployState.Builder deployStateBuilder) throws IOException, SAXException { - Map<TenantName, Map<ApplicationId, Application>> tMap = new LinkedHashMap<>(); + private Map<TenantName, Map<ApplicationId, ApplicationInfo>> createTestModel(DeployState.Builder deployStateBuilder) throws IOException, SAXException { + Map<TenantName, Map<ApplicationId, ApplicationInfo>> tMap = new LinkedHashMap<>(); TenantName foo = TenantName.from("foo"); TenantName bar = TenantName.from("bar"); TenantName baz = TenantName.from("baz"); @@ -144,8 +147,8 @@ public class LbServicesProducerTest { return tMap; } - private Map<ApplicationId, Application> createTestApplications(TenantName tenant, DeployState.Builder deploystateBuilder) throws IOException, SAXException { - Map<ApplicationId, Application> aMap = new LinkedHashMap<>(); + private Map<ApplicationId, ApplicationInfo> createTestApplications(TenantName tenant, DeployState.Builder deploystateBuilder) throws IOException, SAXException { + Map<ApplicationId, ApplicationInfo> aMap = new LinkedHashMap<>(); ApplicationId fooApp = new ApplicationId.Builder().tenant(tenant).applicationName("foo").build(); ApplicationId barApp = new ApplicationId.Builder().tenant(tenant).applicationName("bar").build(); ApplicationId bazApp = new ApplicationId.Builder().tenant(tenant).applicationName("baz").build(); @@ -155,15 +158,13 @@ public class LbServicesProducerTest { return aMap; } - private Application createApplication(ApplicationId appId, DeployState.Builder deploystateBuilder) throws IOException, SAXException { - return new Application(createVespaModel(createApplicationPackage( - appId.tenant() + "." + appId.application() + ".yahoo.com", appId.tenant().value() + "." + appId.application().value() + "2.yahoo.com"), - deploystateBuilder), - new ServerCache(), + private ApplicationInfo createApplication(ApplicationId appId, DeployState.Builder deploystateBuilder) throws IOException, SAXException { + return new ApplicationInfo( + appId, 3l, - Version.fromIntValues(1, 2, 3), - MetricUpdater.createTestUpdater(), - appId); + createVespaModel(createApplicationPackage( + appId.tenant() + "." + appId.application() + ".yahoo.com", appId.tenant().value() + "." + appId.application().value() + "2.yahoo.com"), + deploystateBuilder)); } private ApplicationPackage createApplicationPackage(String host1, String host2) { diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/model/RoutingProducerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/model/RoutingProducerTest.java index f50dc7828b0..25030332664 100755 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/model/RoutingProducerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/model/RoutingProducerTest.java @@ -4,15 +4,12 @@ package com.yahoo.vespa.config.server.model; import com.yahoo.cloud.config.RoutingConfig; import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.model.NullConfigModelRegistry; +import com.yahoo.config.model.api.ApplicationInfo; import com.yahoo.config.model.api.Model; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.test.MockApplicationPackage; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.TenantName; -import com.yahoo.config.provision.Version; -import com.yahoo.vespa.config.server.ServerCache; -import com.yahoo.vespa.config.server.application.Application; -import com.yahoo.vespa.config.server.monitoring.MetricUpdater; import com.yahoo.vespa.config.server.tenant.Tenants; import com.yahoo.vespa.model.VespaModel; import org.junit.Test; @@ -31,7 +28,7 @@ import static org.junit.Assert.assertThat; public class RoutingProducerTest { @Test public void testNodesFromRoutingAppOnly() throws Exception { - Map<TenantName, Map<ApplicationId, Application>> testModel = createTestModel(new DeployState.Builder()); + Map<TenantName, Map<ApplicationId, ApplicationInfo>> testModel = createTestModel(new DeployState.Builder()); RoutingProducer producer = new RoutingProducer(testModel); RoutingConfig.Builder builder = new RoutingConfig.Builder(); producer.getConfig(builder); @@ -41,8 +38,8 @@ public class RoutingProducerTest { assertThat(config.hosts(1), is("hosted-vespa.routing2.yahoo.com")); } - private Map<TenantName, Map<ApplicationId, Application>> createTestModel(DeployState.Builder deployStateBuilder) throws IOException, SAXException { - Map<TenantName, Map<ApplicationId, Application>> tMap = new LinkedHashMap<>(); + private Map<TenantName, Map<ApplicationId, ApplicationInfo>> createTestModel(DeployState.Builder deployStateBuilder) throws IOException, SAXException { + Map<TenantName, Map<ApplicationId, ApplicationInfo>> tMap = new LinkedHashMap<>(); TenantName foo = TenantName.from("foo"); TenantName bar = TenantName.from("bar"); TenantName routing = TenantName.from(Tenants.HOSTED_VESPA_TENANT.value()); @@ -52,8 +49,8 @@ public class RoutingProducerTest { return tMap; } - private Map<ApplicationId, Application> createTestApplications(TenantName tenant, DeployState.Builder deploystateBuilder) throws IOException, SAXException { - Map<ApplicationId, Application> aMap = new LinkedHashMap<>(); + private Map<ApplicationId, ApplicationInfo> createTestApplications(TenantName tenant, DeployState.Builder deploystateBuilder) throws IOException, SAXException { + Map<ApplicationId, ApplicationInfo> aMap = new LinkedHashMap<>(); ApplicationId fooApp = new ApplicationId.Builder().tenant(tenant).applicationName("foo").build(); ApplicationId barApp = new ApplicationId.Builder().tenant(tenant).applicationName("bar").build(); ApplicationId routingApp = new ApplicationId.Builder().tenant(tenant).applicationName(RoutingProducer.ROUTING_APPLICATION.value()).build(); @@ -63,15 +60,15 @@ public class RoutingProducerTest { return aMap; } - private Application createApplication(ApplicationId appId, DeployState.Builder deploystateBuilder) throws IOException, SAXException { - return new Application(createVespaModel(createApplicationPackage( - appId.tenant() + "." + appId.application() + ".yahoo.com", appId.tenant().value() + "." + appId.application().value() + "2.yahoo.com"), - deploystateBuilder), - new ServerCache(), + private ApplicationInfo createApplication(ApplicationId appId, DeployState.Builder deploystateBuilder) throws IOException, SAXException { + return new ApplicationInfo( + appId, 3l, - Version.fromIntValues(1, 2, 3), - MetricUpdater.createTestUpdater(), - appId); + createVespaModel( + createApplicationPackage( + appId.tenant() + "." + appId.application() + ".yahoo.com", + appId.tenant().value() + "." + appId.application().value() + "2.yahoo.com"), + deploystateBuilder)); } private ApplicationPackage createApplicationPackage(String host1, String host2) { diff --git a/container-accesslogging/CMakeLists.txt b/container-accesslogging/CMakeLists.txt new file mode 100644 index 00000000000..fb2ba9ac031 --- /dev/null +++ b/container-accesslogging/CMakeLists.txt @@ -0,0 +1,2 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +install_config_definition(src/main/resources/configdefinitions/access-log.def container.core.access-log.def) diff --git a/container-core/CMakeLists.txt b/container-core/CMakeLists.txt new file mode 100644 index 00000000000..2d5723865eb --- /dev/null +++ b/container-core/CMakeLists.txt @@ -0,0 +1,16 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +install_config_definition(src/main/resources/configdefinitions/application-metadata.def container.core.application-metadata.def) +install_config_definition(src/main/resources/configdefinitions/container-document.def container.core.document.container-document.def) +install_config_definition(src/main/resources/configdefinitions/container-http.def container.core.container-http.def) +install_config_definition(src/main/resources/configdefinitions/diagnostics.def container.core.diagnostics.def) +install_config_definition(src/main/resources/configdefinitions/health-monitor.def container.jdisc.config.health-monitor.def) +install_config_definition(src/main/resources/configdefinitions/http-filter.def container.core.http.http-filter.def) +install_config_definition(src/main/resources/configdefinitions/metrics-presentation.def metrics.metrics-presentation.def) +install_config_definition(src/main/resources/configdefinitions/mockservice.def container.handler.test.mockservice.def) +install_config_definition(src/main/resources/configdefinitions/qr-logging.def container.core.qr-logging.def) +install_config_definition(src/main/resources/configdefinitions/qr-searchers.def container.qr-searchers.def) +install_config_definition(src/main/resources/configdefinitions/qr-templates.def container.core.qr-templates.def) +install_config_definition(src/main/resources/configdefinitions/qr.def container.qr.def) +install_config_definition(src/main/resources/configdefinitions/servlet-config.def container.servlet.servlet-config.def) +install_config_definition(src/main/resources/configdefinitions/threadpool.def container.handler.threadpool.def) +install_config_definition(src/main/resources/configdefinitions/vip-status.def container.core.vip-status.def) diff --git a/container-di/CMakeLists.txt b/container-di/CMakeLists.txt new file mode 100644 index 00000000000..c2b033baa92 --- /dev/null +++ b/container-di/CMakeLists.txt @@ -0,0 +1,5 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +install_config_definition(src/main/resources/configdefinitions/bundles.def container.bundles.def) +install_config_definition(src/main/resources/configdefinitions/components.def container.components.def) +install_config_definition(src/main/resources/configdefinitions/jersey-bundles.def container.di.config.jersey-bundles.def) +install_config_definition(src/main/resources/configdefinitions/jersey-injection.def container.di.config.jersey-injection.def) diff --git a/container-disc/CMakeLists.txt b/container-disc/CMakeLists.txt new file mode 100644 index 00000000000..1b661020166 --- /dev/null +++ b/container-disc/CMakeLists.txt @@ -0,0 +1,12 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +install_fat_java_artifact(container-disc) + +vespa_install_script(src/main/sh/vespa-start-container-daemon.sh vespa-start-container-daemon bin) + +install_config_definition(src/main/resources/configdefinitions/container.jdisc.config.http-server.def) +install_config_definition(src/main/resources/configdefinitions/jdisc-bindings.def container.jdisc.jdisc-bindings.def) +install_config_definition(src/main/resources/configdefinitions/jersey-connection.def container.config.jersey.jersey-connection.def) +install_config_definition(src/main/resources/configdefinitions/jersey-init.def container.config.jersey.jersey-init.def) +install_config_definition(src/main/resources/configdefinitions/jersey-web-app-pool.def container.config.jersey.jersey-web-app-pool.def) +install_config_definition(src/main/resources/configdefinitions/metric-defaults.def container.jdisc.config.metric-defaults.def) +install_config_definition(src/main/resources/configdefinitions/score-board.def jdisc.metrics.yamasconsumer.cloud.score-board.def) diff --git a/container-jersey2/CMakeLists.txt b/container-jersey2/CMakeLists.txt new file mode 100644 index 00000000000..d2490563372 --- /dev/null +++ b/container-jersey2/CMakeLists.txt @@ -0,0 +1,2 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +install_fat_java_artifact(container-jersey2) diff --git a/container-messagebus/CMakeLists.txt b/container-messagebus/CMakeLists.txt new file mode 100644 index 00000000000..3cd62d83b73 --- /dev/null +++ b/container-messagebus/CMakeLists.txt @@ -0,0 +1,3 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +install_config_definition(src/main/resources/configdefinitions/container-mbus.def container.jdisc.container-mbus.def) +install_config_definition(src/main/resources/configdefinitions/session.def container.jdisc.config.session.def) diff --git a/container-search-and-docproc/CMakeLists.txt b/container-search-and-docproc/CMakeLists.txt new file mode 100644 index 00000000000..29bbe5bdb0f --- /dev/null +++ b/container-search-and-docproc/CMakeLists.txt @@ -0,0 +1,4 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +install_fat_java_artifact(container-search-and-docproc) + +install_config_definition(src/main/resources/configdefinitions/application-userdata.def container.handler.observability.application-userdata.def) diff --git a/container-search/CMakeLists.txt b/container-search/CMakeLists.txt new file mode 100644 index 00000000000..dcf6c3461d3 --- /dev/null +++ b/container-search/CMakeLists.txt @@ -0,0 +1,28 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +install_config_definition(src/main/resources/configdefinitions/cluster.def search.config.cluster.def) +install_config_definition(src/main/resources/configdefinitions/documentdb-info.def prelude.fastsearch.documentdb-info.def) +install_config_definition(src/main/resources/configdefinitions/emulation.def prelude.emulation.def) +install_config_definition(src/main/resources/configdefinitions/federation.def search.federation.federation.def) +install_config_definition(src/main/resources/configdefinitions/fs4.def container.search.fs4.def) +install_config_definition(src/main/resources/configdefinitions/index-info.def search.config.index-info.def) +install_config_definition(src/main/resources/configdefinitions/keyvalue.def prelude.searcher.keyvalue.def) +install_config_definition(src/main/resources/configdefinitions/legacy-emulation.def container.search.legacy-emulation.def) +install_config_definition(src/main/resources/configdefinitions/lowercasing.def search.querytransform.lowercasing.def) +install_config_definition(src/main/resources/configdefinitions/measure-qps.def search.statistics.measure-qps.def) +install_config_definition(src/main/resources/configdefinitions/page-templates.def search.pagetemplates.page-templates.def) +install_config_definition(src/main/resources/configdefinitions/provider.def search.federation.provider.def) +install_config_definition(src/main/resources/configdefinitions/qr-binary-cache-region.def search.cache.qr-binary-cache-region.def) +install_config_definition(src/main/resources/configdefinitions/qr-binary-cache.def search.cache.qr-binary-cache.def) +install_config_definition(src/main/resources/configdefinitions/qr-monitor.def prelude.cluster.qr-monitor.def) +install_config_definition(src/main/resources/configdefinitions/qr-quotetable.def prelude.searcher.qr-quotetable.def) +install_config_definition(src/main/resources/configdefinitions/qr-start.def search.config.qr-start.def) +install_config_definition(src/main/resources/configdefinitions/query-profiles.def search.query.profile.config.query-profiles.def) +install_config_definition(src/main/resources/configdefinitions/rate-limiting.def search.config.rate-limiting.def) +install_config_definition(src/main/resources/configdefinitions/resolvers.def search.pagetemplates.resolvers.def) +install_config_definition(src/main/resources/configdefinitions/rewrites.def search.query.rewrite.rewrites.def) +install_config_definition(src/main/resources/configdefinitions/search-nodes.def search.config.dispatchprototype.search-nodes.def) +install_config_definition(src/main/resources/configdefinitions/search-with-renderer-handler.def search.handler.search-with-renderer-handler.def) +install_config_definition(src/main/resources/configdefinitions/searchchain-forward.def search.federation.searchchain-forward.def) +install_config_definition(src/main/resources/configdefinitions/semantic-rules.def prelude.semantics.semantic-rules.def) +install_config_definition(src/main/resources/configdefinitions/strict-contracts.def search.federation.strict-contracts.def) +install_config_definition(src/main/resources/configdefinitions/timing-searcher.def search.statistics.timing-searcher.def) diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationList.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationList.java index 5af4523b579..fd3fa867c99 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationList.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationList.java @@ -111,6 +111,11 @@ public class ApplicationList { return listOf(list.stream().filter(a -> ! a.id().instance().value().startsWith("default-pr"))); } + /** Returns the subset of applications that are allowed to upgrade at the given time */ + public ApplicationList canUpgradeAt(Instant instant) { + return listOf(list.stream().filter(a -> a.deploymentSpec().canUpgradeAt(instant))); + } + // ----------------------------------- Sorting /** diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/PolledBuildSystem.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/PolledBuildSystem.java index 41adb4abe6a..0d9330ed8ea 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/PolledBuildSystem.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/PolledBuildSystem.java @@ -12,6 +12,8 @@ import java.util.ArrayList; import java.util.Collections; import java.util.Deque; import java.util.List; +import java.util.Optional; +import java.util.logging.Logger; /** * Stores a queue for each type of job, and offers jobs from each of these to a periodic @@ -22,6 +24,8 @@ import java.util.List; */ public class PolledBuildSystem implements BuildSystem { + private static final Logger log = Logger.getLogger(PolledBuildSystem.class.getName()); + private final Controller controller; private final CuratorDb curator; @@ -77,7 +81,14 @@ public class PolledBuildSystem implements BuildSystem { Deque<ApplicationId> queue = curator.readJobQueue(jobType); for (ApplicationId a : queue) { ApplicationId application = removeFromQueue ? queue.poll() : a; - jobsToRun.add(new BuildJob(projectIdFor(application), jobType.id())); + + Optional<Long> projectId = projectId(application); + if (projectId.isPresent()) { + jobsToRun.add(new BuildJob(projectId.get(), jobType.id())); + } else { + log.warning("Not queuing " + jobType.id() + " for " + application.toShortString() + + " because project ID is missing"); + } // Return only one job at a time for capacity constrained queues if (removeFromQueue && isCapacityConstrained(jobType)) break; @@ -89,8 +100,8 @@ public class PolledBuildSystem implements BuildSystem { } } - private Long projectIdFor(ApplicationId applicationId) { - return controller.applications().require(applicationId).deploymentJobs().projectId().get(); + private Optional<Long> projectId(ApplicationId applicationId) { + return controller.applications().require(applicationId).deploymentJobs().projectId(); } private static boolean isCapacityConstrained(JobType jobType) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/FailureRedeployer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/FailureRedeployer.java index 2c048bfa3ce..43c1755af34 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/FailureRedeployer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/FailureRedeployer.java @@ -50,7 +50,7 @@ public class FailureRedeployer extends Maintainer { } private void retryStuckJobs(List<Application> applications) { - Instant maxAge = controller().clock().instant().minus(jobTimeout); + Instant startOfGracePeriod = controller().clock().instant().minus(jobTimeout); for (Application application : applications) { Optional<JobStatus> job = oldestRunningJob(application); if (!job.isPresent()) { @@ -60,7 +60,7 @@ public class FailureRedeployer extends Maintainer { if (!job.get().type().zone(controller().system()).isPresent()) { continue; } - if (job.get().lastTriggered().get().at().isBefore(maxAge)) { + if (job.get().lastTriggered().get().at().isBefore(startOfGracePeriod)) { triggerFailing(application, "Job " + job.get().type().id() + " has been running for more than " + jobTimeout); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java index a4d50d0c150..54ef5df8ac0 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java @@ -68,6 +68,7 @@ public class Upgrader extends Maintainer { applications = applications.notDeployingApplication(); // wait with applications deploying an application change applications = applications.notFailingOn(version); // try to upgrade only if it hasn't failed on this version applications = applications.notRunningJobFor(change); // do not trigger multiple jobs simultaneously for same upgrade + applications = applications.canUpgradeAt(controller().clock().instant()); // wait with applications that are currently blocking upgrades for (Application application : applications.byIncreasingDeployedVersion().asList()) { try { controller().applications().deploymentTrigger().triggerChange(application.id(), change); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java index 3ab98d31a82..91fca0d37d1 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java @@ -217,7 +217,7 @@ public class ControllerTest { app1 = applications.require(app1.id()); assertEquals("First deployment gets system version", systemVersion, app1.deployedVersion().get()); - assertEquals(systemVersion, tester.configServerClientMock().lastPrepareVersion.get()); + assertEquals(systemVersion, tester.configServer().lastPrepareVersion.get()); // Unexpected deployment tester.deploy(productionUsWest1, app1, applicationPackage); @@ -240,7 +240,7 @@ public class ControllerTest { app1 = applications.require(app1.id()); assertEquals("Application change preserves version", systemVersion, app1.deployedVersion().get()); - assertEquals(systemVersion, tester.configServerClientMock().lastPrepareVersion.get()); + assertEquals(systemVersion, tester.configServer().lastPrepareVersion.get()); // A deployment to the new region gets the same version applicationPackage = new ApplicationPackageBuilder() @@ -251,7 +251,7 @@ public class ControllerTest { tester.deployAndNotify(app1, applicationPackage, true, productionUsEast3); app1 = applications.require(app1.id()); assertEquals("Application change preserves version", systemVersion, app1.deployedVersion().get()); - assertEquals(systemVersion, tester.configServerClientMock().lastPrepareVersion.get()); + assertEquals(systemVersion, tester.configServer().lastPrepareVersion.get()); // Version upgrade changes system version Change.VersionChange change = new Change.VersionChange(newSystemVersion); @@ -263,7 +263,7 @@ public class ControllerTest { app1 = applications.require(app1.id()); assertEquals("Version upgrade changes version", newSystemVersion, app1.deployedVersion().get()); - assertEquals(newSystemVersion, tester.configServerClientMock().lastPrepareVersion.get()); + assertEquals(newSystemVersion, tester.configServer().lastPrepareVersion.get()); } /** Adds a new version, higher than the current system version, makes it the system version and returns it */ @@ -600,14 +600,14 @@ public class ControllerTest { @Test public void testCleanupOfStaleDeploymentData() throws IOException { DeploymentTester tester = new DeploymentTester(); - tester.controllerTester().getZoneRegistryMock().setSystem(SystemName.cd); + tester.controllerTester().zoneRegistry().setSystem(SystemName.cd); Supplier<Map<JobType, JobStatus>> statuses = () -> tester.application(ApplicationId.from("vespa", "canary", "default")).deploymentJobs().jobStatus(); // Current system version, matches version in test data Version version = Version.fromString("6.141.117"); - tester.configServerClientMock().setDefaultConfigServerVersion(version); + tester.configServer().setDefaultConfigServerVersion(version); tester.updateVersionStatus(version); assertEquals(version, tester.controller().versionStatus().systemVersion().get().versionNumber()); @@ -637,7 +637,7 @@ public class ControllerTest { // New version is released version = Version.fromString("6.142.1"); - tester.configServerClientMock().setDefaultConfigServerVersion(version); + tester.configServer().setDefaultConfigServerVersion(version); tester.updateVersionStatus(version); assertEquals(version, tester.controller().versionStatus().systemVersion().get().versionNumber()); tester.upgrader().maintain(); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java index 63feb01f1f2..b5419bca0a5 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java @@ -9,7 +9,6 @@ import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.Zone; import com.yahoo.test.ManualClock; -import com.yahoo.vespa.curator.Lock; import com.yahoo.vespa.hosted.controller.api.Tenant; import com.yahoo.vespa.hosted.controller.api.application.v4.model.DeployOptions; import com.yahoo.vespa.hosted.controller.api.application.v4.model.GitRevision; @@ -22,6 +21,8 @@ import com.yahoo.vespa.hosted.controller.api.identifiers.Property; import com.yahoo.vespa.hosted.controller.api.identifiers.PropertyId; import com.yahoo.vespa.hosted.controller.api.identifiers.ScrewdriverId; import com.yahoo.vespa.hosted.controller.api.identifiers.TenantId; +import com.yahoo.vespa.hosted.controller.api.integration.athens.mock.AthensDbMock; +import com.yahoo.vespa.hosted.controller.api.integration.athens.mock.AthensMock; import com.yahoo.vespa.hosted.controller.api.integration.chef.ChefMock; import com.yahoo.vespa.hosted.controller.api.integration.dns.MemoryNameService; import com.yahoo.vespa.hosted.controller.api.integration.entity.MemoryEntityService; @@ -29,7 +30,6 @@ import com.yahoo.vespa.hosted.controller.api.integration.github.GitHubMock; import com.yahoo.vespa.hosted.controller.api.integration.jira.JiraMock; import com.yahoo.vespa.hosted.controller.api.integration.routing.MemoryGlobalRoutingService; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; -import com.yahoo.vespa.hosted.controller.application.Change; import com.yahoo.vespa.hosted.controller.cost.CostMock; import com.yahoo.vespa.hosted.controller.cost.MockInsightBackend; import com.yahoo.vespa.hosted.controller.integration.MockMetricsService; @@ -40,8 +40,6 @@ import com.yahoo.vespa.hosted.controller.persistence.MockCuratorDb; import com.yahoo.vespa.hosted.controller.routing.MockRoutingGenerator; import com.yahoo.vespa.hosted.controller.versions.VersionStatus; import com.yahoo.vespa.hosted.rotation.MemoryRotationRepository; -import com.yahoo.vespa.hosted.controller.api.integration.athens.mock.AthensMock; -import com.yahoo.vespa.hosted.controller.api.integration.athens.mock.AthensDbMock; import java.util.Optional; @@ -50,72 +48,69 @@ import static org.junit.Assert.assertTrue; /** * Convenience methods for controller tests. - * This completely wraps TestEnvironment to make it easier to get rid of that in the future. - * + * * @author bratseth + * @author mpolden */ public final class ControllerTester { - private final ControllerDb db = new MemoryControllerDb(); - private final AthensDbMock athensDb = new AthensDbMock(); - private final ManualClock clock = new ManualClock(); - private final ConfigServerClientMock configServerClientMock = new ConfigServerClientMock(); - private final ZoneRegistryMock zoneRegistryMock = new ZoneRegistryMock(); - private final GitHubMock gitHubMock = new GitHubMock(); - private final CuratorDb curator = new MockCuratorDb(); - private final MemoryNameService memoryNameService = new MemoryNameService(); - private Controller controller = createController(db, curator, configServerClientMock, clock, gitHubMock, - zoneRegistryMock, athensDb, memoryNameService); - - private static final Controller createController(ControllerDb db, CuratorDb curator, - ConfigServerClientMock configServerClientMock, ManualClock clock, - GitHubMock gitHubClientMock, ZoneRegistryMock zoneRegistryMock, - AthensDbMock athensDb, MemoryNameService nameService) { - Controller controller = new Controller(db, - curator, - new MemoryRotationRepository(), - gitHubClientMock, - new JiraMock(), - new MemoryEntityService(), - new MemoryGlobalRoutingService(), - zoneRegistryMock, - new CostMock(new MockInsightBackend()), - configServerClientMock, - new MockMetricsService(), - nameService, - new MockRoutingGenerator(), - new ChefMock(), - clock, - new AthensMock(athensDb)); - controller.updateVersionStatus(VersionStatus.compute(controller)); - return controller; + private final ControllerDb db; + private final AthensDbMock athensDb; + private final ManualClock clock; + private final ConfigServerClientMock configServer; + private final ZoneRegistryMock zoneRegistry; + private final GitHubMock gitHub; + private final CuratorDb curator; + private final MemoryNameService nameService; + + private Controller controller; + + public ControllerTester() { + this(new MemoryControllerDb(), new AthensDbMock(), new ManualClock(), new ConfigServerClientMock(), + new ZoneRegistryMock(), new GitHubMock(), new MockCuratorDb(), new MemoryNameService()); + } + + public ControllerTester(ManualClock clock) { + this(new MemoryControllerDb(), new AthensDbMock(), clock, new ConfigServerClientMock(), + new ZoneRegistryMock(), new GitHubMock(), new MockCuratorDb(), new MemoryNameService()); + } + + private ControllerTester(ControllerDb db, AthensDbMock athensDb, ManualClock clock, + ConfigServerClientMock configServer, ZoneRegistryMock zoneRegistry, + GitHubMock gitHub, CuratorDb curator, MemoryNameService nameService) { + this.db = db; + this.athensDb = athensDb; + this.clock = clock; + this.configServer = configServer; + this.zoneRegistry = zoneRegistry; + this.gitHub = gitHub; + this.curator = curator; + this.nameService = nameService; + this.controller = createController(db, curator, configServer, clock, gitHub, zoneRegistry, + athensDb, nameService); } public Controller controller() { return controller; } + public CuratorDb curator() { return curator; } + public ManualClock clock() { return clock; } + public AthensDbMock athensDb() { return athensDb; } - public MemoryNameService nameService() { return memoryNameService; } - /** Create a new controller instance. Useful to verify that controller state is rebuilt from persistence */ - public final void createNewController() { - controller = createController(db, curator, configServerClientMock, clock, gitHubMock, zoneRegistryMock, - athensDb, memoryNameService); - } + public MemoryNameService nameService() { return nameService; } - public ZoneRegistryMock getZoneRegistryMock() { return zoneRegistryMock; } + public ZoneRegistryMock zoneRegistry() { return zoneRegistry; } - public ConfigServerClientMock configServerClientMock() { return configServerClientMock; } + public ConfigServerClientMock configServer() { return configServer; } - public GitHubMock gitHubClientMock () { return gitHubMock; } + public GitHubMock gitHub() { return gitHub; } - /** Set the application with the given id to currently be in the progress of rolling out the given change */ - public void setDeploying(ApplicationId id, Optional<Change> change) { - try (Lock lock = controller.applications().lock(id)) { - controller.applications().store(controller.applications().require(id).withDeploying(change), lock); - } + /** Create a new controller instance. Useful to verify that controller state is rebuilt from persistence */ + public final void createNewController() { + controller = createController(db, curator, configServer, clock, gitHub, zoneRegistry, athensDb, nameService); } - + /** Creates the given tenant and application and deploys it */ public Application createAndDeploy(String tenantName, String domainName, String applicationName, Environment environment, long projectId, Long propertyId) { return createAndDeploy(tenantName, domainName, applicationName, toZone(environment), projectId, propertyId); @@ -203,4 +198,28 @@ public final class ControllerTester { InstanceName.from(instance)); } + private static Controller createController(ControllerDb db, CuratorDb curator, + ConfigServerClientMock configServerClientMock, ManualClock clock, + GitHubMock gitHubClientMock, ZoneRegistryMock zoneRegistryMock, + AthensDbMock athensDb, MemoryNameService nameService) { + Controller controller = new Controller(db, + curator, + new MemoryRotationRepository(), + gitHubClientMock, + new JiraMock(), + new MemoryEntityService(), + new MemoryGlobalRoutingService(), + zoneRegistryMock, + new CostMock(new MockInsightBackend()), + configServerClientMock, + new MockMetricsService(), + nameService, + new MockRoutingGenerator(), + new ChefMock(), + clock, + new AthensMock(athensDb)); + controller.updateVersionStatus(VersionStatus.compute(controller)); + return controller; + } + } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ApplicationPackageBuilder.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ApplicationPackageBuilder.java index 29cd35d1efc..f193d4718c6 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ApplicationPackageBuilder.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ApplicationPackageBuilder.java @@ -28,6 +28,7 @@ public class ApplicationPackageBuilder { private Environment environment = Environment.prod; private final StringBuilder environmentBody = new StringBuilder(); private final StringBuilder validationOverridesBody = new StringBuilder(); + private final StringBuilder blockUpgrade = new StringBuilder(); public ApplicationPackageBuilder upgradePolicy(String upgradePolicy) { this.upgradePolicy = upgradePolicy; @@ -60,6 +61,17 @@ public class ApplicationPackageBuilder { return this; } + public ApplicationPackageBuilder blockUpgrade(String daySpec, String hourSpec, String zoneSpec) { + blockUpgrade.append(" <block-upgrade days=\""); + blockUpgrade.append(daySpec); + blockUpgrade.append("\" hours=\""); + blockUpgrade.append(hourSpec); + blockUpgrade.append("\" time-zone=\""); + blockUpgrade.append(zoneSpec); + blockUpgrade.append("\"/>\n"); + return this; + } + public ApplicationPackageBuilder allow(ValidationId validationId) { validationOverridesBody.append(" <allow until='"); validationOverridesBody.append(asIso8601Date(Instant.now().plus(Duration.ofDays(29)))); @@ -76,6 +88,7 @@ public class ApplicationPackageBuilder { xml.append(upgradePolicy); xml.append("'/>\n"); } + xml.append(blockUpgrade); xml.append(" <"); xml.append(environment.value()); xml.append(">\n"); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java index 3edbcea9843..44309b43a5f 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java @@ -30,26 +30,48 @@ import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; /** + * This class provides convenience methods for testing deployments + * * @author bratseth + * @author mpolden */ public class DeploymentTester { - private ControllerTester tester = new ControllerTester(); - - private Upgrader upgrader = new Upgrader(tester.controller(), Duration.ofMinutes(2), - new JobControl(tester.curator())); - private FailureRedeployer failureRedeployer = new FailureRedeployer(tester.controller(), Duration.ofMinutes(2), - new JobControl(tester.curator())); + private final ControllerTester tester; + private final Upgrader upgrader; + private final FailureRedeployer failureRedeployer; + + public DeploymentTester() { + this(new ControllerTester()); + } + + public DeploymentTester(ControllerTester tester) { + this.tester = tester; + this.upgrader = new Upgrader(tester.controller(), Duration.ofMinutes(2), + new JobControl(tester.curator())); + this.failureRedeployer = new FailureRedeployer(tester.controller(), + Duration.ofMinutes(2), + new JobControl(tester.curator())); + } public Upgrader upgrader() { return upgrader; } + public FailureRedeployer failureRedeployer() { return failureRedeployer; } + public Controller controller() { return tester.controller(); } + public ApplicationController applications() { return tester.controller().applications(); } + public BuildSystem buildSystem() { return tester.controller().applications().deploymentTrigger().buildSystem(); } + public DeploymentTrigger deploymentTrigger() { return tester.controller().applications().deploymentTrigger(); } + public ManualClock clock() { return tester.clock(); } + public ControllerTester controllerTester() { return tester; } + public ConfigServerClientMock configServer() { return tester.configServer(); } + public Application application(String name) { return application(ApplicationId.from("tenant1", name, "default")); } @@ -63,8 +85,6 @@ public class DeploymentTester { .filter(c -> c instanceof Change.VersionChange) .map(Change.VersionChange.class::cast); } - - public ConfigServerClientMock configServerClientMock() { return tester.configServerClientMock(); } public void updateVersionStatus(Version currentVersion) { controller().updateVersionStatus(VersionStatus.compute(controller(), currentVersion)); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentExpirerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentExpirerTest.java index 4c53a6d37e4..ee4f3631b54 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentExpirerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentExpirerTest.java @@ -22,7 +22,7 @@ public class DeploymentExpirerTest { @Test public void testDeploymentExpiry() throws IOException, InterruptedException { ControllerTester tester = new ControllerTester(); - tester.getZoneRegistryMock().setDeploymentTimeToLive(new Zone(Environment.dev, RegionName.from("us-east-1")), Duration.ofDays(14)); + tester.zoneRegistry().setDeploymentTimeToLive(new Zone(Environment.dev, RegionName.from("us-east-1")), Duration.ofDays(14)); DeploymentExpirer expirer = new DeploymentExpirer(tester.controller(), Duration.ofDays(10), tester.clock(), new JobControl(new MockCuratorDb())); ApplicationId devApp = tester.createAndDeploy("tenant1", "domain1", "app1", Environment.dev, 123).id(); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/FailureRedeployerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/FailureRedeployerTest.java index fd7a3605766..2d1b7463dea 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/FailureRedeployerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/FailureRedeployerTest.java @@ -243,11 +243,11 @@ public class FailureRedeployerTest { @Test public void retryIgnoresStaleJobData() throws Exception { DeploymentTester tester = new DeploymentTester(); - tester.controllerTester().getZoneRegistryMock().setSystem(SystemName.cd); + tester.controllerTester().zoneRegistry().setSystem(SystemName.cd); // Current system version, matches version in test data Version version = Version.fromString("6.141.117"); - tester.configServerClientMock().setDefaultConfigServerVersion(version); + tester.configServer().setDefaultConfigServerVersion(version); tester.updateVersionStatus(version); assertEquals(version, tester.controller().versionStatus().systemVersion().get().versionNumber()); @@ -266,7 +266,7 @@ public class FailureRedeployerTest { // New version is released version = Version.fromString("6.142.1"); - tester.configServerClientMock().setDefaultConfigServerVersion(version); + tester.configServer().setDefaultConfigServerVersion(version); tester.updateVersionStatus(version); assertEquals(version, tester.controller().versionStatus().systemVersion().get().versionNumber()); tester.upgrader().maintain(); @@ -299,11 +299,11 @@ public class FailureRedeployerTest { @Test public void ignoresPullRequestInstances() throws Exception { DeploymentTester tester = new DeploymentTester(); - tester.controllerTester().getZoneRegistryMock().setSystem(SystemName.cd); + tester.controllerTester().zoneRegistry().setSystem(SystemName.cd); // Current system version, matches version in test data Version version = Version.fromString("6.42.1"); - tester.configServerClientMock().setDefaultConfigServerVersion(version); + tester.configServer().setDefaultConfigServerVersion(version); tester.updateVersionStatus(version); assertEquals(version, tester.controller().versionStatus().systemVersion().get().versionNumber()); @@ -322,4 +322,29 @@ public class FailureRedeployerTest { assertTrue("No jobs scheduled", tester.buildSystem().jobs().isEmpty()); } + @Test + public void applicationWithoutProjectIdIsNotTriggered() throws Exception { + DeploymentTester tester = new DeploymentTester(); + + // Current system version, matches version in test data + Version version = Version.fromString("6.42.1"); + tester.configServer().setDefaultConfigServerVersion(version); + tester.updateVersionStatus(version); + assertEquals(version, tester.controller().versionStatus().systemVersion().get().versionNumber()); + + // Load test data data + ApplicationSerializer serializer = new ApplicationSerializer(); + byte[] json = Files.readAllBytes(Paths.get("src/test/java/com/yahoo/vespa/hosted/controller/maintenance/testdata/application-without-project-id.json")); + Slime slime = SlimeUtils.jsonToSlime(json); + Application application = serializer.fromSlime(slime); + + try (Lock lock = tester.controller().applications().lock(application.id())) { + tester.controller().applications().store(application, lock); + } + + // Failure redeployer does not restart deployment + tester.failureRedeployer().maintain(); + assertTrue("No jobs scheduled", tester.buildSystem().jobs().isEmpty()); + } + } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java index 3046a89efe6..ccdfc8d042e 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java @@ -3,7 +3,9 @@ package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.component.Version; import com.yahoo.config.provision.Environment; +import com.yahoo.test.ManualClock; import com.yahoo.vespa.hosted.controller.Application; +import com.yahoo.vespa.hosted.controller.ControllerTester; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; import com.yahoo.vespa.hosted.controller.application.DeploymentJobs; import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; @@ -12,6 +14,7 @@ import com.yahoo.vespa.hosted.controller.versions.VespaVersion; import org.junit.Test; import java.time.Duration; +import java.time.Instant; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -54,7 +57,7 @@ public class UpgraderTest { assertEquals("New system version: Should upgrade Canaries", 2, tester.buildSystem().jobs().size()); tester.completeUpgrade(canary0, version, "canary"); - assertEquals(version, tester.configServerClientMock().lastPrepareVersion.get()); + assertEquals(version, tester.configServer().lastPrepareVersion.get()); tester.updateVersionStatus(version); tester.upgrader().maintain(); @@ -104,7 +107,7 @@ public class UpgraderTest { assertEquals("New system version: Should upgrade Canaries", 2, tester.buildSystem().jobs().size()); tester.completeUpgrade(canary0, version, "canary"); - assertEquals(version, tester.configServerClientMock().lastPrepareVersion.get()); + assertEquals(version, tester.configServer().lastPrepareVersion.get()); tester.updateVersionStatus(version); tester.upgrader().maintain(); @@ -184,7 +187,7 @@ public class UpgraderTest { assertEquals("New system version: Should upgrade Canaries", 2, tester.buildSystem().jobs().size()); tester.completeUpgrade(canary0, version, "canary"); - assertEquals(version, tester.configServerClientMock().lastPrepareVersion.get()); + assertEquals(version, tester.configServer().lastPrepareVersion.get()); tester.updateVersionStatus(version); tester.upgrader().maintain(); @@ -360,4 +363,41 @@ public class UpgraderTest { assertEquals(VespaVersion.Confidence.normal, tester.controller().versionStatus().systemVersion().get().confidence()); } + @Test + public void testConsidersBlockUpgradeWindow() { + ManualClock clock = new ManualClock(Instant.parse("2017-09-26T18:00:00.00Z")); // A tuesday + DeploymentTester tester = new DeploymentTester(new ControllerTester(clock)); + Version version = Version.fromString("5.0"); + tester.updateVersionStatus(version); + + ApplicationPackage applicationPackage = new ApplicationPackageBuilder() + .upgradePolicy("canary") + // Block upgrades on tuesday in hours 18 and 19 + .blockUpgrade("tue", "18-19", "UTC") + .region("us-west-1") + .build(); + + Application app = tester.createAndDeploy("app1", 1, applicationPackage); + + // New version is released + version = Version.fromString("5.1"); + tester.updateVersionStatus(version); + + // Application is not upgraded at this time + tester.upgrader().maintain(); + assertTrue("No jobs scheduled", tester.buildSystem().jobs().isEmpty()); + + // One hour passes, time is 19:00, still no upgrade + tester.clock().advance(Duration.ofHours(1)); + tester.upgrader().maintain(); + assertTrue("No jobs scheduled", tester.buildSystem().jobs().isEmpty()); + + // Two hours pass in total, time is 20:00 and application upgrades + tester.clock().advance(Duration.ofHours(1)); + tester.upgrader().maintain(); + assertFalse("Job is scheduled", tester.buildSystem().jobs().isEmpty()); + tester.completeUpgrade(app, version, "canary"); + assertTrue("All jobs consumed", tester.buildSystem().jobs().isEmpty()); + } + } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/testdata/application-without-project-id.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/testdata/application-without-project-id.json new file mode 100644 index 00000000000..912d1c2f4dc --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/testdata/application-without-project-id.json @@ -0,0 +1,20 @@ +{ + "id": "tenant1:app1:default", + "deploymentSpecField": "<deployment version='1.0'>\n <test />\n <staging />\n <prod>\n <region active=\"true\">cd-us-central-1</region>\n <region active=\"true\">cd-us-central-2</region>\n </prod>\n</deployment>\n", + "validationOverrides": "<deployment version='1.0'/>", + "deployments": [], + "deploymentJobs": { + "jobStatus": [ + { + "jobType": "system-test", + "lastTriggered": { + "version": "6.42.1", + "upgrade": false, + "at": 1506330088050 + } + } + ], + "selfTriggering": false + }, + "outstandingChangeField": false +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/VersionStatusTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/VersionStatusTest.java index a2fb5f38457..c4a3bd9cd81 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/VersionStatusTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/VersionStatusTest.java @@ -46,7 +46,7 @@ public class VersionStatusTest { public void testSystemVersionIsControllerVersionIfConfigserversAreNewer() { ControllerTester tester = new ControllerTester(); Version largerThanCurrent = new Version(Vtag.currentVersion.getMajor() + 1); - tester.configServerClientMock().setDefaultConfigServerVersion(largerThanCurrent); + tester.configServer().setDefaultConfigServerVersion(largerThanCurrent); VersionStatus versionStatus = VersionStatus.compute(tester.controller()); assertEquals(Vtag.currentVersion, versionStatus.systemVersion().get().versionNumber()); } @@ -55,7 +55,7 @@ public class VersionStatusTest { public void testSystemVersionIsVersionOfOldestConfigServer() throws URISyntaxException { ControllerTester tester = new ControllerTester(); Version oldest = new Version(5); - tester.configServerClientMock().configServerVersions().put(new URI("http://cfg.prod.corp-us-east-1.test"), oldest); + tester.configServer().configServerVersions().put(new URI("http://cfg.prod.corp-us-east-1.test"), oldest); VersionStatus versionStatus = VersionStatus.compute(tester.controller()); assertEquals(oldest, versionStatus.systemVersion().get().versionNumber()); } @@ -95,7 +95,7 @@ public class VersionStatusTest { assertEquals("The version of this controller, the default config server version, plus the two versions above exist", 4, versions.size()); VespaVersion v0 = versions.get(2); - assertEquals(tester.configServerClientMock().getDefaultConfigServerVersion(), v0.versionNumber()); + assertEquals(tester.configServer().getDefaultConfigServerVersion(), v0.versionNumber()); assertEquals(0, v0.statistics().failing().size()); assertEquals(0, v0.statistics().production().size()); @@ -242,7 +242,7 @@ public class VersionStatusTest { ControllerTester tester = new ControllerTester(); ApplicationController applications = tester.controller().applications(); - tester.gitHubClientMock() + tester.gitHub() .mockAny(false) .knownTag(Vtag.currentVersion.toFullString(), "foo") // controller .knownTag("6.1.0", "bar"); // config server diff --git a/defaults/CMakeLists.txt b/defaults/CMakeLists.txt index 2b7f719d297..c42e5402688 100644 --- a/defaults/CMakeLists.txt +++ b/defaults/CMakeLists.txt @@ -6,3 +6,5 @@ vespa_define_module( APPS src/apps/printdefault ) + +install_fat_java_artifact(defaults) @@ -9,5 +9,5 @@ fi VERSION="$1" mkdir -p ~/rpmbuild/{SOURCES,SPECS} -GZIP=-1 tar -zcf ~/rpmbuild/SOURCES/vespa-$VERSION.tar.gz --transform "flags=r;s,^,vespa-$VERSION/," * +GZIP=-1 tar -zcf ~/rpmbuild/SOURCES/vespa-$VERSION.tar.gz --exclude target --exclude cmake-build-debug --transform "flags=r;s,^,vespa-$VERSION/," * sed -e "s,VESPA_VERSION,$VERSION," < dist/vespa.spec > ~/rpmbuild/SPECS/vespa-$VERSION.spec diff --git a/dist/post_install.sh b/dist/post_install.sh deleted file mode 100755 index fb0e19370ee..00000000000 --- a/dist/post_install.sh +++ /dev/null @@ -1,76 +0,0 @@ -#!/bin/bash -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -if [ $# -ne 1 ]; then - echo "Usage: $0 <install prefix>" - exit 1 -fi - -declare -r PREFIX="$1" -declare -r INSTALLPATH="$DESTDIR/$PREFIX" - -# Rewrite config def file names -for path in $INSTALLPATH/var/db/vespa/config_server/serverdb/classes/*.def; do - dir=$(dirname $path) - filename=$(basename $path) - namespace=$(grep '^ *namespace *=' $path | sed 's/ *namespace *= *//') - if [ "$namespace" ]; then - case $filename in - $namespace.*) - ;; - *) - mv $path $dir/$namespace.$filename ;; - esac - fi -done - -mkdir -p $INSTALLPATH/conf/configserver/ -mkdir -p $INSTALLPATH/conf/configserver-app/ -mkdir -p $INSTALLPATH/conf/configserver-app/config-models/ -mkdir -p $INSTALLPATH/conf/configserver-app/components/ -mkdir -p $INSTALLPATH/conf/filedistributor/ -mkdir -p $INSTALLPATH/conf/node-admin-app/ -mkdir -p $INSTALLPATH/conf/node-admin-app/components/ -mkdir -p $INSTALLPATH/conf/zookeeper/ -mkdir -p $INSTALLPATH/libexec/jdisc_core/ -mkdir -p $INSTALLPATH/libexec/vespa/modelplugins/ -mkdir -p $INSTALLPATH/libexec/vespa/plugins/qrs/ -mkdir -p $INSTALLPATH/logs/jdisc_core/ -mkdir -p $INSTALLPATH/logs/vespa/ -mkdir -p $INSTALLPATH/logs/vespa/ -mkdir -p $INSTALLPATH/logs/vespa/configserver/ -mkdir -p $INSTALLPATH/logs/vespa/search/ -mkdir -p $INSTALLPATH/logs/vespa/qrs/ -mkdir -p $INSTALLPATH/share/vespa/ -mkdir -p $INSTALLPATH/share/vespa/schema/version/6.x/schema/ -mkdir -p $INSTALLPATH/tmp/vespa/ -mkdir -p $INSTALLPATH/var/db/jdisc/logcontrol/ -mkdir -p $INSTALLPATH/var/db/vespa/ -mkdir -p $INSTALLPATH/var/db/vespa/config_server/serverdb/applications/ -mkdir -p $INSTALLPATH/var/db/vespa/logcontrol/ -mkdir -p $INSTALLPATH/var/jdisc_container/ -mkdir -p $INSTALLPATH/var/jdisc_core/ -mkdir -p $INSTALLPATH/var/run/ -mkdir -p $INSTALLPATH/var/spool/vespa/ -mkdir -p $INSTALLPATH/var/spool/master/inbox/ -mkdir -p $INSTALLPATH/var/vespa/bundlecache/ -mkdir -p $INSTALLPATH/var/vespa/cache/config/ -mkdir -p $INSTALLPATH/var/vespa/cmdlines/ -mkdir -p $INSTALLPATH/var/zookeeper/version-2/ -mkdir -p $INSTALLPATH/sbin - -ln -sf $PREFIX/lib/jars/config-model-fat.jar $INSTALLPATH/conf/configserver-app/components/config-model-fat.jar -ln -sf $PREFIX/lib/jars/configserver-jar-with-dependencies.jar $INSTALLPATH/conf/configserver-app/components/configserver.jar -ln -sf $PREFIX/lib/jars/orchestrator-jar-with-dependencies.jar $INSTALLPATH/conf/configserver-app/components/orchestrator.jar -ln -sf $PREFIX/lib/jars/node-repository-jar-with-dependencies.jar $INSTALLPATH/conf/configserver-app/components/node-repository.jar -ln -sf $PREFIX/lib/jars/zkfacade-jar-with-dependencies.jar $INSTALLPATH/conf/configserver-app/components/zkfacade.jar -ln -snf $PREFIX/conf/configserver-app/components $INSTALLPATH/lib/jars/config-models -ln -sf vespa-storaged-bin $INSTALLPATH/sbin/vespa-distributord-bin - -# Setup default enviroment -mkdir -p $INSTALLPATH/conf/vespa -cat > $INSTALLPATH/conf/vespa/default-env.txt <<EOF -fallback VESPA_HOME $PREFIX -override VESPA_USER vespa -EOF - diff --git a/docker-api/CMakeLists.txt b/docker-api/CMakeLists.txt new file mode 100644 index 00000000000..25957c81e4c --- /dev/null +++ b/docker-api/CMakeLists.txt @@ -0,0 +1,2 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +install_config_definition(src/main/resources/configdefinitions/docker.def vespa.hosted.dockerapi.docker.def) diff --git a/docproc/CMakeLists.txt b/docproc/CMakeLists.txt new file mode 100644 index 00000000000..bacb45df319 --- /dev/null +++ b/docproc/CMakeLists.txt @@ -0,0 +1,4 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +install_config_definition(src/main/resources/configdefinitions/docproc.def config.docproc.docproc.def) +install_config_definition(src/main/resources/configdefinitions/schemamapping.def config.docproc.schemamapping.def) +install_config_definition(src/main/resources/configdefinitions/splitter-joiner-document-processor.def config.docproc.splitter-joiner-document-processor.def) diff --git a/docprocs/CMakeLists.txt b/docprocs/CMakeLists.txt new file mode 100644 index 00000000000..8786a77cbf8 --- /dev/null +++ b/docprocs/CMakeLists.txt @@ -0,0 +1,2 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +install_fat_java_artifact(docprocs) diff --git a/document/CMakeLists.txt b/document/CMakeLists.txt index ca2ee029c87..72f4ee445fb 100644 --- a/document/CMakeLists.txt +++ b/document/CMakeLists.txt @@ -40,3 +40,5 @@ vespa_define_module( src/tests/struct_anno src/tests/tensor_fieldvalue ) + +install_java_artifact(document) diff --git a/document/src/vespa/document/annotation/spantree.h b/document/src/vespa/document/annotation/spantree.h index 2518db32fe3..e2839c4793b 100644 --- a/document/src/vespa/document/annotation/spantree.h +++ b/document/src/vespa/document/annotation/spantree.h @@ -2,7 +2,7 @@ #pragma once -#include <vespa/document/annotation/annotation.h> +#include "annotation.h" #include <vector> #include <cassert> diff --git a/document/src/vespa/document/config/CMakeLists.txt b/document/src/vespa/document/config/CMakeLists.txt index 32bbce210a5..fc711c5e57a 100644 --- a/document/src/vespa/document/config/CMakeLists.txt +++ b/document/src/vespa/document/config/CMakeLists.txt @@ -4,7 +4,5 @@ vespa_add_library(document_documentconfig OBJECT DEPENDS ) vespa_generate_config(document_documentconfig documenttypes.def) -install(FILES documenttypes.def RENAME document.documenttypes.def - DESTINATION var/db/vespa/config_server/serverdb/classes) -install(FILES documentmanager.def RENAME document.config.documentmanager.def - DESTINATION var/db/vespa/config_server/serverdb/classes) +install_config_definition(documenttypes.def document.documenttypes.def) +install_config_definition(documentmanager.def document.config.documentmanager.def) diff --git a/documentapi/src/vespa/documentapi/messagebus/policies/CMakeLists.txt b/documentapi/src/vespa/documentapi/messagebus/policies/CMakeLists.txt index 143310d1f67..f1a691bc46d 100644 --- a/documentapi/src/vespa/documentapi/messagebus/policies/CMakeLists.txt +++ b/documentapi/src/vespa/documentapi/messagebus/policies/CMakeLists.txt @@ -20,4 +20,4 @@ vespa_add_library(documentapi_documentapipolicies OBJECT DEPENDS ) vespa_generate_config(documentapi_documentapipolicies ../../../../main/resources/configdefinitions/documentrouteselectorpolicy.def) -install(FILES ../../../../main/resources/configdefinitions/documentrouteselectorpolicy.def RENAME documentapi.messagebus.protocol.documentrouteselectorpolicy.def DESTINATION var/db/vespa/config_server/serverdb/classes) +install_config_definition(../../../../main/resources/configdefinitions/documentrouteselectorpolicy.def documentapi.messagebus.protocol.documentrouteselectorpolicy.def) diff --git a/fileacquirer/CMakeLists.txt b/fileacquirer/CMakeLists.txt index 1ae83a6f6c2..7a366aa3882 100644 --- a/fileacquirer/CMakeLists.txt +++ b/fileacquirer/CMakeLists.txt @@ -9,3 +9,5 @@ vespa_define_module( LIBS src/vespa/fileacquirer ) + +install_config_definition(src/main/resources/configdefinitions/filedistributorrpc.def cloud.config.filedistribution.filedistributorrpc.def) diff --git a/filedistribution/src/vespa/filedistribution/distributor/CMakeLists.txt b/filedistribution/src/vespa/filedistribution/distributor/CMakeLists.txt index 5345cba1e3f..f85ab85fb39 100644 --- a/filedistribution/src/vespa/filedistribution/distributor/CMakeLists.txt +++ b/filedistribution/src/vespa/filedistribution/distributor/CMakeLists.txt @@ -12,4 +12,4 @@ vespa_add_library(filedistribution_distributor STATIC ) target_compile_options(filedistribution_distributor PRIVATE -DTORRENT_DISABLE_ENCRYPTION -DTORRENT_DISABLE_DHT -DWITH_SHIPPED_GEOIP_H -DBOOST_ASIO_HASH_MAP_BUCKETS=1021 -DBOOST_EXCEPTION_DISABLE -DBOOST_ASIO_ENABLE_CANCELIO -DBOOST_ASIO_DYN_LINK -DTORRENT_LINKING_SHARED) vespa_generate_config(filedistribution_distributor filedistributor.def) -install(FILES filedistributor.def RENAME cloud.config.filedistribution.filedistributor.def DESTINATION var/db/vespa/config_server/serverdb/classes) +install_config_definition(filedistributor.def cloud.config.filedistribution.filedistributor.def) diff --git a/filedistribution/src/vespa/filedistribution/model/CMakeLists.txt b/filedistribution/src/vespa/filedistribution/model/CMakeLists.txt index 7136aba475b..5b92aa4086d 100644 --- a/filedistribution/src/vespa/filedistribution/model/CMakeLists.txt +++ b/filedistribution/src/vespa/filedistribution/model/CMakeLists.txt @@ -17,4 +17,4 @@ vespa_add_library(filedistribution_filedistributionmodel STATIC vespa_generate_config(filedistribution_filedistributionmodel filereferences.def) vespa_add_target_external_dependency(filedistribution_filedistributionmodel zookeeper_mt) -install(FILES filereferences.def RENAME cloud.config.filedistribution.filereferences.def DESTINATION var/db/vespa/config_server/serverdb/classes) +install_config_definition(filereferences.def cloud.config.filedistribution.filereferences.def) diff --git a/functions.cmake b/functions.cmake index 38ae339cd53..1c3d1ff7a6e 100644 --- a/functions.cmake +++ b/functions.cmake @@ -536,3 +536,23 @@ function(__export_include_directories TARGET) target_include_directories(${TARGET} PUBLIC ${LOCAL_INCLUDE_DIRS}) endif() endfunction() + +function(install_config_definition) + if(ARGC GREATER 1) + install(FILES ${ARGV0} RENAME ${ARGV1} DESTINATION var/db/vespa/config_server/serverdb/classes) + else() + install(FILES ${ARGV0} DESTINATION var/db/vespa/config_server/serverdb/classes) + endif() +endfunction() + +function(install_java_artifact NAME) + install(FILES "target/${NAME}.jar" DESTINATION lib/jars/) +endfunction() + +function(install_java_artifact_dependencies NAME) + install(DIRECTORY "target/dependency/" DESTINATION lib/jars FILES_MATCHING PATTERN "*.jar") +endfunction() + +function(install_fat_java_artifact NAME) + install(FILES "target/${NAME}-jar-with-dependencies.jar" DESTINATION lib/jars/) +endfunction() diff --git a/install_java.cmake b/install_java.cmake deleted file mode 100644 index e0611306b5f..00000000000 --- a/install_java.cmake +++ /dev/null @@ -1,161 +0,0 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -function(install_java_artifact NAME) - install(FILES "${NAME}/target/${NAME}.jar" DESTINATION lib/jars/) -endfunction() - -function(install_java_artifact_dependencies NAME) - install(DIRECTORY "${NAME}/target/dependency/" DESTINATION lib/jars FILES_MATCHING PATTERN "*.jar") -endfunction() - -function(install_fat_java_artifact NAME) - install(FILES "${NAME}/target/${NAME}-jar-with-dependencies.jar" DESTINATION lib/jars/) -endfunction() - -install_java_artifact(config-model-fat) -install_java_artifact(document) -install_java_artifact(jdisc_jetty) -install_java_artifact_dependencies(jdisc_jetty) -install_java_artifact_dependencies(vespa_jersey2) -install_java_artifact(searchlib) -install_java_artifact(vespajlib) - -install_fat_java_artifact(application-preprocessor) -install_fat_java_artifact(clustercontroller-apps) -install_fat_java_artifact(clustercontroller-apputil) -install_fat_java_artifact(clustercontroller-utils) -install_fat_java_artifact(clustercontroller-core) -install_fat_java_artifact(component) -install_fat_java_artifact(config-bundle) -install_fat_java_artifact(config-model-api) -install_fat_java_artifact(config-model) -install_fat_java_artifact(config-provisioning) -install_fat_java_artifact(config-proxy) -install_fat_java_artifact(configdefinitions) -install_fat_java_artifact(configserver) -install_fat_java_artifact(container-disc) -install_fat_java_artifact(container-jersey2) -install_fat_java_artifact(container-search-and-docproc) -install_fat_java_artifact(defaults) -install_fat_java_artifact(docprocs) -install_fat_java_artifact(jdisc_core) -install_fat_java_artifact(jdisc_http_service) -install_fat_java_artifact(logserver) -install_fat_java_artifact(node-repository) -install_fat_java_artifact(orchestrator) -install_fat_java_artifact(persistence) -install_fat_java_artifact(searchlib) -install_fat_java_artifact(simplemetrics) -install_fat_java_artifact(standalone-container) -install_fat_java_artifact(vespa-http-client) -install_fat_java_artifact(vespaclient-container-plugin) -install_fat_java_artifact(vespaclient-java) -install_fat_java_artifact(zkfacade) - -vespa_install_script(application-preprocessor/src/main/sh/vespa-preprocess-application bin) -vespa_install_script(config-proxy/src/main/sh/vespa-config-ctl.sh vespa-config-ctl bin) -vespa_install_script(config-proxy/src/main/sh/vespa-config-loadtester.sh vespa-config-loadtester bin) -vespa_install_script(config-proxy/src/main/sh/vespa-config-verification.sh vespa-config-verification bin) -vespa_install_script(config-model/src/main/perl/vespa-deploy bin) -vespa_install_script(config-model/src/main/perl/vespa-expand-config.pl bin) -vespa_install_script(config-model/src/main/perl/vespa-replicate-log-stream bin) -vespa_install_script(config-model/src/main/sh/vespa-validate-application bin) -vespa_install_script(container-disc/src/main/sh/vespa-start-container-daemon.sh vespa-start-container-daemon bin) -vespa_install_script(searchlib/src/main/sh/vespa-gbdt-converter bin) -vespa_install_script(searchlib/src/main/sh/vespa-treenet-converter bin) -vespa_install_script(vespaclient-java/src/main/sh/vespa-document-statistics.sh vespa-document-statistics bin) -vespa_install_script(vespaclient-java/src/main/sh/vespa-stat.sh vespa-stat bin) -vespa_install_script(vespaclient-java/src/main/sh/vespa-query-profile-dump-tool.sh vespa-query-profile-dump-tool bin) -vespa_install_script(vespaclient-java/src/main/sh/vespa-summary-benchmark.sh vespa-summary-benchmark bin) -vespa_install_script(vespaclient-java/src/main/sh/vespa-destination.sh vespa-destination bin) -vespa_install_script(vespaclient-java/src/main/sh/vespa-feeder.sh vespa-feeder bin) -vespa_install_script(vespaclient-java/src/main/sh/vespa-get.sh vespa-get bin) -vespa_install_script(vespaclient-java/src/main/sh/vespa-visit.sh vespa-visit bin) -vespa_install_script(vespaclient-java/src/main/sh/vespa-visit-target.sh vespa-visit-target bin) - -vespa_install_script(logserver/bin/logserver-start.sh vespa-logserver-start bin) - -install(DIRECTORY config-model/src/main/resources/schema DESTINATION share/vespa PATTERN ".gitignore" EXCLUDE) -install(DIRECTORY config-model/src/main/resources/schema DESTINATION share/vespa/schema/version/6.x PATTERN ".gitignore" EXCLUDE) - -install(FILES jdisc_core/src/main/perl/vespa-jdisc-logfmt.1 DESTINATION man/man1) - -install(FILES - config-model-fat/src/main/resources/config-models.xml - node-repository/src/main/config/node-repository.xml - DESTINATION conf/configserver-app) - -install(FILES - chain/src/main/resources/configdefinitions/chains.def - config-provisioning/src/main/resources/configdefinitions/flavors.def - container-accesslogging/src/main/resources/configdefinitions/access-log.def - container-core/src/main/resources/configdefinitions/application-metadata.def - container-core/src/main/resources/configdefinitions/container-document.def - container-core/src/main/resources/configdefinitions/container-http.def - container-core/src/main/resources/configdefinitions/diagnostics.def - container-core/src/main/resources/configdefinitions/health-monitor.def - container-core/src/main/resources/configdefinitions/http-filter.def - container-core/src/main/resources/configdefinitions/metrics-presentation.def - container-core/src/main/resources/configdefinitions/mockservice.def - container-core/src/main/resources/configdefinitions/qr-logging.def - container-core/src/main/resources/configdefinitions/qr-searchers.def - container-core/src/main/resources/configdefinitions/qr-templates.def - container-core/src/main/resources/configdefinitions/qr.def - container-core/src/main/resources/configdefinitions/servlet-config.def - container-core/src/main/resources/configdefinitions/threadpool.def - container-core/src/main/resources/configdefinitions/vip-status.def - container-di/src/main/resources/configdefinitions/bundles.def - container-di/src/main/resources/configdefinitions/components.def - container-di/src/main/resources/configdefinitions/jersey-bundles.def - container-di/src/main/resources/configdefinitions/jersey-injection.def - container-disc/src/main/resources/configdefinitions/container.jdisc.config.http-server.def - container-disc/src/main/resources/configdefinitions/jdisc-bindings.def - container-disc/src/main/resources/configdefinitions/jersey-connection.def - container-disc/src/main/resources/configdefinitions/jersey-init.def - container-disc/src/main/resources/configdefinitions/jersey-web-app-pool.def - container-disc/src/main/resources/configdefinitions/metric-defaults.def - container-disc/src/main/resources/configdefinitions/score-board.def - container-messagebus/src/main/resources/configdefinitions/container-mbus.def - container-messagebus/src/main/resources/configdefinitions/session.def - container-search-and-docproc/src/main/resources/configdefinitions/application-userdata.def - container-search/src/main/resources/configdefinitions/cluster.def - container-search/src/main/resources/configdefinitions/documentdb-info.def - container-search/src/main/resources/configdefinitions/emulation.def - container-search/src/main/resources/configdefinitions/federation.def - container-search/src/main/resources/configdefinitions/fs4.def - container-search/src/main/resources/configdefinitions/index-info.def - container-search/src/main/resources/configdefinitions/keyvalue.def - container-search/src/main/resources/configdefinitions/legacy-emulation.def - container-search/src/main/resources/configdefinitions/lowercasing.def - container-search/src/main/resources/configdefinitions/measure-qps.def - container-search/src/main/resources/configdefinitions/page-templates.def - container-search/src/main/resources/configdefinitions/provider.def - container-search/src/main/resources/configdefinitions/qr-binary-cache-region.def - container-search/src/main/resources/configdefinitions/qr-binary-cache.def - container-search/src/main/resources/configdefinitions/qr-monitor.def - container-search/src/main/resources/configdefinitions/qr-quotetable.def - container-search/src/main/resources/configdefinitions/qr-start.def - container-search/src/main/resources/configdefinitions/query-profiles.def - container-search/src/main/resources/configdefinitions/rate-limiting.def - container-search/src/main/resources/configdefinitions/resolvers.def - container-search/src/main/resources/configdefinitions/rewrites.def - container-search/src/main/resources/configdefinitions/search-nodes.def - container-search/src/main/resources/configdefinitions/search-with-renderer-handler.def - container-search/src/main/resources/configdefinitions/searchchain-forward.def - container-search/src/main/resources/configdefinitions/semantic-rules.def - container-search/src/main/resources/configdefinitions/strict-contracts.def - container-search/src/main/resources/configdefinitions/timing-searcher.def - docproc/src/main/resources/configdefinitions/docproc.def - docproc/src/main/resources/configdefinitions/schemamapping.def - docproc/src/main/resources/configdefinitions/splitter-joiner-document-processor.def - fileacquirer/src/main/resources/configdefinitions/filedistributorrpc.def - jdisc_http_service/src/main/resources/configdefinitions/jdisc.http.client.http-client.def - jdisc_http_service/src/main/resources/configdefinitions/jdisc.http.connector.def - jdisc_http_service/src/main/resources/configdefinitions/jdisc.http.server.def - jdisc_http_service/src/main/resources/configdefinitions/jdisc.http.servlet-paths.def - persistence/src/main/resources/configdefinitions/persistence-rpc.def - simplemetrics/src/main/resources/configdefinitions/manager.def - statistics/src/main/resources/configdefinitions/statistics.def - vespaclient-core/src/main/resources/configdefinitions/feeder.def - vespaclient-core/src/main/resources/configdefinitions/spooler.def - docker-api/src/main/resources/configdefinitions/docker.def - DESTINATION var/db/vespa/config_server/serverdb/classes) diff --git a/jdisc_core/CMakeLists.txt b/jdisc_core/CMakeLists.txt new file mode 100644 index 00000000000..b56d6ce1e5b --- /dev/null +++ b/jdisc_core/CMakeLists.txt @@ -0,0 +1,4 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +install_fat_java_artifact(jdisc_core) + +install(FILES src/main/perl/vespa-jdisc-logfmt.1 DESTINATION man/man1) diff --git a/jdisc_http_service/CMakeLists.txt b/jdisc_http_service/CMakeLists.txt new file mode 100644 index 00000000000..06af545bf34 --- /dev/null +++ b/jdisc_http_service/CMakeLists.txt @@ -0,0 +1,10 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +install_fat_java_artifact(jdisc_http_service) + +install(FILES + src/main/resources/configdefinitions/jdisc.http.client.http-client.def + src/main/resources/configdefinitions/jdisc.http.connector.def + src/main/resources/configdefinitions/jdisc.http.server.def + src/main/resources/configdefinitions/jdisc.http.servlet-paths.def + DESTINATION var/db/vespa/config_server/serverdb/classes) + diff --git a/jdisc_jetty/CMakeLists.txt b/jdisc_jetty/CMakeLists.txt new file mode 100644 index 00000000000..9059f68d953 --- /dev/null +++ b/jdisc_jetty/CMakeLists.txt @@ -0,0 +1,3 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +install_java_artifact(jdisc_jetty) +install_java_artifact_dependencies(jdisc_jetty) diff --git a/logd/src/logd/CMakeLists.txt b/logd/src/logd/CMakeLists.txt index 03ade1d66c8..be3fe2d1794 100644 --- a/logd/src/logd/CMakeLists.txt +++ b/logd/src/logd/CMakeLists.txt @@ -12,4 +12,4 @@ vespa_add_library(logd STATIC DEPENDS ) vespa_generate_config(logd ../main/resources/configdefinitions/logd.def) -install(FILES ../main/resources/configdefinitions/logd.def RENAME cloud.config.log.logd.def DESTINATION var/db/vespa/config_server/serverdb/classes) +install_config_definition(../main/resources/configdefinitions/logd.def cloud.config.log.logd.def) diff --git a/logserver/CMakeLists.txt b/logserver/CMakeLists.txt new file mode 100644 index 00000000000..736878948fb --- /dev/null +++ b/logserver/CMakeLists.txt @@ -0,0 +1,4 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +install_fat_java_artifact(logserver) + +vespa_install_script(bin/logserver-start.sh vespa-logserver-start bin) diff --git a/messagebus/src/main/java/com/yahoo/messagebus/network/rpc/RPCNetwork.java b/messagebus/src/main/java/com/yahoo/messagebus/network/rpc/RPCNetwork.java index 96a0a5fecef..bb0b7bdd878 100644 --- a/messagebus/src/main/java/com/yahoo/messagebus/network/rpc/RPCNetwork.java +++ b/messagebus/src/main/java/com/yahoo/messagebus/network/rpc/RPCNetwork.java @@ -67,7 +67,7 @@ public class RPCNetwork implements Network, MethodHandler { private NetworkOwner owner; private final SlobrokConfigSubscriber slobroksConfig; private final LinkedHashMap<String, Route> lruRouteMap = new LinkedHashMap<>(10000, 0.5f, true); - private final ExecutorService sendService = + private final ExecutorService executor = new ThreadPoolExecutor(Runtime.getRuntime().availableProcessors(), Runtime.getRuntime().availableProcessors(), 0L, TimeUnit.SECONDS, new SynchronousQueue<>(false), @@ -258,7 +258,7 @@ public class RPCNetwork implements Network, MethodHandler { } else if (ctx.hasError) { replyError(ctx, ErrorCode.HANDSHAKE_FAILED, "An error occured while resolving version."); } else { - sendService.execute(new SendTask(owner.getProtocol(ctx.msg.getProtocol()), ctx)); + executor.execute(new SendTask(owner.getProtocol(ctx.msg.getProtocol()), ctx)); } } @@ -278,7 +278,7 @@ public class RPCNetwork implements Network, MethodHandler { listener.shutdown().join(); orb.transport().shutdown().join(); targetPool.flushTargets(true); - sendService.shutdown(); + executor.shutdown(); return true; } return false; @@ -411,6 +411,10 @@ public class RPCNetwork implements Network, MethodHandler { return oosManager; } + ExecutorService getExecutor() { + return executor; + } + private class SendTask implements Runnable { final Protocol protocol; diff --git a/messagebus/src/main/java/com/yahoo/messagebus/network/rpc/RPCSend.java b/messagebus/src/main/java/com/yahoo/messagebus/network/rpc/RPCSend.java index d7b4887bd36..daa31ae2701 100644 --- a/messagebus/src/main/java/com/yahoo/messagebus/network/rpc/RPCSend.java +++ b/messagebus/src/main/java/com/yahoo/messagebus/network/rpc/RPCSend.java @@ -107,6 +107,10 @@ public abstract class RPCSend implements MethodHandler, ReplyHandler, RequestWai @Override public final void handleRequestDone(Request req) { + net.getExecutor().execute(() -> doRequestDone(req)); + } + + private void doRequestDone(Request req) { SendContext ctx = (SendContext)req.getContext(); String serviceName = ((RPCServiceAddress)ctx.recipient.getServiceAddress()).getServiceName(); Reply reply = null; @@ -157,6 +161,10 @@ public abstract class RPCSend implements MethodHandler, ReplyHandler, RequestWai @Override public final void invoke(Request request) { request.detach(); + net.getExecutor().execute(() -> doInvoke(request)); + } + + private void doInvoke(Request request) { Params p = toParams(request.parameters()); request.discardParameters(); // allow garbage collection of request parameters diff --git a/messagebus/src/tests/throttling/throttling.cpp b/messagebus/src/tests/throttling/throttling.cpp index 279f31201c4..5d3525e8ba6 100644 --- a/messagebus/src/tests/throttling/throttling.cpp +++ b/messagebus/src/tests/throttling/throttling.cpp @@ -197,6 +197,7 @@ Test::testMaxPendingSize() EXPECT_EQUAL(2u, SimpleMessage("12").getApproxSize()); EXPECT_TRUE(ss->send(Message::UP(new SimpleMessage("1")), "dst").isAccepted()); + EXPECT_TRUE(waitQueueSize(dstQ, 1)); EXPECT_TRUE(ss->send(Message::UP(new SimpleMessage("12")), "dst").isAccepted()); EXPECT_TRUE(!ss->send(Message::UP(new SimpleMessage("1")), "dst").isAccepted()); diff --git a/messagebus/src/vespa/messagebus/CMakeLists.txt b/messagebus/src/vespa/messagebus/CMakeLists.txt index 7c922f36a8c..9ff3bae67c8 100644 --- a/messagebus/src/vespa/messagebus/CMakeLists.txt +++ b/messagebus/src/vespa/messagebus/CMakeLists.txt @@ -37,4 +37,4 @@ vespa_add_library(messagebus DEPENDS ) vespa_generate_config(messagebus ../../main/config/messagebus.def) -install(FILES ../../main/config/messagebus.def RENAME messagebus.messagebus.def DESTINATION var/db/vespa/config_server/serverdb/classes) +install_config_definition(../../main/config/messagebus.def messagebus.messagebus.def) diff --git a/messagebus/src/vespa/messagebus/network/rpcnetwork.cpp b/messagebus/src/vespa/messagebus/network/rpcnetwork.cpp index e5c292fedd9..8ff7ac87edc 100644 --- a/messagebus/src/vespa/messagebus/network/rpcnetwork.cpp +++ b/messagebus/src/vespa/messagebus/network/rpcnetwork.cpp @@ -15,6 +15,7 @@ #include <vespa/slobrok/sbmirror.h> #include <vespa/vespalib/component/vtag.h> #include <vespa/vespalib/util/stringfmt.h> +#include <vespa/vespalib/util/lambdatask.h> #include <vespa/fnet/scheduler.h> #include <vespa/fnet/transport.h> #include <vespa/fnet/frt/supervisor.h> @@ -119,6 +120,7 @@ RPCNetwork::RPCNetwork(const RPCNetworkParams ¶ms) : _regAPI(std::make_unique<slobrok::api::RegisterAPI>(*_orb, *_slobrokCfgFactory)), _oosManager(std::make_unique<OOSManager>(*_orb, *_mirror, params.getOOSServerPattern())), _requestedPort(params.getListenPort()), + _executor(std::make_unique<vespalib::ThreadStackExecutor>(4,65536)), _sendV1(std::make_unique<RPCSendV1>()), _sendV2(std::make_unique<RPCSendV2>()), _sendAdapters(), @@ -222,7 +224,10 @@ RPCNetwork::start() return true; } - +vespalib::Executor & +RPCNetwork::getExecutor() { + return *_executor; +} bool RPCNetwork::waitUntilReady(double seconds) const @@ -371,6 +376,7 @@ void RPCNetwork::sync() { SyncTask task(_scheduler); + _executor->sync(); task.await(); } @@ -379,6 +385,8 @@ RPCNetwork::shutdown() { _transport->ShutDown(false); _threadPool->Close(); + _executor->shutdown(); + _executor->sync(); } void diff --git a/messagebus/src/vespa/messagebus/network/rpcnetwork.h b/messagebus/src/vespa/messagebus/network/rpcnetwork.h index 5e762f1a2a9..13fab018c3b 100644 --- a/messagebus/src/vespa/messagebus/network/rpcnetwork.h +++ b/messagebus/src/vespa/messagebus/network/rpcnetwork.h @@ -60,24 +60,25 @@ private: using SendAdapterMap = std::map<vespalib::Version, RPCSendAdapter*>; - INetworkOwner *_owner; - Identity _ident; - std::unique_ptr<FastOS_ThreadPool> _threadPool; - std::unique_ptr<FNET_Transport> _transport; - std::unique_ptr<FRT_Supervisor> _orb; - FNET_Scheduler &_scheduler; - std::unique_ptr<RPCTargetPool> _targetPool; - TargetPoolTask _targetPoolTask; - std::unique_ptr<RPCServicePool> _servicePool; - std::unique_ptr<slobrok::ConfiguratorFactory> _slobrokCfgFactory; - std::unique_ptr<slobrok::api::IMirrorAPI> _mirror; - std::unique_ptr<slobrok::api::RegisterAPI> _regAPI; - std::unique_ptr<OOSManager> _oosManager; - int _requestedPort; - std::unique_ptr<RPCSendAdapter> _sendV1; - std::unique_ptr<RPCSendAdapter> _sendV2; - SendAdapterMap _sendAdapters; - CompressionConfig _compressionConfig; + INetworkOwner *_owner; + Identity _ident; + std::unique_ptr<FastOS_ThreadPool> _threadPool; + std::unique_ptr<FNET_Transport> _transport; + std::unique_ptr<FRT_Supervisor> _orb; + FNET_Scheduler &_scheduler; + std::unique_ptr<RPCTargetPool> _targetPool; + TargetPoolTask _targetPoolTask; + std::unique_ptr<RPCServicePool> _servicePool; + std::unique_ptr<slobrok::ConfiguratorFactory> _slobrokCfgFactory; + std::unique_ptr<slobrok::api::IMirrorAPI> _mirror; + std::unique_ptr<slobrok::api::RegisterAPI> _regAPI; + std::unique_ptr<OOSManager> _oosManager; + int _requestedPort; + std::unique_ptr<vespalib::ThreadStackExecutor> _executor; + std::unique_ptr<RPCSendAdapter> _sendV1; + std::unique_ptr<RPCSendAdapter> _sendV2; + SendAdapterMap _sendAdapters; + CompressionConfig _compressionConfig; /** * Resolves and assigns a service address for the given recipient using the @@ -235,6 +236,7 @@ public: const slobrok::api::IMirrorAPI &getMirror() const override; CompressionConfig getCompressionConfig() { return _compressionConfig; } void invoke(FRT_RPCRequest *req); + vespalib::Executor & getExecutor(); }; } // namespace mbus diff --git a/messagebus/src/vespa/messagebus/network/rpcsend.cpp b/messagebus/src/vespa/messagebus/network/rpcsend.cpp index 705b8648442..cc6b7086756 100644 --- a/messagebus/src/vespa/messagebus/network/rpcsend.cpp +++ b/messagebus/src/vespa/messagebus/network/rpcsend.cpp @@ -7,13 +7,15 @@ #include <vespa/messagebus/tracelevel.h> #include <vespa/messagebus/emptyreply.h> #include <vespa/messagebus/errorcode.h> -#include <vespa/vespalib/util/stringfmt.h> #include <vespa/fnet/channel.h> #include <vespa/fnet/frt/reflection.h> +#include <vespa/vespalib/util/stringfmt.h> +#include <vespa/vespalib/util/lambdatask.h> #include <vespa/vespalib/data/slime/cursor.h> using vespalib::make_string; +using vespalib::makeLambdaTask; namespace mbus { @@ -146,6 +148,11 @@ RPCSend::send(RoutingNode &recipient, const vespalib::Version &version, void RPCSend::RequestDone(FRT_RPCRequest *req) { + doRequestDone(req); +} + +void +RPCSend::doRequestDone(FRT_RPCRequest *req) { SendContext::UP ctx(static_cast<SendContext*>(req->GetContext()._value.VOIDP)); const string &serviceName = static_cast<RPCServiceAddress&>(ctx->getRecipient().getServiceAddress()).getServiceName(); Reply::UP reply; @@ -213,6 +220,11 @@ RPCSend::decode(vespalib::stringref protocolName, const vespalib::Version & vers void RPCSend::handleReply(Reply::UP reply) { + doHandleReply(std::move(reply)); +} + +void +RPCSend::doHandleReply(Reply::UP reply) { ReplyContext::UP ctx(static_cast<ReplyContext*>(reply->getContext().value.PTR)); FRT_RPCRequest &req = ctx->getRequest(); string version = ctx->getVersion().toString(); @@ -236,6 +248,12 @@ void RPCSend::invoke(FRT_RPCRequest *req) { req->Detach(); + doRequest(req); +} + +void +RPCSend::doRequest(FRT_RPCRequest *req) +{ FRT_Values &args = *req->GetParams(); std::unique_ptr<Params> params = toParams(args); diff --git a/messagebus/src/vespa/messagebus/network/rpcsend.h b/messagebus/src/vespa/messagebus/network/rpcsend.h index c707b47f548..ec455aea7bd 100644 --- a/messagebus/src/vespa/messagebus/network/rpcsend.h +++ b/messagebus/src/vespa/messagebus/network/rpcsend.h @@ -82,6 +82,9 @@ public: void invoke(FRT_RPCRequest *req); private: + void doRequest(FRT_RPCRequest *req); + void doRequestDone(FRT_RPCRequest *req); + void doHandleReply(std::unique_ptr<Reply> reply); void attach(RPCNetwork &net) final override; void handleDiscard(Context ctx) final override; void sendByHandover(RoutingNode &recipient, const vespalib::Version &version, diff --git a/metrics/src/vespa/metrics/CMakeLists.txt b/metrics/src/vespa/metrics/CMakeLists.txt index 27a1f25e887..6eae8cd75e4 100644 --- a/metrics/src/vespa/metrics/CMakeLists.txt +++ b/metrics/src/vespa/metrics/CMakeLists.txt @@ -24,4 +24,4 @@ vespa_add_library(metrics DEPENDS ) vespa_generate_config(metrics metricsmanager.def) -install(FILES metricsmanager.def RENAME metrics.metricsmanager.def DESTINATION var/db/vespa/config_server/serverdb/classes) +install_config_definition(metricsmanager.def metrics.metricsmanager.def) diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java index cac924cbf18..a3a647e1d14 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java @@ -399,7 +399,7 @@ public class StorageMaintainer { } void updateNextHandleCoredumpsTime() { - nextHandleOldCoredumpsAt = clock.instant().plus(Duration.ofHours(1)); + nextHandleOldCoredumpsAt = clock.instant().plus(Duration.ofMinutes(5)); } boolean shouldHandleCoredumpsNow() { diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/maintainer/CoredumpHandler.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/maintainer/CoredumpHandler.java index 21584aee18d..99dfdb48334 100644 --- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/maintainer/CoredumpHandler.java +++ b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/maintainer/CoredumpHandler.java @@ -15,6 +15,7 @@ import java.io.InputStreamReader; import java.nio.file.Files; import java.nio.file.Path; import java.time.Duration; +import java.util.Comparator; import java.util.HashMap; import java.util.Map; import java.util.Optional; @@ -28,10 +29,10 @@ import java.util.stream.Collectors; * * @author freva */ -public class CoredumpHandler { +class CoredumpHandler { - public static final String PROCESSING_DIRECTORY_NAME = "processing"; - public static final String METADATA_FILE_NAME = "metadata.json"; + static final String PROCESSING_DIRECTORY_NAME = "processing"; + static final String METADATA_FILE_NAME = "metadata.json"; private final Logger logger = Logger.getLogger(CoredumpHandler.class.getName()); private final ObjectMapper objectMapper = new ObjectMapper(); @@ -57,7 +58,7 @@ public class CoredumpHandler { public void processAll() throws IOException { removeJavaCoredumps(); - processAndReportCoredumps(); + handleNewCoredumps(); removeOldCoredumps(); } @@ -71,21 +72,27 @@ public class CoredumpHandler { FileHelper.deleteDirectories(doneCoredumpsPath, Duration.ofDays(10), Optional.empty()); } - private void processAndReportCoredumps() throws IOException { - Path processingCoredumps = processCoredumps(); - reportCoredumps(processingCoredumps); + private void handleNewCoredumps() throws IOException { + Path processingCoredumps = enqueueCoredumps(); + processAndReportCoredumps(processingCoredumps); } - Path processCoredumps() throws IOException { + /** + * Moves a coredump to a new directory under the processing/ directory. Limit to only processing + * one coredump at the time, starting with the oldest. + */ + Path enqueueCoredumps() throws IOException { Path processingCoredumpsPath = coredumpsPath.resolve(PROCESSING_DIRECTORY_NAME); processingCoredumpsPath.toFile().mkdirs(); + if (Files.list(processingCoredumpsPath).count() > 0) return processingCoredumpsPath; Files.list(coredumpsPath) .filter(path -> path.toFile().isFile() && ! path.getFileName().toString().startsWith(".")) - .forEach(coredumpPath -> { + .min((Comparator.comparingLong(o -> o.toFile().lastModified()))) + .ifPresent(coredumpPath -> { try { - startProcessing(coredumpPath, processingCoredumpsPath); + enqueueCoredumpForProcessing(coredumpPath, processingCoredumpsPath); } catch (Throwable e) { logger.log(Level.WARNING, "Failed to process coredump " + coredumpPath, e); } @@ -94,7 +101,7 @@ public class CoredumpHandler { return processingCoredumpsPath; } - void reportCoredumps(Path processingCoredumpsPath) throws IOException { + void processAndReportCoredumps(Path processingCoredumpsPath) throws IOException { doneCoredumpsPath.toFile().mkdirs(); Files.list(processingCoredumpsPath) @@ -110,7 +117,7 @@ public class CoredumpHandler { }); } - Path startProcessing(Path coredumpPath, Path processingCoredumpsPath) throws IOException { + Path enqueueCoredumpForProcessing(Path coredumpPath, Path processingCoredumpsPath) throws IOException { // Make coredump readable coredumpPath.toFile().setReadable(true, false); diff --git a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/maintainer/CoredumpHandlerTest.java b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/maintainer/CoredumpHandlerTest.java index d469e5efc46..2cc96567e20 100644 --- a/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/maintainer/CoredumpHandlerTest.java +++ b/node-maintainer/src/test/java/com/yahoo/vespa/hosted/node/maintainer/CoredumpHandlerTest.java @@ -22,7 +22,9 @@ import java.net.URI; import java.net.URISyntaxException; import java.nio.file.Files; import java.nio.file.Path; +import java.time.Instant; import java.util.Arrays; +import java.util.Collections; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -84,8 +86,8 @@ public class CoredumpHandlerTest { @Test public void ignoresIncompleteCoredumps() throws IOException { - Path coredumpPath = createCoredump(".core.dump"); - Path processingPath = coredumpHandler.processCoredumps(); + Path coredumpPath = createCoredump(".core.dump", Instant.now()); + Path processingPath = coredumpHandler.enqueueCoredumps(); // The 'processing' directory should be empty assertFolderContents(processingPath); @@ -96,9 +98,9 @@ public class CoredumpHandlerTest { @Test public void startProcessingTest() throws IOException { - Path coredumpPath = createCoredump("core.dump"); + Path coredumpPath = createCoredump("core.dump", Instant.now()); Path processingPath = crashPath.resolve("processing_dir"); - coredumpHandler.startProcessing(coredumpPath, processingPath); + coredumpHandler.enqueueCoredumpForProcessing(coredumpPath, processingPath); // Contents of 'crash' should be only the 'processing' directory assertFolderContents(crashPath, processingPath.getFileName().toString()); @@ -112,9 +114,36 @@ public class CoredumpHandlerTest { } @Test + public void limitToProcessingOneCoredumpAtTheTimeTest() throws IOException { + final String oldestCoredump = "core.dump0"; + final Instant startTime = Instant.now(); + createCoredump(oldestCoredump, startTime.minusSeconds(3600)); + createCoredump("core.dump1", startTime.minusSeconds(1000)); + createCoredump("core.dump2", startTime); + Path processingPath = coredumpHandler.enqueueCoredumps(); + + List<Path> processingCoredumps = Files.list(processingPath).collect(Collectors.toList()); + assertEquals(1, processingCoredumps.size()); + + // Make sure that the 1 coredump that we are processing is the oldest one + Set<String> filenamesInProcessingDirectory = Files.list(processingCoredumps.get(0)) + .map(file -> file.getFileName().toString()) + .collect(Collectors.toSet()); + assertEquals(Collections.singleton(oldestCoredump), filenamesInProcessingDirectory); + + // Running enqueueCoredumps should not start processing any new coredumps as we already are processing one + coredumpHandler.enqueueCoredumps(); + assertEquals(processingCoredumps, Files.list(processingPath).collect(Collectors.toList())); + filenamesInProcessingDirectory = Files.list(processingCoredumps.get(0)) + .map(file -> file.getFileName().toString()) + .collect(Collectors.toSet()); + assertEquals(Collections.singleton(oldestCoredump), filenamesInProcessingDirectory); + } + + @Test public void coredumpMetadataCollectAndWriteTest() throws IOException, InterruptedException { - createCoredump("core.dump"); - Path processingPath = coredumpHandler.processCoredumps(); + createCoredump("core.dump", Instant.now()); + Path processingPath = coredumpHandler.enqueueCoredumps(); Path processingCoredumpPath = Files.list(processingPath).findFirst().orElseThrow(() -> new RuntimeException("Expected to find directory with coredump in processing dir")); when(coreCollector.collect(eq(processingCoredumpPath.resolve("core.dump")), any())).thenReturn(metadata); @@ -153,7 +182,7 @@ public class CoredumpHandlerTest { Path metadataPath = createProcessedCoredump(documentId); setNextHttpResponse(500, Optional.of("Internal server error")); - coredumpHandler.reportCoredumps(crashPath.resolve(CoredumpHandler.PROCESSING_DIRECTORY_NAME)); + coredumpHandler.processAndReportCoredumps(crashPath.resolve(CoredumpHandler.PROCESSING_DIRECTORY_NAME)); validateNextHttpPost(documentId, expectedMetadataFileContents); // The coredump should not have been moved out of 'processing' and into 'done' as the report failed @@ -182,9 +211,10 @@ public class CoredumpHandlerTest { assertEquals(expectedContentsOfFolder, actualContentsOfFolder); } - private Path createCoredump(String coredumpName) throws IOException { + private Path createCoredump(String coredumpName, Instant lastModified) throws IOException { Path coredumpPath = crashPath.resolve(coredumpName); coredumpPath.toFile().createNewFile(); + coredumpPath.toFile().setLastModified(lastModified.toEpochMilli()); return coredumpPath; } diff --git a/node-repository/CMakeLists.txt b/node-repository/CMakeLists.txt new file mode 100644 index 00000000000..24a8f7e4177 --- /dev/null +++ b/node-repository/CMakeLists.txt @@ -0,0 +1,5 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +install_fat_java_artifact(node-repository) + +install(FILES src/main/config/node-repository.xml + DESTINATION conf/configserver-app) diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirer.java index a86ba955a9a..b7971e61117 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirer.java @@ -4,7 +4,6 @@ package com.yahoo.vespa.hosted.provision.maintenance; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.Flavor; import com.yahoo.config.provision.NodeType; -import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.Zone; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeRepository; @@ -80,7 +79,6 @@ public class FailedExpirer extends Expirer { private boolean failCountIndicatesHwFail(Zone zone, Node node) { if (node.flavor().getType() == Flavor.Type.DOCKER_CONTAINER) return false; - if (zone.system() == SystemName.cd) return false; return zone.environment() == Environment.prod || zone.environment() == Environment.staging; } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirerTest.java index 3ccacb3ff02..51991a844d7 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirerTest.java @@ -75,8 +75,8 @@ public class FailedExpirerTest { clock.advance(Duration.ofDays(5)); failedExpirer.run(); + assertNodeHostnames(Node.State.failed, "node1"); assertNodeHostnames(Node.State.parked, "node2", "node3"); - assertNodeHostnames(Node.State.dirty, "node1"); } @Test diff --git a/orchestrator/CMakeLists.txt b/orchestrator/CMakeLists.txt new file mode 100644 index 00000000000..5ff6a7ac576 --- /dev/null +++ b/orchestrator/CMakeLists.txt @@ -0,0 +1,2 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +install_fat_java_artifact(orchestrator) diff --git a/persistence/CMakeLists.txt b/persistence/CMakeLists.txt index b773fccc09a..b4cb36e8bcf 100644 --- a/persistence/CMakeLists.txt +++ b/persistence/CMakeLists.txt @@ -29,3 +29,6 @@ vespa_define_module( src/tests/proxy src/tests/spi ) + +install_fat_java_artifact(persistence) +install_config_definition(src/main/resources/configdefinitions/persistence-rpc.def persistence.persistence-rpc.def) diff --git a/searchcore/src/tests/proton/documentdb/document_subdbs/document_subdbs_test.cpp b/searchcore/src/tests/proton/documentdb/document_subdbs/document_subdbs_test.cpp index 9eddef36436..b14eb9051d0 100644 --- a/searchcore/src/tests/proton/documentdb/document_subdbs/document_subdbs_test.cpp +++ b/searchcore/src/tests/proton/documentdb/document_subdbs/document_subdbs_test.cpp @@ -23,7 +23,7 @@ #include <vespa/searchcore/proton/test/thread_utils.h> #include <vespa/searchcorespi/plugin/iindexmanagerfactory.h> #include <vespa/searchlib/common/idestructorcallback.h> -#include <vespa/searchlib/common/lambdatask.h> +#include <vespa/vespalib/util/lambdatask.h> #include <vespa/searchlib/index/docbuilder.h> #include <vespa/searchlib/test/directory_handler.h> #include <vespa/vespalib/io/fileutil.h> diff --git a/searchcore/src/tests/proton/documentdb/feedhandler/feedhandler_test.cpp b/searchcore/src/tests/proton/documentdb/feedhandler/feedhandler_test.cpp index bb43aa6c63d..1ccbff6e6cf 100644 --- a/searchcore/src/tests/proton/documentdb/feedhandler/feedhandler_test.cpp +++ b/searchcore/src/tests/proton/documentdb/feedhandler/feedhandler_test.cpp @@ -50,7 +50,7 @@ using search::IDestructorCallback; using search::SerialNum; using search::index::schema::CollectionType; using search::index::schema::DataType; -using search::makeLambdaTask; +using vespalib::makeLambdaTask; using search::transactionlog::TransLogServer; using storage::spi::PartitionId; using storage::spi::RemoveResult; diff --git a/searchcore/src/tests/proton/documentmetastore/lidreusedelayer/lidreusedelayer_test.cpp b/searchcore/src/tests/proton/documentmetastore/lidreusedelayer/lidreusedelayer_test.cpp index 5109d994f61..d56340be2b2 100644 --- a/searchcore/src/tests/proton/documentmetastore/lidreusedelayer/lidreusedelayer_test.cpp +++ b/searchcore/src/tests/proton/documentmetastore/lidreusedelayer/lidreusedelayer_test.cpp @@ -7,9 +7,9 @@ LOG_SETUP("lidreusedelayer_test"); #include <vespa/searchcore/proton/server/executorthreadingservice.h> #include <vespa/searchcore/proton/test/thread_utils.h> #include <vespa/searchcore/proton/test/threading_service_observer.h> -#include <vespa/searchlib/common/lambdatask.h> +#include <vespa/vespalib/util/lambdatask.h> -using search::makeLambdaTask; +using vespalib::makeLambdaTask; namespace proton { diff --git a/searchcore/src/tests/proton/index/indexmanager_test.cpp b/searchcore/src/tests/proton/index/indexmanager_test.cpp index 84198fb9b42..e8afd738e84 100644 --- a/searchcore/src/tests/proton/index/indexmanager_test.cpp +++ b/searchcore/src/tests/proton/index/indexmanager_test.cpp @@ -43,7 +43,7 @@ using search::index::DocBuilder; using search::index::DummyFileHeaderContext; using search::index::Schema; using search::index::schema::DataType; -using search::makeLambdaTask; +using vespalib::makeLambdaTask; using search::memoryindex::CompactDocumentWordsStore; using search::memoryindex::Dictionary; using search::queryeval::Source; diff --git a/searchcore/src/tests/proton/reference/gid_to_lid_change_handler/gid_to_lid_change_handler_test.cpp b/searchcore/src/tests/proton/reference/gid_to_lid_change_handler/gid_to_lid_change_handler_test.cpp index 625e9c1f6a9..1201bc4720b 100644 --- a/searchcore/src/tests/proton/reference/gid_to_lid_change_handler/gid_to_lid_change_handler_test.cpp +++ b/searchcore/src/tests/proton/reference/gid_to_lid_change_handler/gid_to_lid_change_handler_test.cpp @@ -4,7 +4,7 @@ #include <vespa/document/base/documentid.h> #include <vespa/vespalib/util/threadstackexecutor.h> #include <vespa/searchcore/proton/server/executor_thread_service.h> -#include <vespa/searchlib/common/lambdatask.h> +#include <vespa/vespalib/util/lambdatask.h> #include <vespa/searchcore/proton/reference/i_gid_to_lid_change_listener.h> #include <vespa/searchcore/proton/reference/gid_to_lid_change_handler.h> #include <map> @@ -13,7 +13,7 @@ LOG_SETUP("gid_to_lid_change_handler_test"); using document::GlobalId; using document::DocumentId; -using search::makeLambdaTask; +using vespalib::makeLambdaTask; using search::SerialNum; namespace proton { diff --git a/searchcore/src/tests/proton/server/visibility_handler/visibility_handler_test.cpp b/searchcore/src/tests/proton/server/visibility_handler/visibility_handler_test.cpp index 69b8a482476..5b5bea412f7 100644 --- a/searchcore/src/tests/proton/server/visibility_handler/visibility_handler_test.cpp +++ b/searchcore/src/tests/proton/server/visibility_handler/visibility_handler_test.cpp @@ -6,7 +6,7 @@ LOG_SETUP("visibility_handler_test"); #include <vespa/searchcore/proton/test/dummy_feed_view.h> #include <vespa/searchcore/proton/test/threading_service_observer.h> #include <vespa/searchcore/proton/server/executorthreadingservice.h> -#include <vespa/searchlib/common/lambdatask.h> +#include <vespa/vespalib/util/lambdatask.h> using search::SerialNum; using proton::IGetSerialNum; @@ -15,7 +15,7 @@ using proton::ExecutorThreadingService; using proton::test::ThreadingServiceObserver; using proton::IFeedView; using proton::VisibilityHandler; -using search::makeLambdaTask; +using vespalib::makeLambdaTask; using fastos::TimeStamp; namespace { diff --git a/searchcore/src/vespa/searchcore/config/CMakeLists.txt b/searchcore/src/vespa/searchcore/config/CMakeLists.txt index c7669efc876..3d62309161c 100644 --- a/searchcore/src/vespa/searchcore/config/CMakeLists.txt +++ b/searchcore/src/vespa/searchcore/config/CMakeLists.txt @@ -4,11 +4,11 @@ vespa_add_library(searchcore_fconfig STATIC DEPENDS ) vespa_generate_config(searchcore_fconfig partitions.def) -install(FILES partitions.def RENAME vespa.config.search.core.partitions.def DESTINATION var/db/vespa/config_server/serverdb/classes) +install_config_definition(partitions.def vespa.config.search.core.partitions.def) vespa_generate_config(searchcore_fconfig fdispatchrc.def) -install(FILES fdispatchrc.def RENAME vespa.config.search.core.fdispatchrc.def DESTINATION var/db/vespa/config_server/serverdb/classes) +install_config_definition(fdispatchrc.def vespa.config.search.core.fdispatchrc.def) vespa_generate_config(searchcore_fconfig proton.def) -install(FILES proton.def RENAME vespa.config.search.core.proton.def DESTINATION var/db/vespa/config_server/serverdb/classes) +install_config_definition(proton.def vespa.config.search.core.proton.def) vespa_generate_config(searchcore_fconfig ranking-constants.def) -install(FILES ranking-constants.def RENAME vespa.config.search.core.ranking-constants.def DESTINATION var/db/vespa/config_server/serverdb/classes) +install_config_definition(ranking-constants.def vespa.config.search.core.ranking-constants.def) vespa_generate_config(searchcore_fconfig hwinfo.def) diff --git a/searchcore/src/vespa/searchcore/config/proton.def b/searchcore/src/vespa/searchcore/config/proton.def index 5311e9bde07..94b7f2e9ef0 100644 --- a/searchcore/src/vespa/searchcore/config/proton.def +++ b/searchcore/src/vespa/searchcore/config/proton.def @@ -402,6 +402,11 @@ writefilter.sampleinterval double default = 60.0 ## The disk size is used when determining if feed should be blocked in writefilter. hwinfo.disk.size long default = 0 +## Whether the disk partition is shared among several instances of proton (e.g. when using docker). +## If shared, disk usage is sampled by doing a recursive directory scan in proton basedir. +## If not, disk usage is sampled by looking at the filesystem space info. +hwinfo.disk.shared bool default = false + ## Override for disk write speed, measured in MiB/s. When zero, the ## actual disk write speed is sampled by writing data to a temporary file. hwinfo.disk.writespeed double default = 200.0 diff --git a/searchcore/src/vespa/searchcore/proton/docsummary/summarycompacttarget.cpp b/searchcore/src/vespa/searchcore/proton/docsummary/summarycompacttarget.cpp index 0549e57a528..6cc1c1eb5b9 100644 --- a/searchcore/src/vespa/searchcore/proton/docsummary/summarycompacttarget.cpp +++ b/searchcore/src/vespa/searchcore/proton/docsummary/summarycompacttarget.cpp @@ -1,13 +1,13 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "summarycompacttarget.h" -#include <vespa/searchlib/common/lambdatask.h> +#include <vespa/vespalib/util/lambdatask.h> #include <vespa/searchcorespi/index/i_thread_service.h> #include <future> using search::IDocumentStore; using search::SerialNum; -using search::makeLambdaTask; +using vespalib::makeLambdaTask; using searchcorespi::FlushStats; using searchcorespi::IFlushTarget; diff --git a/searchcore/src/vespa/searchcore/proton/docsummary/summaryflushtarget.cpp b/searchcore/src/vespa/searchcore/proton/docsummary/summaryflushtarget.cpp index de8eaacf184..2f7681c5909 100644 --- a/searchcore/src/vespa/searchcore/proton/docsummary/summaryflushtarget.cpp +++ b/searchcore/src/vespa/searchcore/proton/docsummary/summaryflushtarget.cpp @@ -1,9 +1,8 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "summaryflushtarget.h" -#include <future> #include <vespa/searchcorespi/index/i_thread_service.h> -#include <vespa/searchlib/common/lambdatask.h> +#include <vespa/vespalib/util/lambdatask.h> using search::IDocumentStore; using search::SerialNum; @@ -91,7 +90,7 @@ SummaryFlushTarget::initFlush(SerialNum currentSerial) // Called by document db executor std::promise<Task::UP> promise; std::future<Task::UP> future = promise.get_future(); - _summaryService.execute(search::makeLambdaTask( + _summaryService.execute(vespalib::makeLambdaTask( [&]() { promise.set_value( internalInitFlush(currentSerial)); })); diff --git a/searchcore/src/vespa/searchcore/proton/docsummary/summarymanager.cpp b/searchcore/src/vespa/searchcore/proton/docsummary/summarymanager.cpp index 79d82108ee8..91b043d0c9f 100644 --- a/searchcore/src/vespa/searchcore/proton/docsummary/summarymanager.cpp +++ b/searchcore/src/vespa/searchcore/proton/docsummary/summarymanager.cpp @@ -9,7 +9,7 @@ #include <vespa/searchcorespi/index/i_thread_service.h> #include <vespa/searchcore/proton/flushengine/shrink_lid_space_flush_target.h> #include <vespa/searchlib/docstore/logdocumentstore.h> -#include <vespa/searchlib/common/lambdatask.h> +#include <vespa/vespalib/util/lambdatask.h> #include <vespa/searchsummary/docsummary/docsumconfig.h> #include <vespa/vespalib/util/exceptions.h> #include <sstream> @@ -32,7 +32,7 @@ using search::IDocumentStore; using search::LogDocumentStore; using search::LogDataStore; using search::WriteableFileChunk; -using search::makeLambdaTask; +using vespalib::makeLambdaTask; using search::TuneFileSummary; using search::common::FileHeaderContext; diff --git a/searchcore/src/vespa/searchcore/proton/flushengine/threadedflushtarget.cpp b/searchcore/src/vespa/searchcore/proton/flushengine/threadedflushtarget.cpp index c7a614dafe8..3c1ac1b5361 100644 --- a/searchcore/src/vespa/searchcore/proton/flushengine/threadedflushtarget.cpp +++ b/searchcore/src/vespa/searchcore/proton/flushengine/threadedflushtarget.cpp @@ -2,13 +2,13 @@ #include "threadedflushtarget.h" #include <vespa/searchcore/proton/server/igetserialnum.h> -#include <vespa/searchlib/common/lambdatask.h> +#include <vespa/vespalib/util/lambdatask.h> #include <future> #include <cassert> using searchcorespi::IFlushTarget; using searchcorespi::FlushStats; -using search::makeLambdaTask; +using vespalib::makeLambdaTask; namespace proton { diff --git a/searchcore/src/vespa/searchcore/proton/initializer/task_runner.cpp b/searchcore/src/vespa/searchcore/proton/initializer/task_runner.cpp index a416a7d18f3..7deb0afa7af 100644 --- a/searchcore/src/vespa/searchcore/proton/initializer/task_runner.cpp +++ b/searchcore/src/vespa/searchcore/proton/initializer/task_runner.cpp @@ -1,15 +1,13 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "task_runner.h" -#include <vespa/searchlib/common/lambdatask.h> +#include <vespa/vespalib/util/lambdatask.h> #include <vespa/vespalib/util/threadstackexecutor.h> #include <future> -using search::makeLambdaTask; +using vespalib::makeLambdaTask; -namespace proton { - -namespace initializer { +namespace proton::initializer { TaskRunner::TaskRunner(vespalib::Executor &executor) : _executor(executor), @@ -126,6 +124,4 @@ TaskRunner::runTask(InitializerTask::SP rootTask, context->execute(makeLambdaTask([=]() { pollTask(context); } )); } -} // namespace proton::initializer - -} // namespace proton +} diff --git a/searchcore/src/vespa/searchcore/proton/reference/gid_to_lid_change_handler.cpp b/searchcore/src/vespa/searchcore/proton/reference/gid_to_lid_change_handler.cpp index bd3574327bc..abaf37f5084 100644 --- a/searchcore/src/vespa/searchcore/proton/reference/gid_to_lid_change_handler.cpp +++ b/searchcore/src/vespa/searchcore/proton/reference/gid_to_lid_change_handler.cpp @@ -2,14 +2,12 @@ #include "gid_to_lid_change_handler.h" #include "i_gid_to_lid_change_listener.h" -#include <vespa/searchlib/common/lambdatask.h> #include <vespa/searchcorespi/index/i_thread_service.h> -#include <vespa/document/base/globalid.h> -#include <cassert> #include <vespa/vespalib/stllike/hash_map.hpp> +#include <vespa/vespalib/util/lambdatask.h> +#include <cassert> -using search::makeLambdaTask; - +using vespalib::makeLambdaTask; namespace proton { @@ -22,7 +20,6 @@ GidToLidChangeHandler::GidToLidChangeHandler() { } - GidToLidChangeHandler::~GidToLidChangeHandler() { assert(_closed); diff --git a/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_forwarder.cpp b/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_forwarder.cpp index 7e8f16ac9a1..22af1dc1692 100644 --- a/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_forwarder.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_forwarder.cpp @@ -1,10 +1,10 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "disk_mem_usage_forwarder.h" -#include <vespa/searchlib/common/lambdatask.h> +#include <vespa/vespalib/util/lambdatask.h> #include <cassert> -using search::makeLambdaTask; +using vespalib::makeLambdaTask; namespace proton { diff --git a/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_sampler.cpp b/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_sampler.cpp index fb7d712db9d..bacf80e69a6 100644 --- a/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_sampler.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_sampler.cpp @@ -2,10 +2,10 @@ #include "disk_mem_usage_sampler.h" #include <vespa/vespalib/util/timer.h> -#include <vespa/searchlib/common/lambdatask.h> +#include <vespa/vespalib/util/lambdatask.h> #include <unistd.h> -using search::makeLambdaTask; +using vespalib::makeLambdaTask; namespace proton { diff --git a/searchcore/src/vespa/searchcore/proton/server/document_subdb_initializer.cpp b/searchcore/src/vespa/searchcore/proton/server/document_subdb_initializer.cpp index f48a9d37e8f..375940ea4df 100644 --- a/searchcore/src/vespa/searchcore/proton/server/document_subdb_initializer.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/document_subdb_initializer.cpp @@ -3,10 +3,10 @@ #include "document_subdb_initializer.h" #include "idocumentsubdb.h" #include <future> -#include <vespa/searchlib/common/lambdatask.h> +#include <vespa/vespalib/util/lambdatask.h> #include <vespa/searchcorespi/index/i_thread_service.h> -using search::makeLambdaTask; +using vespalib::makeLambdaTask; namespace proton { diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp b/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp index 8c89f2a36af..024de835d7a 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp @@ -56,7 +56,7 @@ using storage::spi::Timestamp; using search::common::FileHeaderContext; using proton::initializer::InitializerTask; using proton::initializer::TaskRunner; -using search::makeLambdaTask; +using vespalib::makeLambdaTask; using searchcorespi::IFlushTarget; namespace proton { diff --git a/searchcore/src/vespa/searchcore/proton/server/proton_configurer.cpp b/searchcore/src/vespa/searchcore/proton/server/proton_configurer.cpp index 3e18c05ba0f..fe3314e7976 100644 --- a/searchcore/src/vespa/searchcore/proton/server/proton_configurer.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/proton_configurer.cpp @@ -5,11 +5,11 @@ #include "bootstrapconfig.h" #include "i_proton_configurer_owner.h" #include "i_document_db_config_owner.h" -#include <vespa/searchlib/common/lambdatask.h> +#include <vespa/vespalib/util/lambdatask.h> #include <vespa/vespalib/util/threadstackexecutorbase.h> #include <future> -using search::makeLambdaTask; +using vespalib::makeLambdaTask; using vespa::config::search::core::ProtonConfig; namespace proton { diff --git a/searchcore/src/vespa/searchcore/proton/server/searchable_feed_view.cpp b/searchcore/src/vespa/searchcore/proton/server/searchable_feed_view.cpp index 14556c86c18..6216af77a56 100644 --- a/searchcore/src/vespa/searchcore/proton/server/searchable_feed_view.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/searchable_feed_view.cpp @@ -24,7 +24,7 @@ using storage::spi::BucketInfoResult; using storage::spi::Timestamp; using vespalib::IllegalStateException; using vespalib::make_string; -using search::makeLambdaTask; +using vespalib::makeLambdaTask; namespace proton { diff --git a/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.cpp b/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.cpp index 5fbae951536..e92c11c2fb8 100644 --- a/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.cpp @@ -36,7 +36,7 @@ using search::GrowStrategy; using search::AttributeGuard; using search::AttributeVector; using search::IndexMetaInfo; -using search::makeLambdaTask; +using vespalib::makeLambdaTask; using search::TuneFileDocumentDB; using search::index::Schema; using search::SerialNum; diff --git a/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.cpp b/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.cpp index c35f942ca35..290b7b74e87 100644 --- a/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.cpp @@ -26,7 +26,7 @@ using document::DocumentId; using document::DocumentTypeRepo; using document::DocumentUpdate; using search::index::Schema; -using search::makeLambdaTask; +using vespalib::makeLambdaTask; using search::IDestructorCallback; using search::SerialNum; using storage::spi::BucketInfoResult; diff --git a/searchcore/src/vespa/searchcore/proton/server/tlssyncer.cpp b/searchcore/src/vespa/searchcore/proton/server/tlssyncer.cpp index 9afc818594f..4d104256cbd 100644 --- a/searchcore/src/vespa/searchcore/proton/server/tlssyncer.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/tlssyncer.cpp @@ -3,11 +3,11 @@ #include "tlssyncer.h" #include "igetserialnum.h" #include <vespa/vespalib/util/threadexecutor.h> -#include <vespa/searchlib/common/lambdatask.h> +#include <vespa/vespalib/util/lambdatask.h> #include <vespa/searchlib/transactionlog/syncproxy.h> #include <future> -using search::makeLambdaTask; +using vespalib::makeLambdaTask; using search::SerialNum; namespace proton { diff --git a/searchcorespi/src/vespa/searchcorespi/index/indexmaintainer.cpp b/searchcorespi/src/vespa/searchcorespi/index/indexmaintainer.cpp index a2786c90e95..d18ff417074 100644 --- a/searchcorespi/src/vespa/searchcorespi/index/indexmaintainer.cpp +++ b/searchcorespi/src/vespa/searchcorespi/index/indexmaintainer.cpp @@ -13,7 +13,7 @@ #include <vespa/searchlib/util/filekit.h> #include <vespa/vespalib/util/autoclosurecaller.h> #include <vespa/vespalib/util/closuretask.h> -#include <vespa/searchlib/common/lambdatask.h> +#include <vespa/vespalib/util/lambdatask.h> #include <sstream> #include <vespa/searchcorespi/flush/closureflushtask.h> #include <vespa/vespalib/util/exceptions.h> @@ -31,7 +31,7 @@ using search::common::FileHeaderContext; using search::queryeval::ISourceSelector; using search::queryeval::Source; using search::SerialNum; -using search::makeLambdaTask; +using vespalib::makeLambdaTask; using std::ostringstream; using vespalib::makeClosure; using vespalib::makeTask; diff --git a/searchlib/CMakeLists.txt b/searchlib/CMakeLists.txt index 787ca6ed008..d77ec346cef 100644 --- a/searchlib/CMakeLists.txt +++ b/searchlib/CMakeLists.txt @@ -225,3 +225,10 @@ vespa_define_module( src/tests/util/statefile src/tests/vespa-fileheader-inspect ) + +install_java_artifact(searchlib) +install_fat_java_artifact(searchlib) + +vespa_install_script(src/main/sh/vespa-gbdt-converter bin) +vespa_install_script(src/main/sh/vespa-treenet-converter bin) + diff --git a/searchlib/src/apps/tests/btreestress_test.cpp b/searchlib/src/apps/tests/btreestress_test.cpp index 94f2db165d3..ca92ad4865b 100644 --- a/searchlib/src/apps/tests/btreestress_test.cpp +++ b/searchlib/src/apps/tests/btreestress_test.cpp @@ -1,10 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/log/log.h> -LOG_SETUP("btreestress_test"); + #include <vespa/vespalib/testkit/test_kit.h> -#include <string> -#include <set> -#include <iostream> #include <vespa/searchlib/btree/btreeroot.h> #include <vespa/searchlib/btree/btreebuilder.h> #include <vespa/searchlib/btree/btreenodeallocator.h> @@ -22,16 +18,17 @@ LOG_SETUP("btreestress_test"); #include <vespa/searchlib/btree/btreestore.hpp> #include <vespa/searchlib/btree/btreeaggregator.hpp> - #include <vespa/vespalib/util/threadstackexecutor.h> -#include <vespa/searchlib/common/lambdatask.h> -#include <vespa/searchlib/util/rand48.h> +#include <vespa/vespalib/util/lambdatask.h> + +#include <vespa/log/log.h> +LOG_SETUP("btreestress_test"); using MyTree = search::btree::BTree<uint32_t, uint32_t>; using MyTreeIterator = typename MyTree::Iterator; using MyTreeConstIterator = typename MyTree::ConstIterator; using GenerationHandler = vespalib::GenerationHandler; -using search::makeLambdaTask; +using vespalib::makeLambdaTask; struct Fixture { diff --git a/searchlib/src/apps/tests/memoryindexstress_test.cpp b/searchlib/src/apps/tests/memoryindexstress_test.cpp index 837952061c8..edb9160a1fb 100644 --- a/searchlib/src/apps/tests/memoryindexstress_test.cpp +++ b/searchlib/src/apps/tests/memoryindexstress_test.cpp @@ -16,9 +16,6 @@ #include <vespa/searchlib/common/scheduletaskcallback.h> #include <vespa/vespalib/util/threadstackexecutor.h> #include <vespa/document/repo/configbuilder.h> -#include <vespa/document/datatype/annotationtype.h> -#include <vespa/document/annotation/annotation.h> -#include <vespa/document/annotation/span.h> #include <vespa/document/annotation/spanlist.h> #include <vespa/document/annotation/spantree.h> #include <vespa/searchlib/util/rand48.h> @@ -37,7 +34,7 @@ using document::SpanList; using document::StringFieldValue; using search::ScheduleTaskCallback; using search::index::schema::DataType; -using search::makeLambdaTask; +using vespalib::makeLambdaTask; using search::query::Node; using search::query::SimplePhrase; using search::query::SimpleStringTerm; diff --git a/searchlib/src/tests/memoryindex/memoryindex/memoryindex_test.cpp b/searchlib/src/tests/memoryindex/memoryindex/memoryindex_test.cpp index cfbb0847cd5..77a687796b3 100644 --- a/searchlib/src/tests/memoryindex/memoryindex/memoryindex_test.cpp +++ b/searchlib/src/tests/memoryindex/memoryindex/memoryindex_test.cpp @@ -1,6 +1,5 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/log/log.h> -LOG_SETUP("memoryindex_test"); + #include <vespa/vespalib/testkit/testapp.h> #include <vespa/searchlib/memoryindex/memoryindex.h> @@ -19,11 +18,14 @@ LOG_SETUP("memoryindex_test"); #include <vespa/searchlib/common/scheduletaskcallback.h> #include <vespa/vespalib/util/threadstackexecutor.h> +#include <vespa/log/log.h> +LOG_SETUP("memoryindex_test"); + using document::Document; using document::FieldValue; using search::ScheduleTaskCallback; using search::index::schema::DataType; -using search::makeLambdaTask; +using vespalib::makeLambdaTask; using search::query::Node; using search::query::SimplePhrase; using search::query::SimpleStringTerm; diff --git a/searchlib/src/vespa/searchlib/common/isequencedtaskexecutor.h b/searchlib/src/vespa/searchlib/common/isequencedtaskexecutor.h index 6984f696117..9b825f1c47e 100644 --- a/searchlib/src/vespa/searchlib/common/isequencedtaskexecutor.h +++ b/searchlib/src/vespa/searchlib/common/isequencedtaskexecutor.h @@ -3,10 +3,9 @@ #include <vespa/vespalib/util/executor.h> #include <vespa/vespalib/stllike/hash_fun.h> -#include "lambdatask.h" +#include <vespa/vespalib/util/lambdatask.h> -namespace search -{ +namespace search { /** * Interface class to run multiple tasks in parallel, but tasks with same @@ -50,7 +49,7 @@ public: */ template <class FunctionType> void executeLambda(uint32_t executorId, FunctionType &&function) { - executeTask(executorId, makeLambdaTask(std::forward<FunctionType>(function))); + executeTask(executorId, vespalib::makeLambdaTask(std::forward<FunctionType>(function))); } /** * Wait for all scheduled tasks to complete. @@ -69,7 +68,7 @@ public: template <class FunctionType> void execute(uint64_t componentId, FunctionType &&function) { uint32_t executorId = getExecutorId(componentId); - executeTask(executorId, makeLambdaTask(std::forward<FunctionType>(function))); + executeTask(executorId, vespalib::makeLambdaTask(std::forward<FunctionType>(function))); } /** @@ -84,7 +83,7 @@ public: template <class FunctionType> void execute(vespalib::stringref componentId, FunctionType &&function) { uint32_t executorId = getExecutorId(componentId); - executeTask(executorId, makeLambdaTask(std::forward<FunctionType>(function))); + executeTask(executorId, vespalib::makeLambdaTask(std::forward<FunctionType>(function))); } }; diff --git a/searchlib/src/vespa/searchlib/config/CMakeLists.txt b/searchlib/src/vespa/searchlib/config/CMakeLists.txt index 2f34d228f60..571cd3ad9f1 100644 --- a/searchlib/src/vespa/searchlib/config/CMakeLists.txt +++ b/searchlib/src/vespa/searchlib/config/CMakeLists.txt @@ -4,4 +4,4 @@ vespa_add_library(searchlib_sconfig OBJECT DEPENDS ) vespa_generate_config(searchlib_sconfig translogserver.def) -install(FILES translogserver.def RENAME searchlib.translogserver.def DESTINATION var/db/vespa/config_server/serverdb/classes) +install_config_definition(translogserver.def searchlib.translogserver.def) diff --git a/searchlib/src/vespa/searchlib/docstore/filechunk.cpp b/searchlib/src/vespa/searchlib/docstore/filechunk.cpp index 4fac42c1421..93a85eda0aa 100644 --- a/searchlib/src/vespa/searchlib/docstore/filechunk.cpp +++ b/searchlib/src/vespa/searchlib/docstore/filechunk.cpp @@ -5,7 +5,7 @@ #include "summaryexceptions.h" #include "randreaders.h" #include <vespa/searchlib/util/filekit.h> -#include <vespa/searchlib/common/lambdatask.h> +#include <vespa/vespalib/util/lambdatask.h> #include <vespa/vespalib/data/fileheader.h> #include <vespa/vespalib/data/databuffer.h> #include <vespa/vespalib/stllike/asciistream.h> @@ -345,14 +345,14 @@ FileChunk::appendTo(vespalib::ThreadExecutor & executor, const IGetLid & db, IWr for (size_t chunkId(0); chunkId < numChunks; chunkId++) { std::promise<Chunk::UP> promisedChunk; std::future<Chunk::UP> futureChunk = promisedChunk.get_future(); - executor.execute(makeLambdaTask([promise = std::move(promisedChunk), chunkId, this]() mutable { + executor.execute(vespalib::makeLambdaTask([promise = std::move(promisedChunk), chunkId, this]() mutable { const ChunkInfo & cInfo(_chunkInfo[chunkId]); vespalib::DataBuffer whole(0ul, ALIGNMENT); FileRandRead::FSP keepAlive(_file->read(cInfo.getOffset(), whole, cInfo.getSize())); promise.set_value(std::make_unique<Chunk>(chunkId, whole.getData(), whole.getDataLen())); })); - singleExecutor.execute(makeLambdaTask([args = &fixedParams, chunk = std::move(futureChunk)]() mutable { + singleExecutor.execute(vespalib::makeLambdaTask([args = &fixedParams, chunk = std::move(futureChunk)]() mutable { appendChunks(args, chunk.get()); })); } diff --git a/searchsummary/src/vespa/searchsummary/config/CMakeLists.txt b/searchsummary/src/vespa/searchsummary/config/CMakeLists.txt index 7f6dc5c4cdd..a01938d8ad4 100644 --- a/searchsummary/src/vespa/searchsummary/config/CMakeLists.txt +++ b/searchsummary/src/vespa/searchsummary/config/CMakeLists.txt @@ -4,4 +4,4 @@ vespa_add_library(searchsummary_config OBJECT DEPENDS ) vespa_generate_config(searchsummary_config juniperrc.def) -install(FILES juniperrc.def RENAME vespa.config.search.summary.juniperrc.def DESTINATION var/db/vespa/config_server/serverdb/classes) +install_config_definition(juniperrc.def vespa.config.search.summary.juniperrc.def) diff --git a/simplemetrics/CMakeLists.txt b/simplemetrics/CMakeLists.txt new file mode 100644 index 00000000000..c145fbb2ec7 --- /dev/null +++ b/simplemetrics/CMakeLists.txt @@ -0,0 +1,4 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +install_fat_java_artifact(simplemetrics) + +install_config_definition(src/main/resources/configdefinitions/manager.def metrics.manager.def) diff --git a/standalone-container/CMakeLists.txt b/standalone-container/CMakeLists.txt new file mode 100644 index 00000000000..83c58e09945 --- /dev/null +++ b/standalone-container/CMakeLists.txt @@ -0,0 +1,2 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +install_fat_java_artifact(standalone-container) diff --git a/statistics/CMakeLists.txt b/statistics/CMakeLists.txt new file mode 100644 index 00000000000..3b187b72d6e --- /dev/null +++ b/statistics/CMakeLists.txt @@ -0,0 +1,2 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +install_config_definition(src/main/resources/configdefinitions/statistics.def container.statistics.def) diff --git a/storage/src/vespa/storage/bucketdb/CMakeLists.txt b/storage/src/vespa/storage/bucketdb/CMakeLists.txt index 5c818631d54..6e3a0c2e986 100644 --- a/storage/src/vespa/storage/bucketdb/CMakeLists.txt +++ b/storage/src/vespa/storage/bucketdb/CMakeLists.txt @@ -17,6 +17,6 @@ vespa_add_library(storage_bucketdb OBJECT storage_storageconfig ) vespa_generate_config(storage_bucketdb stor-bucketdb.def) -install(FILES stor-bucketdb.def RENAME vespa.config.content.core.stor-bucketdb.def DESTINATION var/db/vespa/config_server/serverdb/classes) +install_config_definition(stor-bucketdb.def vespa.config.content.core.stor-bucketdb.def) vespa_generate_config(storage_bucketdb stor-bucket-init.def) -install(FILES stor-bucket-init.def RENAME vespa.config.content.core.stor-bucket-init.def DESTINATION var/db/vespa/config_server/serverdb/classes) +install_config_definition(stor-bucket-init.def vespa.config.content.core.stor-bucket-init.def) diff --git a/storage/src/vespa/storage/config/CMakeLists.txt b/storage/src/vespa/storage/config/CMakeLists.txt index 0f3f392f324..4a20d510043 100644 --- a/storage/src/vespa/storage/config/CMakeLists.txt +++ b/storage/src/vespa/storage/config/CMakeLists.txt @@ -5,26 +5,26 @@ vespa_add_library(storage_storageconfig OBJECT DEPENDS ) vespa_generate_config(storage_storageconfig stor-communicationmanager.def) -install(FILES stor-communicationmanager.def RENAME vespa.config.content.core.stor-communicationmanager.def DESTINATION var/db/vespa/config_server/serverdb/classes) +install_config_definition(stor-communicationmanager.def vespa.config.content.core.stor-communicationmanager.def) vespa_generate_config(storage_storageconfig stor-distributormanager.def) -install(FILES stor-distributormanager.def RENAME vespa.config.content.core.stor-distributormanager.def DESTINATION var/db/vespa/config_server/serverdb/classes) +install_config_definition(stor-distributormanager.def vespa.config.content.core.stor-distributormanager.def) vespa_generate_config(storage_storageconfig stor-server.def) -install(FILES stor-server.def RENAME vespa.config.content.core.stor-server.def DESTINATION var/db/vespa/config_server/serverdb/classes) +install_config_definition(stor-server.def vespa.config.content.core.stor-server.def) vespa_generate_config(storage_storageconfig stor-status.def) -install(FILES stor-status.def RENAME vespa.config.content.core.stor-status.def DESTINATION var/db/vespa/config_server/serverdb/classes) +install_config_definition(stor-status.def vespa.config.content.core.stor-status.def) vespa_generate_config(storage_storageconfig stor-messageforwarder.def) -install(FILES stor-messageforwarder.def RENAME vespa.config.content.core.stor-messageforwarder.def DESTINATION var/db/vespa/config_server/serverdb/classes) +install_config_definition(stor-messageforwarder.def vespa.config.content.core.stor-messageforwarder.def) vespa_generate_config(storage_storageconfig stor-opslogger.def) -install(FILES stor-opslogger.def RENAME vespa.config.content.core.stor-opslogger.def DESTINATION var/db/vespa/config_server/serverdb/classes) +install_config_definition(stor-opslogger.def vespa.config.content.core.stor-opslogger.def) vespa_generate_config(storage_storageconfig stor-visitordispatcher.def) -install(FILES stor-visitordispatcher.def RENAME vespa.config.content.core.stor-visitordispatcher.def DESTINATION var/db/vespa/config_server/serverdb/classes) +install_config_definition(stor-visitordispatcher.def vespa.config.content.core.stor-visitordispatcher.def) vespa_generate_config(storage_storageconfig stor-integritychecker.def) -install(FILES stor-integritychecker.def RENAME vespa.config.content.core.stor-integritychecker.def DESTINATION var/db/vespa/config_server/serverdb/classes) +install_config_definition(stor-integritychecker.def vespa.config.content.core.stor-integritychecker.def) vespa_generate_config(storage_storageconfig stor-bucketmover.def) -install(FILES stor-bucketmover.def RENAME vespa.config.content.core.stor-bucketmover.def DESTINATION var/db/vespa/config_server/serverdb/classes) +install_config_definition(stor-bucketmover.def vespa.config.content.core.stor-bucketmover.def) vespa_generate_config(storage_storageconfig stor-bouncer.def) -install(FILES stor-bouncer.def RENAME vespa.config.content.core.stor-bouncer.def DESTINATION var/db/vespa/config_server/serverdb/classes) +install_config_definition(stor-bouncer.def vespa.config.content.core.stor-bouncer.def) vespa_generate_config(storage_storageconfig stor-prioritymapping.def) -install(FILES stor-prioritymapping.def RENAME vespa.config.content.core.stor-prioritymapping.def DESTINATION var/db/vespa/config_server/serverdb/classes) +install_config_definition(stor-prioritymapping.def vespa.config.content.core.stor-prioritymapping.def) vespa_generate_config(storage_storageconfig rpc-provider.def) -install(FILES rpc-provider.def RENAME vespa.config.content.core.rpc-provider.def DESTINATION var/db/vespa/config_server/serverdb/classes) +install_config_definition(rpc-provider.def vespa.config.content.core.rpc-provider.def) diff --git a/storage/src/vespa/storage/visiting/CMakeLists.txt b/storage/src/vespa/storage/visiting/CMakeLists.txt index ee10bbd58ab..6d93d96114d 100644 --- a/storage/src/vespa/storage/visiting/CMakeLists.txt +++ b/storage/src/vespa/storage/visiting/CMakeLists.txt @@ -17,4 +17,4 @@ vespa_add_library(storage_visitor OBJECT storage_storageconfig ) vespa_generate_config(storage_visitor stor-visitor.def) -install(FILES stor-visitor.def RENAME vespa.config.content.core.stor-visitor.def DESTINATION var/db/vespa/config_server/serverdb/classes) +install_config_definition(stor-visitor.def vespa.config.content.core.stor-visitor.def) diff --git a/storageserver/src/apps/storaged/CMakeLists.txt b/storageserver/src/apps/storaged/CMakeLists.txt index b971ce44339..2c5c837d3cd 100644 --- a/storageserver/src/apps/storaged/CMakeLists.txt +++ b/storageserver/src/apps/storaged/CMakeLists.txt @@ -9,3 +9,5 @@ vespa_add_executable(storageserver_storaged_app storageserver_storageapp searchlib_searchlib_uca ) + +install(CODE "execute_process(COMMAND ln -snf vespa-storaged-bin $ENV{DESTDIR}/${CMAKE_INSTALL_PREFIX}/sbin/vespa-distributord-bin)") diff --git a/vespa-http-client/CMakeLists.txt b/vespa-http-client/CMakeLists.txt new file mode 100644 index 00000000000..511b4a4c985 --- /dev/null +++ b/vespa-http-client/CMakeLists.txt @@ -0,0 +1,2 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +install_fat_java_artifact(vespa-http-client) diff --git a/vespa_jersey2/CMakeLists.txt b/vespa_jersey2/CMakeLists.txt new file mode 100644 index 00000000000..f4f6c44202c --- /dev/null +++ b/vespa_jersey2/CMakeLists.txt @@ -0,0 +1,2 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +install_java_artifact_dependencies(vespa_jersey2) diff --git a/vespabase/CMakeLists.txt b/vespabase/CMakeLists.txt index e658a959d84..ea007b130f9 100644 --- a/vespabase/CMakeLists.txt +++ b/vespabase/CMakeLists.txt @@ -31,3 +31,7 @@ configure_file(src/vespa-configserver.service.in src/vespa-configserver.service install(FILES ${CMAKE_CURRENT_BINARY_DIR}/src/vespa.service ${CMAKE_CURRENT_BINARY_DIR}/src/vespa-configserver.service DESTINATION etc/systemd/system) install(FILES src/Defaults.pm DESTINATION lib/perl5/site_perl/Yahoo/Vespa) + +configure_file(conf/default-env.txt.in conf/default-env.txt @ONLY) +install(FILES ${CMAKE_CURRENT_BINARY_DIR}/conf/default-env.txt DESTINATION conf/vespa) + diff --git a/vespabase/conf/default-env.txt.in b/vespabase/conf/default-env.txt.in new file mode 100644 index 00000000000..4855ac9f571 --- /dev/null +++ b/vespabase/conf/default-env.txt.in @@ -0,0 +1,2 @@ +fallback VESPA_HOME @CMAKE_INSTALL_PREFIX@ +override VESPA_USER vespa diff --git a/vespabase/src/rhel-prestart.sh b/vespabase/src/rhel-prestart.sh index c00557243a7..37fc5e98533 100755 --- a/vespabase/src/rhel-prestart.sh +++ b/vespabase/src/rhel-prestart.sh @@ -97,6 +97,7 @@ fixdir ${VESPA_USER} wheel 755 var/db/vespa/config_server/serverdb/application fixdir ${VESPA_USER} wheel 755 var/db/vespa/index fixdir ${VESPA_USER} wheel 755 var/db/vespa/logcontrol fixdir ${VESPA_USER} wheel 755 var/db/vespa/search +fixdir ${VESPA_USER} wheel 755 var/jdisc_core fixdir ${VESPA_USER} wheel 755 var/vespa/bundlecache fixdir ${VESPA_USER} wheel 755 var/vespa/bundlecache/configserver fixdir ${VESPA_USER} wheel 755 var/vespa/cache/config/ diff --git a/vespaclient-container-plugin/CMakeLists.txt b/vespaclient-container-plugin/CMakeLists.txt new file mode 100644 index 00000000000..4c8a8647b23 --- /dev/null +++ b/vespaclient-container-plugin/CMakeLists.txt @@ -0,0 +1,2 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +install_fat_java_artifact(vespaclient-container-plugin) diff --git a/vespaclient-core/CMakeLists.txt b/vespaclient-core/CMakeLists.txt new file mode 100644 index 00000000000..facea0b447d --- /dev/null +++ b/vespaclient-core/CMakeLists.txt @@ -0,0 +1,3 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +install_config_definition(src/main/resources/configdefinitions/feeder.def vespaclient.config.feeder.def) +install_config_definition(src/main/resources/configdefinitions/spooler.def vespa.config.content.spooler.spooler.def) diff --git a/vespaclient-java/CMakeLists.txt b/vespaclient-java/CMakeLists.txt new file mode 100644 index 00000000000..b9240adee8a --- /dev/null +++ b/vespaclient-java/CMakeLists.txt @@ -0,0 +1,12 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +install_fat_java_artifact(vespaclient-java) + +vespa_install_script(src/main/sh/vespa-document-statistics.sh vespa-document-statistics bin) +vespa_install_script(src/main/sh/vespa-stat.sh vespa-stat bin) +vespa_install_script(src/main/sh/vespa-query-profile-dump-tool.sh vespa-query-profile-dump-tool bin) +vespa_install_script(src/main/sh/vespa-summary-benchmark.sh vespa-summary-benchmark bin) +vespa_install_script(src/main/sh/vespa-destination.sh vespa-destination bin) +vespa_install_script(src/main/sh/vespa-feeder.sh vespa-feeder bin) +vespa_install_script(src/main/sh/vespa-get.sh vespa-get bin) +vespa_install_script(src/main/sh/vespa-visit.sh vespa-visit bin) +vespa_install_script(src/main/sh/vespa-visit-target.sh vespa-visit-target bin) diff --git a/vespajlib/CMakeLists.txt b/vespajlib/CMakeLists.txt new file mode 100644 index 00000000000..7235535ace8 --- /dev/null +++ b/vespajlib/CMakeLists.txt @@ -0,0 +1,2 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +install_java_artifact(vespajlib) diff --git a/vespajlib/src/main/java/com/yahoo/concurrent/classlock/ClassLock.java b/vespajlib/src/main/java/com/yahoo/concurrent/classlock/ClassLock.java index 2a3c70d31d2..b0d26bfeb1c 100644 --- a/vespajlib/src/main/java/com/yahoo/concurrent/classlock/ClassLock.java +++ b/vespajlib/src/main/java/com/yahoo/concurrent/classlock/ClassLock.java @@ -1,6 +1,8 @@ package com.yahoo.concurrent.classlock; /** + * An acquired lock which is released on close + * * @author valerijf */ public class ClassLock implements AutoCloseable { @@ -12,6 +14,11 @@ public class ClassLock implements AutoCloseable { this.clazz = clazz; } + /** + * Releases this lock + * + * @throws IllegalArgumentException if this lock has already been released + */ @Override public void close() { classLocking.unlock(clazz, this); diff --git a/vespajlib/src/main/java/com/yahoo/concurrent/classlock/ClassLocking.java b/vespajlib/src/main/java/com/yahoo/concurrent/classlock/ClassLocking.java index 27b020b578d..5330e869396 100644 --- a/vespajlib/src/main/java/com/yahoo/concurrent/classlock/ClassLocking.java +++ b/vespajlib/src/main/java/com/yahoo/concurrent/classlock/ClassLocking.java @@ -5,16 +5,33 @@ import java.util.Map; import java.util.function.BooleanSupplier; /** + * This class is injectable to Vespa plugins and is used to acquire locks cross + * application deployments. + * * @author valerijf */ public class ClassLocking { private final Map<String, ClassLock> classLocks = new HashMap<>(); private final Object monitor = new Object(); + /** + * Locks key. This will block until the key is acquired. + * Users of this <b>must</b> close any lock acquired. + */ public ClassLock lock(Class<?> clazz) { return lockWhile(clazz, () -> true); } + /** + * Locks key. This will block until the key is acquired or the interrupt condition is + * no longer true. Condition is only checked at the start, everytime a lock is released + * and when {@link #interrupt()} is called. + * + * Users of this <b>must</b> close any lock acquired. + * + * @throws LockInterruptException if interruptCondition returned false before + * the lock could be acquired + */ public ClassLock lockWhile(Class<?> clazz, BooleanSupplier interruptCondition) { synchronized (monitor) { while (classLocks.containsKey(clazz.getName())) { @@ -45,6 +62,9 @@ public class ClassLocking { } } + /** + * Notifies {@link #lockWhile} to check the interrupt condition + */ public void interrupt() { synchronized (monitor) { monitor.notifyAll(); diff --git a/vespalib/src/tests/executor/executor_test.cpp b/vespalib/src/tests/executor/executor_test.cpp index c508417e1c2..9015391beaa 100644 --- a/vespalib/src/tests/executor/executor_test.cpp +++ b/vespalib/src/tests/executor/executor_test.cpp @@ -2,30 +2,13 @@ #include <vespa/vespalib/testkit/testapp.h> #include <vespa/vespalib/util/closuretask.h> +#include <vespa/vespalib/util/lambdatask.h> using namespace vespalib; -namespace { - -class Test : public vespalib::TestApp { - void requireThatClosuresCanBeWrappedInATask(); - -public: - int Main() override; -}; - -int -Test::Main() -{ - TEST_INIT("executor_test"); - - TEST_DO(requireThatClosuresCanBeWrappedInATask()); - - TEST_DONE(); -} - void setBool(bool *b) { *b = true; } -void Test::requireThatClosuresCanBeWrappedInATask() { + +TEST("require that closures can be wrapped as tasks") { bool called = false; Executor::Task::UP task = makeTask(makeClosure(setBool, &called)); EXPECT_TRUE(!called); @@ -33,6 +16,12 @@ void Test::requireThatClosuresCanBeWrappedInATask() { EXPECT_TRUE(called); } -} // namespace +TEST("require that lambdas can be wrapped as tasks") { + bool called = false; + Executor::Task::UP task = makeLambdaTask([&called]() { called = true; }); + EXPECT_TRUE(!called); + task->run(); + EXPECT_TRUE(called); +} -TEST_APPHOOK(Test); +TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchlib/src/vespa/searchlib/common/lambdatask.h b/vespalib/src/vespa/vespalib/util/lambdatask.h index 01b57694d11..35543407aaa 100644 --- a/searchlib/src/vespa/searchlib/common/lambdatask.h +++ b/vespalib/src/vespa/vespalib/util/lambdatask.h @@ -1,9 +1,9 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include <vespa/vespalib/util/executor.h> +#include "executor.h" -namespace search { +namespace vespalib { template <class FunctionType> class LambdaTask : public vespalib::Executor::Task { @@ -26,4 +26,4 @@ makeLambdaTask(FunctionType &&function) (std::forward<FunctionType>(function)); } -} // namespace search +} diff --git a/vsm/src/vespa/vsm/config/CMakeLists.txt b/vsm/src/vespa/vsm/config/CMakeLists.txt index ea65d8c8fb4..e3bd2db68e2 100644 --- a/vsm/src/vespa/vsm/config/CMakeLists.txt +++ b/vsm/src/vespa/vsm/config/CMakeLists.txt @@ -4,8 +4,8 @@ vespa_add_library(vsm_vconfig OBJECT DEPENDS ) vespa_generate_config(vsm_vconfig vsmfields.def) -install(FILES vsmfields.def RENAME vespa.config.search.vsm.vsmfields.def DESTINATION var/db/vespa/config_server/serverdb/classes) +install_config_definition(vsmfields.def vespa.config.search.vsm.vsmfields.def) vespa_generate_config(vsm_vconfig vsm.def) -install(FILES vsm.def RENAME vespa.config.search.vsm.vsm.def DESTINATION var/db/vespa/config_server/serverdb/classes) +install_config_definition(vsm.def vespa.config.search.vsm.vsm.def) vespa_generate_config(vsm_vconfig vsmsummary.def) -install(FILES vsmsummary.def RENAME vespa.config.search.vsm.vsmsummary.def DESTINATION var/db/vespa/config_server/serverdb/classes) +install_config_definition(vsmsummary.def vespa.config.search.vsm.vsmsummary.def) diff --git a/zkfacade/CMakeLists.txt b/zkfacade/CMakeLists.txt new file mode 100644 index 00000000000..6610356c0fd --- /dev/null +++ b/zkfacade/CMakeLists.txt @@ -0,0 +1,2 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +install_fat_java_artifact(zkfacade) |