diff options
| author | Arne Juul <arnej@yahoo-inc.com> | 2017-09-22 10:37:49 +0000 |
|---|---|---|
| committer | Arne Juul <arnej@yahoo-inc.com> | 2017-09-22 10:37:49 +0000 |
| commit | 5767d9b188cd3efadbd30e2decd963c0d168fdbc (patch) | |
| tree | 6ba9f61ce8d24ea8b5707aa21bb6600f8d08e84f | |
| parent | c9f97f7effb5fcd2a8ffe620258cd5629fb7dc73 (diff) | |
| parent | f4a65c8f7cb27bd0e449d99e223ba1a0d9135049 (diff) | |
Merge branch 'master' into bratseth/nonfunctional-changes-4
Conflicts:
vespajlib/src/main/java/com/yahoo/concurrent/lock/Locks.java
496 files changed, 4043 insertions, 4159 deletions
diff --git a/CMakeLists.txt b/CMakeLists.txt index 36d29ece0f1..413d232c00d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -6,6 +6,11 @@ cmake_minimum_required(VERSION 3.5 FATAL_ERROR) project(vespa CXX C) +# allows import of project in CLion on OSX +if (${CMAKE_SYSTEM_NAME} MATCHES "Darwin") + set(CMAKE_THREAD_LIBS_INIT "-lpthread") +endif() + # TODO: Move this to where it's actually needed find_package(JNI REQUIRED) @@ -67,12 +72,12 @@ add_subdirectory(vespalib) add_subdirectory(vespalog) add_subdirectory(vespamalloc) add_subdirectory(vsm) +# Note: Change when cmake gets proper post-install support. +# Post installation steps are run from dist subdirectory which needs to be the last add_subdirectory(...) call in this file. +add_subdirectory(dist) # Create module targets with name ${MODULE}+module depending on every target defined within that module __create_module_targets(TARGETS "module") # Create module targets with name ${MODULE}+test depending on every test target defined within that module __create_module_targets(TEST_TARGETS "test") - -# Post installation steps. Need to be the last install(...) call in this file. -install(CODE "execute_process(COMMAND ${CMAKE_CURRENT_LIST_DIR}/dist/post_install.sh ${CMAKE_INSTALL_PREFIX})") diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 487d7fb8522..a2690f6d127 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -4,7 +4,7 @@ and the [Vespa documentation](http://github.com/vespa-engine/documentation) are welcome. This documents tells you what you need to know to contribute. -##Â Open development +## Open development All work on Vespa happens directly on Github, using the [Github flow model](https://guides.github.com/introduction/flow/). We release the master branch a few times a week and you should expect it to almost always work. @@ -39,4 +39,4 @@ It may help to read the READMEs of each module. ## License and copyright If you add new files you are welcome to use your own copyright. In any case the code (or documentation) you submit will be licensed -under the Apache 2.0 license.
\ No newline at end of file +under the Apache 2.0 license. diff --git a/README.md b/README.md index 9346af970e0..51ecc5fb02e 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ documentation at http://docs.vespa.ai/. Code licensed under the Apache 2.0 license. See [LICENSE](LICENSE) for terms. -[](https://travis-ci.org/vespa-engine/vespa) +Travis-CI build status: [](https://travis-ci.org/vespa-engine/vespa) ## Get started developing diff --git a/application/pom.xml b/application/pom.xml index 3ea1b8270f4..208344042a4 100644 --- a/application/pom.xml +++ b/application/pom.xml @@ -85,6 +85,14 @@ for user projects must be added in compile scope here. These dependencies are explicitly excluded (or set to non-compile scope) in the container-dev module. --> <dependency> + <groupId>com.ibm.icu</groupId> + <artifactId>icu4j</artifactId> + </dependency> + <dependency> + <groupId>commons-collections</groupId> + <artifactId>commons-collections</artifactId> + </dependency> + <dependency> <groupId>org.antlr</groupId> <artifactId>antlr-runtime</artifactId> </dependency> @@ -100,6 +108,31 @@ <groupId>org.hdrhistogram</groupId> <artifactId>HdrHistogram</artifactId> </dependency> + <dependency> + <groupId>xerces</groupId> + <artifactId>xercesImpl</artifactId> + </dependency> + + <!-- These dependencies are necessary in test classpath when using jdisc_http_filters --> + <dependency> + <groupId>commons-beanutils</groupId> + <artifactId>commons-beanutils</artifactId> + <version>1.7.0</version> + </dependency> + <dependency> + <groupId>commons-beanutils</groupId> + <artifactId>commons-beanutils-core</artifactId> + <version>1.8.0</version> + </dependency> + <dependency> + <groupId>commons-configuration</groupId> + <artifactId>commons-configuration</artifactId> + </dependency> + <dependency> + <groupId>commons-digester</groupId> + <artifactId>commons-digester</artifactId> + <version>1.8</version> + </dependency> </dependencies> <build> diff --git a/build_settings.cmake b/build_settings.cmake index e6483785126..578484fc0a7 100644 --- a/build_settings.cmake +++ b/build_settings.cmake @@ -26,7 +26,7 @@ set(CXX_SPECIFIC_WARN_OPTS "-Wsuggest-override -Wnon-virtual-dtor") # Select C++ ABI if(DEFINED VESPA_CXX_ABI_FLAGS) else() -set (VESPA_CXX_ABI_FLAGS "-D_GLIBCXX_USE_CXX11_ABI=0") + set (VESPA_CXX_ABI_FLAGS "-D_GLIBCXX_USE_CXX11_ABI=0") endif() # C and C++ compiler flags @@ -38,7 +38,7 @@ set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -Wl,--build-id -lato SET(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -rdynamic" ) # Use C++ 17 -#TODO renable when cmake 3.8 is out. +# TODO renable when cmake 3.8 is out. # set(CMAKE_CXX_STANDARD 17) # Always build shared libs if not explicitly specified diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetController.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetController.java index 0b0653274a4..120b482e11d 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetController.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetController.java @@ -665,7 +665,8 @@ public class FleetController implements NodeStateOrHostInfoChangeHandler, NodeAd final RemoteClusterControllerTask task = remoteTasks.poll(); log.finest(() -> String.format("Processing remote task of type '%s'", task.getClass().getName())); task.doRemoteFleetControllerTask(context); - if (!task.hasVersionAckDependency()) { + // We cannot introduce a version barrier for tasks when we're not the master (and therefore will not publish new versions). + if (!isMaster() || !task.hasVersionAckDependency()) { log.finest(() -> String.format("Done processing remote task of type '%s'", task.getClass().getName())); task.notifyCompleted(); } else { diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/restapiv2/requests/SetNodeStateRequest.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/restapiv2/requests/SetNodeStateRequest.java index c56ce5da352..2c094769961 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/restapiv2/requests/SetNodeStateRequest.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/restapiv2/requests/SetNodeStateRequest.java @@ -67,7 +67,9 @@ public class SetNodeStateRequest extends Request<SetResponse> { @Override public boolean hasVersionAckDependency() { - return (this.responseWait == SetUnitStateRequest.ResponseWait.WAIT_UNTIL_CLUSTER_ACKED); + // FIXME this is a temporary change while edge cases in interactions between controller + // and orchestration are sorted out. + return false; } static SetResponse setWantedState( diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/StateChangeTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/StateChangeTest.java index 49cfe8bd3b6..6a8ffae8f3f 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/StateChangeTest.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/StateChangeTest.java @@ -1508,4 +1508,19 @@ public class StateChangeTest extends FleetControllerTest { assertTrue(task.isCompleted()); } + @Test + public void synchronous_task_immediately_answered_when_not_leader() throws Exception { + FleetControllerOptions options = optionsWithZeroTransitionTime(); + options.fleetControllerCount = 3; + RemoteTaskFixture fixture = createFixtureWith(options); + + fixture.loseLeadership(); + markAllNodesAsUp(options); + + MockTask task = fixture.scheduleVersionDependentTaskWithSideEffects(); + + assertTrue(task.isInvoked()); + assertTrue(task.isCompleted()); + } + } diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/restapiv2/SetNodeStateTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/restapiv2/SetNodeStateTest.java index 88483e47f4e..0f35b96c31d 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/restapiv2/SetNodeStateTest.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/restapiv2/SetNodeStateTest.java @@ -374,10 +374,11 @@ public class SetNodeStateTest extends StateRestApiTest { .setNewState("user", "maintenance", "whatever reason.")); } + // FIXME requests should be tagged as version dependent; temporary workaround @Test - public void set_node_state_requests_are_by_default_tagged_as_having_version_ack_dependency() { + public void set_node_state_requests_are_by_default_not_tagged_as_having_version_ack_dependency() { SetNodeStateRequest request = createDummySetNodeStateRequest(); - assertTrue(request.hasVersionAckDependency()); + assertFalse(request.hasVersionAckDependency()); } @Test diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ApplicationInfo.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ApplicationInfo.java new file mode 100644 index 00000000000..c6a72ebb3ff --- /dev/null +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ApplicationInfo.java @@ -0,0 +1,26 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.model.api; + +import com.yahoo.config.provision.ApplicationId; + +public class ApplicationInfo { + private final ApplicationId applicationId; + private final long generation; + private final Model model; // NOT immutable + + public ApplicationInfo(ApplicationId applicationId, long generation, Model model) { + this.applicationId = applicationId; + this.generation = generation; + this.model = model; + } + + public ApplicationId getApplicationId() { + return applicationId; + } + public long getGeneration() { + return generation; + } + public Model getModel() { + return model; + } +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelListener.java b/config-model-api/src/main/java/com/yahoo/config/model/api/SuperModelListener.java index 81d706cf2e9..16580c9e9f6 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelListener.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/SuperModelListener.java @@ -1,10 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -package com.yahoo.vespa.config.server; +package com.yahoo.config.model.api; import com.yahoo.config.provision.ApplicationId; -import com.yahoo.vespa.config.server.application.Application; -import com.yahoo.vespa.config.server.model.SuperModel; /** * Interface for those wanting to be notified about changes to the SuperModel. @@ -14,10 +11,10 @@ public interface SuperModelListener { * Application has been activated: Either deployed the first time, * internally redeployed, or externally triggered redeploy. */ - void applicationActivated(SuperModel superModel, Application application); + void applicationActivated(ApplicationInfo application); /** * Application has been removed. */ - void applicationRemoved(SuperModel superModel, ApplicationId applicationId); + void applicationRemoved(ApplicationId id); } diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/SuperModelProvider.java b/config-model-api/src/main/java/com/yahoo/config/model/api/SuperModelProvider.java new file mode 100644 index 00000000000..42437b20b83 --- /dev/null +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/SuperModelProvider.java @@ -0,0 +1,13 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.model.api; + +import java.util.List; + +public interface SuperModelProvider { + /** + * Returns all applications in the SuperModel. All changes to the SuperModel + * following that snapshot will be published to the listener. Warning: The listener + * methods may have been invoked before (or concurrently with) this method returning. + */ + List<ApplicationInfo> snapshot(SuperModelListener listener); +} diff --git a/config-model/fix-from-to.sh b/config-model/fix-from-to.sh deleted file mode 100755 index 6223465a987..00000000000 --- a/config-model/fix-from-to.sh +++ /dev/null @@ -1,71 +0,0 @@ -#!/bin/sh -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -( cat << "EOF" -temp/id src/test/derived/id -temp/mlr src/test/derived/mlr -temp/mail src/test/derived/mail -temp/local src/test/derived/local -temp/music src/test/derived/music -temp/types src/test/derived/types -temp/arrays src/test/derived/arrays -temp/flickr src/test/derived/flickr -temp/mother src/test/derived/inheritance/mother -temp/music3 src/test/derived/music3 -temp/complex src/test/derived/complex -temp/deriver src/test/derived/deriver -temp/mail4_1 src/test/derived/mail4_1 -temp/newrank src/test/derived/newrank -temp/sorting src/test/derived/sorting -temp/advanced src/test/derived/advanced -temp/ranktypes src/test/derived/ranktypes -temp/attributes src/test/derived/attributes -temp/emptychild src/test/derived/emptychild -temp/exactmatch src/test/derived/exactmatch -temp/indexschema src/test/derived/indexschema -temp/inheritance src/test/derived/inheritance -temp/emptydefault src/test/derived/emptydefault -temp/rankprofiles src/test/derived/rankprofiles -temp/attributerank src/test/derived/attributerank -temp/indexsettings src/test/derived/indexsettings -temp/indexswitches src/test/derived/indexswitches -temp/rankexpression src/test/derived/rankexpression -temp/rankproperties src/test/derived/rankproperties -temp/structanyorder src/test/derived/structanyorder -temp/documentderiver src/test/derived/documentderiver -temp/streamingstruct src/test/derived/streamingstruct -temp/annotationssimple src/test/derived/annotationssimple -temp/attributeprefetch src/test/derived/attributeprefetch -temp/multiplesummaries src/test/derived/multiplesummaries -temp/inheritancebadtypes src/test/derived/inheritancebadtypes -temp/twostreamingstructs src/test/derived/twostreamingstructs -temp/annotationsreference src/test/derived/annotationsreference -temp/prefixexactattribute src/test/derived/prefixexactattribute -temp/annotationspolymorphy src/test/derived/annotationspolymorphy -temp/annotationsinheritance src/test/derived/annotationsinheritance -temp/streamingstructdefault src/test/derived/streamingstructdefault -temp/annotationsinheritance2 src/test/derived/annotationsinheritance2 -temp/annotationsimplicitstruct src/test/derived/annotationsimplicitstruct -temp/integerattributetostringindex src/test/derived/integerattributetostringindex -temp/combinedattributeandindexsearch src/test/derived/combinedattributeandindexsearch -tmp/v2/complex/search/cluster.music/tlds/tld.0 src/test/cfg/search/compare/complex/search/cluster.music/tlds/tld.0 -tmp/v2/complex/search/cluster.music/tlds/tld.1 src/test/cfg/search/compare/complex/search/cluster.music/tlds/tld.1 -tmp/v2/complex/search/cluster.rt/tlds/tld.0 src/test/cfg/search/compare/complex/search/cluster.rt/tlds/tld.0 -tmp/v2/optionals/search/cluster.music/tlds/tld.0 src/test/cfg/search/compare/optionals/search/cluster.music/tlds/tld.0 -tmp/v2/simple/search/cluster.music/tlds/tld.0 src/test/cfg/search/compare/simple/search/cluster.music/tlds/tld.0 -tmp/v2/twoFeedTargetClusters/search/cluster.music1/tlds/tld.0 src/test/cfg/search/compare/twoFeedTargetClusters/search/cluster.music1/tlds/tld.0 -tmp/v2/twoFeedTargetClusters/search/cluster.music2/tlds/tld.0 src/test/cfg/search/compare/twoFeedTargetClusters/search/cluster.music2/tlds/tld.0 -EOF -) | while read from to ; do - echo check fromdir $from todir $to 1>&2 - test -d $from || echo missing $from 1>&2 - test -d $to || echo missing $to 1>&2 - for fromfile in $from/*cfg ; do - base=${fromfile##*/} - base=${base%%.*} - tofile=`ls $to/${base}.*cfg 2>/dev/null` - if [ "$tofile" ] && [ -f "$tofile" ]; then - cmp -s $fromfile $tofile || echo cp $fromfile $tofile - fi - done -done | sh -x diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/BooleanIndexDefinition.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/BooleanIndexDefinition.java index efe4d1ade5c..43e1f90790b 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/document/BooleanIndexDefinition.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/BooleanIndexDefinition.java @@ -6,13 +6,12 @@ import java.util.OptionalInt; import java.util.OptionalLong; /** - * Encapsulates values required for native implementation of boolean search. + * Encapsulates values required for predicate fields. * - * @author <a href="mailto:lesters@yahoo-inc.com">Lester Solbakken</a> - * @since 5.2 + * @author lesters */ -public final class BooleanIndexDefinition -{ +public final class BooleanIndexDefinition { + public static final int DEFAULT_ARITY = 8; public static final long DEFAULT_UPPER_BOUND = Long.MAX_VALUE; public static final long DEFAULT_LOWER_BOUND = Long.MIN_VALUE; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/IndexOperation.java b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/IndexOperation.java index 79d99c17973..6df4ca2a6e1 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/IndexOperation.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/IndexOperation.java @@ -15,16 +15,17 @@ import java.util.OptionalInt; import java.util.OptionalLong; /** - * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + * @author Einar M R Rosenvinge */ public class IndexOperation implements FieldOperation { + private String indexName; private Optional<Boolean> prefix = Optional.empty(); private List<String> aliases = new LinkedList<>(); private Optional<String> stemming = Optional.empty(); private Optional<Type> type = Optional.empty(); - private OptionalInt arity = OptionalInt.empty(); // For predicate data type in boolean search + private OptionalInt arity = OptionalInt.empty(); // For predicate data type private OptionalLong lowerBound = OptionalLong.empty(); private OptionalLong upperBound = OptionalLong.empty(); private OptionalDouble densePostingListThreshold = OptionalDouble.empty(); @@ -111,4 +112,5 @@ public class IndexOperation implements FieldOperation { public void setDensePostingListThreshold(double densePostingListThreshold) { this.densePostingListThreshold = OptionalDouble.of(densePostingListThreshold); } + } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/PredicateProcessor.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/PredicateProcessor.java index a4b64ced3c7..450c24d8e3e 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/PredicateProcessor.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/PredicateProcessor.java @@ -73,7 +73,7 @@ public class PredicateProcessor extends Processor { } else if (field.getDataType().getPrimitiveType() == DataType.PREDICATE) { fail(search, field, "Collections of predicates are not allowed."); } else if (field.getDataType() == DataType.RAW && field.doesIndexing()) { - fail(search, field, "Indexing of RAW fields are not supported. If you are using RAW fields for boolean search, use predicate data type instead."); + fail(search, field, "Indexing of RAW fields is not supported."); } else { // if field is not a predicate, disallow predicate-related index parameters for (Index index : field.getIndices().values()) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/NetworkMetrics.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/NetworkMetrics.java index 215e739e350..7f91a19277c 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/NetworkMetrics.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/NetworkMetrics.java @@ -1,4 +1,4 @@ -// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.admin.monitoring; import com.google.common.collect.ImmutableSet; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java index 32a9158dbcf..40e997a0d81 100755 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java @@ -217,6 +217,7 @@ public final class ContainerCluster addSimpleComponent(AccessLog.class); // TODO better modelling addSimpleComponent(ThreadPoolProvider.class); + addSimpleComponent(com.yahoo.concurrent.lock.Locking.class); addSimpleComponent("com.yahoo.jdisc.http.filter.SecurityFilterInvoker"); addSimpleComponent(SIMPLE_LINGUISTICS_PROVIDER); addSimpleComponent("com.yahoo.container.jdisc.SslKeyStoreFactoryProvider"); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/SearchNode.java b/config-model/src/main/java/com/yahoo/vespa/model/search/SearchNode.java index 74e508849e9..66abf3e23cf 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/SearchNode.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/SearchNode.java @@ -262,7 +262,6 @@ public class SearchNode extends AbstractService implements rtcspec(getConnectSpec()). httpport(getHttpPort()). partition(getNodeSpec().partitionId()). - persistenceprovider(new ProtonConfig.Persistenceprovider.Builder().port(getPersistenceProviderRpcPort())). clustername(getClusterName()). basedir(getBaseDir()). tlsspec("tcp/" + getHost().getHostName() + ":" + getTransactionLogServer().getTlsPort()). diff --git a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/RpcConfigSourceClient.java b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/RpcConfigSourceClient.java index b07d1ec681e..d0095121c8e 100644 --- a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/RpcConfigSourceClient.java +++ b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/RpcConfigSourceClient.java @@ -73,29 +73,27 @@ class RpcConfigSourceClient implements ConfigSourceClient { /** * Checks if config sources are available */ - private boolean checkConfigSources() { + private void checkConfigSources() { if (configSourceSet == null || configSourceSet.getSources() == null || configSourceSet.getSources().size() == 0) { log.log(LogLevel.WARNING, "No config sources defined, could not check connection"); - return false; } else { Request req = new Request("ping"); for (String configSource : configSourceSet.getSources()) { Spec spec = new Spec(configSource); Target target = supervisor.connect(spec); - target.invokeSync(req, 10.0); + target.invokeSync(req, 30.0); if (target.isValid()) { log.log(LogLevel.DEBUG, "Created connection to config source at " + spec.toString()); - return true; + return; } else { - log.log(LogLevel.WARNING, "Could not connect to config source at " + spec.toString()); + log.log(LogLevel.INFO, "Could not connect to config source at " + spec.toString()); } target.close(); } String extra = ""; - log.log(LogLevel.WARNING, "Could not connect to any config source in set " + configSourceSet.toString() + + log.log(LogLevel.INFO, "Could not connect to any config source in set " + configSourceSet.toString() + ", please make sure config server(s) are running. " + extra); } - return false; } /** diff --git a/configdefinitions/src/main/java/com/yahoo/vespa/orchestrator/config/package-info.java b/configdefinitions/src/main/java/com/yahoo/vespa/orchestrator/config/package-info.java new file mode 100644 index 00000000000..f6ee478cc9f --- /dev/null +++ b/configdefinitions/src/main/java/com/yahoo/vespa/orchestrator/config/package-info.java @@ -0,0 +1,6 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.vespa.orchestrator.config; + +import com.yahoo.api.annotations.PublicApi; +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/configdefinitions/src/vespa/CMakeLists.txt b/configdefinitions/src/vespa/CMakeLists.txt index 15819b6bd47..016739f4594 100644 --- a/configdefinitions/src/vespa/CMakeLists.txt +++ b/configdefinitions/src/vespa/CMakeLists.txt @@ -5,62 +5,66 @@ vespa_add_library(configdefinitions DEPENDS ) vespa_generate_config(configdefinitions application-id.def) -install(FILES application-id.def DESTINATION var/db/vespa/config_server/serverdb/classes) +install(FILES application-id.def RENAME cloud.config.application-id.def DESTINATION var/db/vespa/config_server/serverdb/classes) vespa_generate_config(configdefinitions attributes.def) -install(FILES attributes.def DESTINATION var/db/vespa/config_server/serverdb/classes) +install(FILES attributes.def RENAME vespa.config.search.attributes.def DESTINATION var/db/vespa/config_server/serverdb/classes) vespa_generate_config(configdefinitions cluster-info.def) -install(FILES cluster-info.def DESTINATION var/db/vespa/config_server/serverdb/classes) +install(FILES cluster-info.def RENAME cloud.config.cluster-info.def DESTINATION var/db/vespa/config_server/serverdb/classes) vespa_generate_config(configdefinitions cluster-list.def) -install(FILES cluster-list.def DESTINATION var/db/vespa/config_server/serverdb/classes) +install(FILES cluster-list.def RENAME cloud.config.cluster-list.def DESTINATION var/db/vespa/config_server/serverdb/classes) vespa_generate_config(configdefinitions configserver.def) -install(FILES configserver.def DESTINATION var/db/vespa/config_server/serverdb/classes) +install(FILES configserver.def RENAME cloud.config.configserver.def DESTINATION var/db/vespa/config_server/serverdb/classes) vespa_generate_config(configdefinitions dispatch.def) -install(FILES dispatch.def DESTINATION var/db/vespa/config_server/serverdb/classes) +install(FILES dispatch.def RENAME vespa.config.search.dispatch.def DESTINATION var/db/vespa/config_server/serverdb/classes) vespa_generate_config(configdefinitions fleetcontroller.def) -install(FILES fleetcontroller.def DESTINATION var/db/vespa/config_server/serverdb/classes) +install(FILES fleetcontroller.def RENAME vespa.config.content.fleetcontroller.def DESTINATION var/db/vespa/config_server/serverdb/classes) vespa_generate_config(configdefinitions ilscripts.def) -install(FILES ilscripts.def DESTINATION var/db/vespa/config_server/serverdb/classes) +install(FILES ilscripts.def RENAME vespa.configdefinition.ilscripts.def DESTINATION var/db/vespa/config_server/serverdb/classes) vespa_generate_config(configdefinitions imported-fields.def) -install(FILES imported-fields.def DESTINATION var/db/vespa/config_server/serverdb/classes) +install(FILES imported-fields.def RENAME vespa.config.search.imported-fields.def DESTINATION var/db/vespa/config_server/serverdb/classes) vespa_generate_config(configdefinitions indexschema.def) -install(FILES indexschema.def DESTINATION var/db/vespa/config_server/serverdb/classes) +install(FILES indexschema.def RENAME vespa.config.search.indexschema.def DESTINATION var/db/vespa/config_server/serverdb/classes) vespa_generate_config(configdefinitions lb-services.def) -install(FILES lb-services.def DESTINATION var/db/vespa/config_server/serverdb/classes) +install(FILES lb-services.def RENAME cloud.config.lb-services.def DESTINATION var/db/vespa/config_server/serverdb/classes) vespa_generate_config(configdefinitions load-type.def) -install(FILES load-type.def DESTINATION var/db/vespa/config_server/serverdb/classes) +install(FILES load-type.def RENAME vespa.config.content.load-type.def DESTINATION var/db/vespa/config_server/serverdb/classes) vespa_generate_config(configdefinitions messagetyperouteselectorpolicy.def) -install(FILES messagetyperouteselectorpolicy.def DESTINATION var/db/vespa/config_server/serverdb/classes) +install(FILES messagetyperouteselectorpolicy.def RENAME vespa.config.content.messagetyperouteselectorpolicy.def DESTINATION var/db/vespa/config_server/serverdb/classes) vespa_generate_config(configdefinitions model.def) -install(FILES model.def DESTINATION var/db/vespa/config_server/serverdb/classes) +install(FILES model.def RENAME cloud.config.model.def DESTINATION var/db/vespa/config_server/serverdb/classes) +vespa_generate_config(configdefinitions orchestrator.def) +install(FILES orchestrator.def RENAME vespa.orchestrator.config.orchestrator.def DESTINATION var/db/vespa/config_server/serverdb/classes) vespa_generate_config(configdefinitions persistence.def) -install(FILES persistence.def DESTINATION var/db/vespa/config_server/serverdb/classes) +install(FILES persistence.def RENAME vespa.config.content.persistence.def DESTINATION var/db/vespa/config_server/serverdb/classes) vespa_generate_config(configdefinitions rank-profiles.def) -install(FILES rank-profiles.def DESTINATION var/db/vespa/config_server/serverdb/classes) +install(FILES rank-profiles.def RENAME vespa.config.search.rank-profiles.def DESTINATION var/db/vespa/config_server/serverdb/classes) +vespa_generate_config(configdefinitions routing.def) +install(FILES routing.def RENAME cloud.config.routing.def DESTINATION var/db/vespa/config_server/serverdb/classes) vespa_generate_config(configdefinitions routing-provider.def) -install(FILES routing-provider.def DESTINATION var/db/vespa/config_server/serverdb/classes) +install(FILES routing-provider.def RENAME cloud.config.routing-provider.def DESTINATION var/db/vespa/config_server/serverdb/classes) vespa_generate_config(configdefinitions sentinel.def) -install(FILES sentinel.def DESTINATION var/db/vespa/config_server/serverdb/classes) +install(FILES sentinel.def RENAME cloud.config.sentinel.def DESTINATION var/db/vespa/config_server/serverdb/classes) vespa_generate_config(configdefinitions slobroks.def) -install(FILES slobroks.def DESTINATION var/db/vespa/config_server/serverdb/classes) +install(FILES slobroks.def RENAME cloud.config.slobroks.def DESTINATION var/db/vespa/config_server/serverdb/classes) vespa_generate_config(configdefinitions specialtokens.def) -install(FILES specialtokens.def DESTINATION var/db/vespa/config_server/serverdb/classes) +install(FILES specialtokens.def RENAME vespa.configdefinition.specialtokens.def DESTINATION var/db/vespa/config_server/serverdb/classes) vespa_generate_config(configdefinitions stor-devices.def) -install(FILES stor-devices.def DESTINATION var/db/vespa/config_server/serverdb/classes) +install(FILES stor-devices.def RENAME vespa.config.storage.stor-devices.def DESTINATION var/db/vespa/config_server/serverdb/classes) vespa_generate_config(configdefinitions stor-distribution.def) -install(FILES stor-distribution.def DESTINATION var/db/vespa/config_server/serverdb/classes) +install(FILES stor-distribution.def RENAME vespa.config.content.stor-distribution.def DESTINATION var/db/vespa/config_server/serverdb/classes) vespa_generate_config(configdefinitions stor-filestor.def) -install(FILES stor-filestor.def DESTINATION var/db/vespa/config_server/serverdb/classes) +install(FILES stor-filestor.def RENAME vespa.config.content.stor-filestor.def DESTINATION var/db/vespa/config_server/serverdb/classes) vespa_generate_config(configdefinitions stor-memfilepersistence.def) -install(FILES stor-memfilepersistence.def DESTINATION var/db/vespa/config_server/serverdb/classes) +install(FILES stor-memfilepersistence.def RENAME vespa.config.storage.stor-memfilepersistence.def DESTINATION var/db/vespa/config_server/serverdb/classes) vespa_generate_config(configdefinitions summary.def) -install(FILES summary.def DESTINATION var/db/vespa/config_server/serverdb/classes) +install(FILES summary.def RENAME vespa.config.search.summary.def DESTINATION var/db/vespa/config_server/serverdb/classes) vespa_generate_config(configdefinitions summarymap.def) -install(FILES summarymap.def DESTINATION var/db/vespa/config_server/serverdb/classes) +install(FILES summarymap.def RENAME vespa.config.search.summarymap.def DESTINATION var/db/vespa/config_server/serverdb/classes) vespa_generate_config(configdefinitions upgrading.def) -install(FILES upgrading.def DESTINATION var/db/vespa/config_server/serverdb/classes) +install(FILES upgrading.def RENAME vespa.config.content.upgrading.def DESTINATION var/db/vespa/config_server/serverdb/classes) vespa_generate_config(configdefinitions ymon.def) -install(FILES ymon.def DESTINATION var/db/vespa/config_server/serverdb/classes) +install(FILES ymon.def RENAME cloud.config.ymon.def DESTINATION var/db/vespa/config_server/serverdb/classes) vespa_generate_config(configdefinitions zookeeper-server.def) -install(FILES zookeeper-server.def DESTINATION var/db/vespa/config_server/serverdb/classes) +install(FILES zookeeper-server.def RENAME cloud.config.zookeeper-server.def DESTINATION var/db/vespa/config_server/serverdb/classes) vespa_generate_config(configdefinitions zookeepers.def) -install(FILES zookeepers.def DESTINATION var/db/vespa/config_server/serverdb/classes) +install(FILES zookeepers.def RENAME cloud.config.zookeepers.def DESTINATION var/db/vespa/config_server/serverdb/classes) diff --git a/orchestrator/src/main/resources/configdefinitions/orchestrator.def b/configdefinitions/src/vespa/orchestrator.def index a546f8e8a1c..ab52c777398 100644 --- a/orchestrator/src/main/resources/configdefinitions/orchestrator.def +++ b/configdefinitions/src/vespa/orchestrator.def @@ -1,5 +1,5 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -namespace=vespa.orchestrator +namespace=vespa.orchestrator.config # TODO: Change the default to actual latency in real setup. serviceMonitorConvergenceLatencySeconds int default=0 diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationListener.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationListener.java deleted file mode 100644 index bea25183eef..00000000000 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationListener.java +++ /dev/null @@ -1,24 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -package com.yahoo.vespa.config.server; - -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.TenantName; -import com.yahoo.vespa.config.server.application.ApplicationSet; - -public interface ApplicationListener { - /** - * Configs has been activated for an application: Either an application - * has been deployed for the first time, or it has been externally or internally redeployed. - * - * Must be thread-safe. - */ - void configActivated(TenantName tenant, ApplicationSet application); - - /** - * Application has been removed. - * - * Must be thread-safe. - */ - void applicationRemoved(ApplicationId applicationId); -} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java index 61458d871ac..f49c32b061a 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java @@ -26,6 +26,7 @@ import com.yahoo.vespa.config.server.application.LogServerLogGrabber; import com.yahoo.vespa.config.server.application.TenantApplications; import com.yahoo.vespa.config.server.configchange.ConfigChangeActions; import com.yahoo.vespa.config.server.deploy.Deployment; +import com.yahoo.vespa.config.server.http.SimpleHttpFetcher; import com.yahoo.vespa.config.server.provision.HostProvisionerProvider; import com.yahoo.vespa.config.server.session.LocalSession; import com.yahoo.vespa.config.server.session.LocalSessionRepo; @@ -81,13 +82,35 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye ApplicationConvergenceChecker applicationConvergenceChecker, HttpProxy httpProxy, ConfigserverConfig configserverConfig) { + this(tenants, hostProvisionerProvider.getHostProvisioner(), curator, logServerLogGrabber, + applicationConvergenceChecker, httpProxy, configserverConfig, Clock.systemUTC()); + } + + // For testing + public ApplicationRepository(Tenants tenants, + Provisioner hostProvisioner, + Curator curator, + Clock clock) { + this(tenants, Optional.of(hostProvisioner), curator, new LogServerLogGrabber(), + new ApplicationConvergenceChecker(), new HttpProxy(new SimpleHttpFetcher()), + new ConfigserverConfig(new ConfigserverConfig.Builder()), clock); + } + + private ApplicationRepository(Tenants tenants, + Optional<Provisioner> hostProvisioner, + Curator curator, + LogServerLogGrabber logServerLogGrabber, + ApplicationConvergenceChecker applicationConvergenceChecker, + HttpProxy httpProxy, + ConfigserverConfig configserverConfig, + Clock clock) { this.tenants = tenants; - this.hostProvisioner = hostProvisionerProvider.getHostProvisioner(); + this.hostProvisioner = hostProvisioner; this.curator = curator; this.logServerLogGrabber = logServerLogGrabber; this.convergeChecker = applicationConvergenceChecker; this.httpProxy = httpProxy; - this.clock = Clock.systemUTC(); + this.clock = clock; this.configserverConfig = configserverConfig; this.environment = Environment.from(configserverConfig.environment()); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerDB.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerDB.java index 29381404562..7e6327075ac 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerDB.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerDB.java @@ -42,7 +42,6 @@ public class ConfigServerDB { return new ConfigServerDB(new ConfigserverConfig(new ConfigserverConfig.Builder().configServerDBDir(dir))); } - public File dest() { return new File(serverDB, "configs"); } public File classes() { return new File(serverDB, "classes"); } public File vespaapps() { return new File(serverDB, "vespaapps"); } public File serverdefs() { return new File(serverDB, "serverdefs"); } @@ -52,7 +51,6 @@ public class ConfigServerDB { * Creates all the config server db's dirs that are global. */ public void create() { - cr(dest()); cr(classes()); cr(vespaapps()); cr(serverdefs()); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/GlobalComponentRegistry.java b/configserver/src/main/java/com/yahoo/vespa/config/server/GlobalComponentRegistry.java index 4f26cfa265b..b6e8f013fd1 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/GlobalComponentRegistry.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/GlobalComponentRegistry.java @@ -14,6 +14,7 @@ import com.yahoo.vespa.config.server.tenant.TenantListener; import com.yahoo.vespa.config.server.zookeeper.ConfigCurator; import com.yahoo.vespa.curator.Curator; +import java.time.Clock; import java.util.Optional; /** @@ -38,5 +39,6 @@ public interface GlobalComponentRegistry { ModelFactoryRegistry getModelFactoryRegistry(); Optional<Provisioner> getHostProvisioner(); Zone getZone(); + Clock getClock(); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/InjectedGlobalComponentRegistry.java b/configserver/src/main/java/com/yahoo/vespa/config/server/InjectedGlobalComponentRegistry.java index fa5224732f6..d0b830aceaa 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/InjectedGlobalComponentRegistry.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/InjectedGlobalComponentRegistry.java @@ -17,6 +17,7 @@ import com.yahoo.vespa.config.server.tenant.TenantListener; import com.yahoo.vespa.config.server.zookeeper.ConfigCurator; import com.yahoo.vespa.curator.Curator; +import java.time.Clock; import java.util.Optional; /** @@ -108,4 +109,7 @@ public class InjectedGlobalComponentRegistry implements GlobalComponentRegistry public Zone getZone() { return zone; } + + @Override + public Clock getClock() {return Clock.systemUTC();} } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ReloadListener.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ReloadListener.java index e392472480b..71c6d311172 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/ReloadListener.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ReloadListener.java @@ -15,7 +15,7 @@ import java.util.Collection; * @author lulf * @since 5.1 */ -public interface ReloadListener extends ApplicationListener { +public interface ReloadListener { /** * Signal the listener that hosts used by by a particular tenant. * @@ -32,4 +32,19 @@ public interface ReloadListener extends ApplicationListener { * @throws java.lang.IllegalArgumentException if one or more of the hosts are in use by another tenant. */ void verifyHostsAreAvailable(TenantName tenant, Collection<String> newHosts); + + /** + * Configs has been activated for an application: Either an application + * has been deployed for the first time, or it has been externally or internally redeployed. + * + * Must be thread-safe. + */ + void configActivated(TenantName tenant, ApplicationSet application); + + /** + * Application has been removed. + * + * Must be thread-safe. + */ + void applicationRemoved(ApplicationId applicationId); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelManager.java b/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelManager.java index c3fd417f434..06f7ffa66c4 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelManager.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelManager.java @@ -4,6 +4,9 @@ package com.yahoo.vespa.config.server; import com.google.inject.Inject; import com.yahoo.cloud.config.ConfigserverConfig; +import com.yahoo.config.model.api.ApplicationInfo; +import com.yahoo.config.model.api.SuperModelListener; +import com.yahoo.config.model.api.SuperModelProvider; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.NodeFlavors; import com.yahoo.config.provision.TenantName; @@ -14,13 +17,19 @@ import com.yahoo.vespa.config.server.application.ApplicationSet; import com.yahoo.vespa.config.server.model.SuperModel; import java.time.Instant; -import java.util.*; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.stream.Collectors; /** * Provides a SuperModel - a model of all application instances, and makes it stays * up to date as applications are added, redeployed, and removed. */ -public class SuperModelManager implements SuperModelProvider, ApplicationListener { +public class SuperModelManager implements SuperModelProvider { private final Zone zone; private SuperModel superModel; // Guarded by 'this' monitor private final List<SuperModelListener> listeners = new ArrayList<>(); // Guarded by 'this' monitor @@ -40,22 +49,23 @@ public class SuperModelManager implements SuperModelProvider, ApplicationListene makeNewSuperModel(new HashMap<>()); } - @Override public synchronized SuperModel getSuperModel() { return superModel; } - @Override public synchronized long getGeneration() { return generation; } @Override - public synchronized void registerListener(SuperModelListener listener) { + public synchronized List<ApplicationInfo> snapshot(SuperModelListener listener) { listeners.add(listener); + return superModel.applicationModels().values().stream() + .flatMap(applications -> applications.values().stream()) + .map(Application::toApplicationInfo) + .collect(Collectors.toList()); } - @Override public synchronized void configActivated(TenantName tenant, ApplicationSet applicationSet) { Map<TenantName, Map<ApplicationId, Application>> newModels = createModelCopy(); if (!newModels.containsKey(tenant)) { @@ -70,10 +80,9 @@ public class SuperModelManager implements SuperModelProvider, ApplicationListene newModels.get(tenant).put(applicationSet.getId(), application); makeNewSuperModel(newModels); - listeners.stream().forEach(listener -> listener.applicationActivated(superModel, application)); + listeners.stream().forEach(listener -> listener.applicationActivated(application.toApplicationInfo())); } - @Override public synchronized void applicationRemoved(ApplicationId applicationId) { Map<TenantName, Map<ApplicationId, Application>> newModels = createModelCopy(); if (newModels.containsKey(applicationId.tenant())) { @@ -84,7 +93,7 @@ public class SuperModelManager implements SuperModelProvider, ApplicationListene } makeNewSuperModel(newModels); - listeners.stream().forEach(listener -> listener.applicationRemoved(superModel, applicationId)); + listeners.stream().forEach(listener -> listener.applicationRemoved(applicationId)); } private void makeNewSuperModel(Map<TenantName, Map<ApplicationId, Application>> newModels) { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelProvider.java b/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelProvider.java deleted file mode 100644 index 81eb823c026..00000000000 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelProvider.java +++ /dev/null @@ -1,12 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -package com.yahoo.vespa.config.server; - -import com.yahoo.vespa.config.server.model.SuperModel; - -interface SuperModelProvider { - SuperModel getSuperModel(); - long getGeneration(); - - void registerListener(SuperModelListener listener); -} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/application/Application.java b/configserver/src/main/java/com/yahoo/vespa/config/server/application/Application.java index 369bce70660..f8f11416811 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/application/Application.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/application/Application.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.config.server.application; import com.yahoo.config.ConfigurationRuntimeException; +import com.yahoo.config.model.api.ApplicationInfo; import com.yahoo.config.model.api.Model; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Version; @@ -74,6 +75,10 @@ public class Application implements ModelResult { return sb.toString(); } + public ApplicationInfo toApplicationInfo() { + return new ApplicationInfo(app, appGeneration, model); + } + public ServerCache getCache() { return cache; } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/application/ZKTenantApplications.java b/configserver/src/main/java/com/yahoo/vespa/config/server/application/ZKTenantApplications.java index 5260dd9228c..e651fd6c97c 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/application/ZKTenantApplications.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/application/ZKTenantApplications.java @@ -26,8 +26,8 @@ import java.util.logging.Logger; /** * The applications of a tenant, backed by ZooKeeper. - * Each application is stored as a single file, named the same as the application id and containing the id - * of the session storing the content of the application. + * Each application is stored as a single node under /config/v2/tenants/<tenant>/applications/<applications>, + * named the same as the application id and containing the id of the session storing the content of the application. * * @author Ulf Lilleengen */ @@ -36,26 +36,27 @@ public class ZKTenantApplications implements TenantApplications, PathChildrenCac private static final Logger log = Logger.getLogger(ZKTenantApplications.class.getName()); private final Curator curator; - private final Path tenantRoot; - private final ExecutorService pathChildrenExecutor = Executors.newFixedThreadPool(1, ThreadFactoryFactory.getThreadFactory(ZKTenantApplications.class.getName())); + private final Path applicationsPath; + private final ExecutorService pathChildrenExecutor = + Executors.newFixedThreadPool(1, ThreadFactoryFactory.getThreadFactory(ZKTenantApplications.class.getName())); private final Curator.DirectoryCache directoryCache; private final ReloadHandler reloadHandler; private final TenantName tenant; - private ZKTenantApplications(Curator curator, Path tenantRoot, ReloadHandler reloadHandler, TenantName tenant) { + private ZKTenantApplications(Curator curator, Path applicationsPath, ReloadHandler reloadHandler, TenantName tenant) { this.curator = curator; - this.tenantRoot = tenantRoot; + this.applicationsPath = applicationsPath; this.reloadHandler = reloadHandler; this.tenant = tenant; rewriteApplicationIds(); - this.directoryCache = curator.createDirectoryCache(tenantRoot.getAbsolute(), false, false, pathChildrenExecutor); + this.directoryCache = curator.createDirectoryCache(applicationsPath.getAbsolute(), false, false, pathChildrenExecutor); this.directoryCache.start(); this.directoryCache.addListener(this); } - public static TenantApplications create(Curator curator, Path root, ReloadHandler reloadHandler, TenantName tenant) { + public static TenantApplications create(Curator curator, Path applicationsPath, ReloadHandler reloadHandler, TenantName tenant) { try { - return new ZKTenantApplications(curator, root, reloadHandler, tenant); + return new ZKTenantApplications(curator, applicationsPath, reloadHandler, tenant); } catch (Exception e) { throw new RuntimeException(Tenants.logPre(tenant) + "Error creating application repo", e); } @@ -63,7 +64,7 @@ public class ZKTenantApplications implements TenantApplications, PathChildrenCac private void rewriteApplicationIds() { try { - List<String> appNodes = curator.framework().getChildren().forPath(tenantRoot.getAbsolute()); + List<String> appNodes = curator.framework().getChildren().forPath(applicationsPath.getAbsolute()); for (String appNode : appNodes) { Optional<ApplicationId> appId = parseApplication(appNode); appId.filter(id -> shouldBeRewritten(appNode, id)) @@ -75,7 +76,7 @@ public class ZKTenantApplications implements TenantApplications, PathChildrenCac } private long readSessionId(ApplicationId appId, String appNode) { - String path = tenantRoot.append(appNode).getAbsolute(); + String path = applicationsPath.append(appNode).getAbsolute(); try { return Long.parseLong(Utf8.toString(curator.framework().getData().forPath(path))); } catch (Exception e) { @@ -88,8 +89,8 @@ public class ZKTenantApplications implements TenantApplications, PathChildrenCac } private void rewriteApplicationId(ApplicationId appId, String origNode, long sessionId) { - String newPath = tenantRoot.append(appId.serializedForm()).getAbsolute(); - String oldPath = tenantRoot.append(origNode).getAbsolute(); + String newPath = applicationsPath.append(appId.serializedForm()).getAbsolute(); + String oldPath = applicationsPath.append(origNode).getAbsolute(); try (CuratorTransaction transaction = new CuratorTransaction(curator)) { if (curator.framework().checkExists().forPath(newPath) == null) { transaction.add(CuratorOperations.create(newPath, Utf8.toAsciiBytes(sessionId))); @@ -104,7 +105,7 @@ public class ZKTenantApplications implements TenantApplications, PathChildrenCac @Override public List<ApplicationId> listApplications() { try { - List<String> appNodes = curator.framework().getChildren().forPath(tenantRoot.getAbsolute()); + List<String> appNodes = curator.framework().getChildren().forPath(applicationsPath.getAbsolute()); List<ApplicationId> applicationIds = new ArrayList<>(); for (String appNode : appNodes) { parseApplication(appNode).ifPresent(applicationIds::add); @@ -127,9 +128,9 @@ public class ZKTenantApplications implements TenantApplications, PathChildrenCac @Override public Transaction createPutApplicationTransaction(ApplicationId applicationId, long sessionId) { if (listApplications().contains(applicationId)) { - return new CuratorTransaction(curator).add(CuratorOperations.setData(tenantRoot.append(applicationId.serializedForm()).getAbsolute(), Utf8.toAsciiBytes(sessionId))); + return new CuratorTransaction(curator).add(CuratorOperations.setData(applicationsPath.append(applicationId.serializedForm()).getAbsolute(), Utf8.toAsciiBytes(sessionId))); } else { - return new CuratorTransaction(curator).add(CuratorOperations.create(tenantRoot.append(applicationId.serializedForm()).getAbsolute(), Utf8.toAsciiBytes(sessionId))); + return new CuratorTransaction(curator).add(CuratorOperations.create(applicationsPath.append(applicationId.serializedForm()).getAbsolute(), Utf8.toAsciiBytes(sessionId))); } } @@ -140,7 +141,7 @@ public class ZKTenantApplications implements TenantApplications, PathChildrenCac @Override public CuratorTransaction deleteApplication(ApplicationId applicationId) { - Path path = tenantRoot.append(applicationId.serializedForm()); + Path path = applicationsPath.append(applicationId.serializedForm()); return CuratorTransaction.from(CuratorOperations.delete(path.getAbsolute()), curator); } @@ -150,13 +151,13 @@ public class ZKTenantApplications implements TenantApplications, PathChildrenCac pathChildrenExecutor.shutdown(); } - @Override public void childEvent(CuratorFramework client, PathChildrenCacheEvent event) throws Exception { switch (event.getType()) { case CHILD_ADDED: applicationAdded(ApplicationId.fromSerializedForm(Path.fromString(event.getData().getPath()).getName())); break; + // Event CHILD_REMOVED will be triggered on all config servers if deleteApplication() above is called on one of them case CHILD_REMOVED: applicationRemoved(ApplicationId.fromSerializedForm(Path.fromString(event.getData().getPath()).getName())); break; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/LocalSessionRepo.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/LocalSessionRepo.java index ae15be83062..d2270ddddfd 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/LocalSessionRepo.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/LocalSessionRepo.java @@ -23,22 +23,30 @@ public class LocalSessionRepo extends SessionRepo<LocalSession> { private static final Logger log = Logger.getLogger(LocalSessionRepo.class.getName()); - private final static FilenameFilter sessionApplicationsFilter = new FilenameFilter() { - @Override - public boolean accept(File dir, String name) { - return name.matches("\\d+"); - } - }; + private final static FilenameFilter sessionApplicationsFilter = (dir, name) -> name.matches("\\d+"); private final long sessionLifetime; // in seconds private final TenantApplications applicationRepo; private final Clock clock; - public LocalSessionRepo(TenantFileSystemDirs tenantFileSystemDirs, LocalSessionLoader loader, TenantApplications applicationRepo, Clock clock, long sessionLifeTime) { + public LocalSessionRepo(TenantFileSystemDirs tenantFileSystemDirs, LocalSessionLoader loader, + TenantApplications applicationRepo, Clock clock, long sessionLifeTime) { this(applicationRepo, clock, sessionLifeTime); loadSessions(tenantFileSystemDirs.path(), loader); } + // Constructor public only for testing + public LocalSessionRepo(TenantApplications applicationRepo, Clock clock) { + this(applicationRepo, clock, TimeUnit.DAYS.toMillis(1)); + } + + // Constructor public only for testing + public LocalSessionRepo(TenantApplications applicationRepo, Clock clock, long sessionLifetime) { + this.applicationRepo = applicationRepo; + this.sessionLifetime = sessionLifetime; + this.clock = clock; + } + private void loadSessions(File applicationsDir, LocalSessionLoader loader) { File[] applications = applicationsDir.listFiles(sessionApplicationsFilter); if (applications == null) { @@ -48,7 +56,8 @@ public class LocalSessionRepo extends SessionRepo<LocalSession> { try { addSession(loader.loadSession(Long.parseLong(application.getName()))); } catch (IllegalArgumentException e) { - log.log(LogLevel.WARNING, "Could not load application '" + application.getAbsolutePath() + "':" + e.getMessage() + ", skipping it."); + log.log(LogLevel.WARNING, "Could not load application '" + + application.getAbsolutePath() + "':" + e.getMessage() + ", skipping it."); } } } @@ -66,17 +75,6 @@ public class LocalSessionRepo extends SessionRepo<LocalSession> { return null; } - // Constructor only for testing - public LocalSessionRepo(TenantApplications applicationRepo, Clock clock, long sessionLifetime) { - this.applicationRepo = applicationRepo; - this.sessionLifetime = sessionLifetime; - this.clock = clock; - } - - public LocalSessionRepo(TenantApplications applicationRepo) { - this(applicationRepo, Clock.systemUTC(), TimeUnit.DAYS.toMillis(1)); - } - @Override public synchronized void addSession(LocalSession session) { purgeOldSessions(); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionFactoryImpl.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionFactoryImpl.java index c2655d1a1b3..1d5025f2e61 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionFactoryImpl.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionFactoryImpl.java @@ -22,6 +22,7 @@ import com.yahoo.vespa.config.server.zookeeper.ConfigCurator; import com.yahoo.vespa.curator.Curator; import java.io.File; +import java.time.Clock; import java.util.List; import java.util.Optional; import java.util.concurrent.TimeUnit; @@ -52,12 +53,15 @@ public class SessionFactoryImpl implements SessionFactory, LocalSessionLoader { private final TenantName tenant; private final String serverId; private final Optional<NodeFlavors> nodeFlavors; + private final Clock clock; public SessionFactoryImpl(GlobalComponentRegistry globalComponentRegistry, SessionCounter sessionCounter, Path sessionsPath, TenantApplications applicationRepo, - TenantFileSystemDirs tenantFileSystemDirs, HostValidator<ApplicationId> hostRegistry, TenantName tenant) { + TenantFileSystemDirs tenantFileSystemDirs, + HostValidator<ApplicationId> hostRegistry, + TenantName tenant) { this.hostRegistry = hostRegistry; this.tenant = tenant; this.sessionPreparer = globalComponentRegistry.getSessionPreparer(); @@ -71,6 +75,7 @@ public class SessionFactoryImpl implements SessionFactory, LocalSessionLoader { this.defRepo = globalComponentRegistry.getConfigDefinitionRepo(); this.serverId = globalComponentRegistry.getConfigserverConfig().serverId(); this.nodeFlavors = globalComponentRegistry.getZone().nodeFlavors(); + this.clock = globalComponentRegistry.getClock(); } @Override @@ -100,9 +105,11 @@ public class SessionFactoryImpl implements SessionFactory, LocalSessionLoader { private LocalSession createSessionFromApplication(ApplicationPackage applicationPackage, long sessionId, - SessionZooKeeperClient sessionZKClient, TimeoutBudget timeoutBudget) { + SessionZooKeeperClient sessionZKClient, + TimeoutBudget timeoutBudget, + Clock clock) { log.log(LogLevel.DEBUG, Tenants.logPre(tenant) + "Creating session " + sessionId + " in ZooKeeper"); - sessionZKClient.createNewSession(System.currentTimeMillis(), TimeUnit.MILLISECONDS); + sessionZKClient.createNewSession(clock.instant().toEpochMilli(), TimeUnit.MILLISECONDS); log.log(LogLevel.DEBUG, Tenants.logPre(tenant) + "Creating upload waiter for session " + sessionId); Curator.CompletionWaiter waiter = sessionZKClient.getUploadWaiter(); log.log(LogLevel.DEBUG, Tenants.logPre(tenant) + "Done creating upload waiter for session " + sessionId); @@ -145,7 +152,7 @@ public class SessionFactoryImpl implements SessionFactory, LocalSessionLoader { IOUtils.copyDirectory(applicationFile, userApplicationDir); ApplicationPackage applicationPackage = createApplication(applicationFile, userApplicationDir, applicationName, sessionId, currentlyActiveSession); applicationPackage.writeMetaData(); - return createSessionFromApplication(applicationPackage, sessionId, sessionZooKeeperClient, timeoutBudget); + return createSessionFromApplication(applicationPackage, sessionId, sessionZooKeeperClient, timeoutBudget, clock); } catch (Exception e) { throw new RuntimeException("Error creating session " + sessionIdPath, e); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClient.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClient.java index 8f056898374..6a90eaacb02 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClient.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClient.java @@ -27,8 +27,8 @@ import java.util.Optional; import java.util.concurrent.TimeUnit; /** - * Zookeeper client for a specific session. Can be used to read and write session status - * and create and get prepare and active barrier. + * Zookeeper client for a specific session. Path for a session is /config/v2/tenants/<tenant>/sessions/<sessionid> + * Can be used to read and write session status and create and get prepare and active barrier. * * @author Ulf Lilleengen */ @@ -39,34 +39,34 @@ public class SessionZooKeeperClient { // NOTE: Any state added here MUST also be propagated in com.yahoo.vespa.config.server.deploy.Deployment.prepare() static final String APPLICATION_ID_PATH = "applicationId"; - static final String VERSION_PATH = "version"; - static final String CREATE_TIME_PATH = "createTime"; + private static final String VERSION_PATH = "version"; + private static final String CREATE_TIME_PATH = "createTime"; private final Curator curator; private final ConfigCurator configCurator; - private final Path rootPath; + private final Path sessionPath; private final Path sessionStatusPath; private final String serverId; private final ServerCacheLoader cacheLoader; private final Optional<NodeFlavors> nodeFlavors; // Only for testing when cache loader does not need cache entries. - public SessionZooKeeperClient(Curator curator, Path rootPath) { - this(curator, ConfigCurator.create(curator), rootPath, new StaticConfigDefinitionRepo(), "", Optional.empty()); + public SessionZooKeeperClient(Curator curator, Path sessionPath) { + this(curator, ConfigCurator.create(curator), sessionPath, new StaticConfigDefinitionRepo(), "", Optional.empty()); } public SessionZooKeeperClient(Curator curator, ConfigCurator configCurator, - Path rootPath, + Path sessionPath, ConfigDefinitionRepo definitionRepo, String serverId, Optional<NodeFlavors> nodeFlavors) { this.curator = curator; this.configCurator = configCurator; - this.rootPath = rootPath; + this.sessionPath = sessionPath; this.serverId = serverId; this.nodeFlavors = nodeFlavors; - this.sessionStatusPath = rootPath.append(ConfigCurator.SESSIONSTATE_ZK_SUBPATH); - this.cacheLoader = new ServerCacheLoader(configCurator, rootPath, definitionRepo); + this.sessionStatusPath = sessionPath.append(ConfigCurator.SESSIONSTATE_ZK_SUBPATH); + this.cacheLoader = new ServerCacheLoader(configCurator, sessionPath, definitionRepo); } public void writeStatus(Session.Status sessionStatus) { @@ -112,7 +112,7 @@ public class SessionZooKeeperClient { private static final String UPLOAD_BARRIER = "uploadBarrier"; private Path getWaiterPath(String barrierName) { - return rootPath.append(barrierName); + return sessionPath.append(barrierName); } /** Returns the number of node members needed in a barrier */ @@ -121,7 +121,7 @@ public class SessionZooKeeperClient { } private Curator.CompletionWaiter createCompletionWaiter(String waiterNode) { - return curator.createCompletionWaiter(rootPath, waiterNode, getNumberOfMembers(), serverId); + return curator.createCompletionWaiter(sessionPath, waiterNode, getNumberOfMembers(), serverId); } private Curator.CompletionWaiter getCompletionWaiter(Path path) { @@ -130,20 +130,20 @@ public class SessionZooKeeperClient { public void delete() { try { - log.log(LogLevel.DEBUG, "Deleting " + rootPath.getAbsolute()); - configCurator.deleteRecurse(rootPath.getAbsolute()); + log.log(LogLevel.DEBUG, "Deleting " + sessionPath.getAbsolute()); + configCurator.deleteRecurse(sessionPath.getAbsolute()); } catch (RuntimeException e) { - log.log(LogLevel.INFO, "Error deleting session (" + rootPath.getAbsolute() + ") from zookeeper"); + log.log(LogLevel.INFO, "Error deleting session (" + sessionPath.getAbsolute() + ") from zookeeper"); } } /** Returns a transaction deleting this session on commit */ public CuratorTransaction deleteTransaction() { - return CuratorTransaction.from(CuratorOperations.deleteAll(rootPath.getAbsolute(), curator), curator); + return CuratorTransaction.from(CuratorOperations.deleteAll(sessionPath.getAbsolute(), curator), curator); } public ApplicationPackage loadApplicationPackage() { - return new ZKApplicationPackage(configCurator, rootPath, nodeFlavors); + return new ZKApplicationPackage(configCurator, sessionPath, nodeFlavors); } public ServerCache loadServerCache() { @@ -151,7 +151,7 @@ public class SessionZooKeeperClient { } private String applicationIdPath() { - return rootPath.append(APPLICATION_ID_PATH).getAbsolute(); + return sessionPath.append(APPLICATION_ID_PATH).getAbsolute(); } public void writeApplicationId(ApplicationId id) { @@ -164,7 +164,7 @@ public class SessionZooKeeperClient { } private String versionPath() { - return rootPath.append(VERSION_PATH).getAbsolute(); + return sessionPath.append(VERSION_PATH).getAbsolute(); } public void writeVespaVersion(Version version) { @@ -184,7 +184,7 @@ public class SessionZooKeeperClient { } private String getCreateTimePath() { - return rootPath.append(CREATE_TIME_PATH).getAbsolute(); + return sessionPath.append(CREATE_TIME_PATH).getAbsolute(); } AllocatedHosts getAllocatedHosts() { @@ -193,7 +193,7 @@ public class SessionZooKeeperClient { } public ZooKeeperDeployer createDeployer(DeployLogger logger) { - ZooKeeperClient zkClient = new ZooKeeperClient(configCurator, logger, true, rootPath); + ZooKeeperClient zkClient = new ZooKeeperClient(configCurator, logger, true, sessionPath); return new ZooKeeperDeployer(zkClient); } @@ -215,8 +215,8 @@ public class SessionZooKeeperClient { */ public void createNewSession(long createTime, TimeUnit timeUnit) { CuratorTransaction transaction = new CuratorTransaction(curator); - transaction.add(CuratorOperations.create(rootPath.getAbsolute())); - transaction.add(CuratorOperations.create(rootPath.append(UPLOAD_BARRIER).getAbsolute())); + transaction.add(CuratorOperations.create(sessionPath.getAbsolute())); + transaction.add(CuratorOperations.create(sessionPath.append(UPLOAD_BARRIER).getAbsolute())); transaction.add(createWriteStatusTransaction(Session.Status.NEW).operations()); transaction.add(CuratorOperations.create(getCreateTimePath(), Utf8.toBytes(String.valueOf(timeUnit.toSeconds(createTime))))); transaction.commit(); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantBuilder.java b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantBuilder.java index f4a5e941ac2..ed4339e9914 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantBuilder.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantBuilder.java @@ -46,7 +46,6 @@ public class TenantBuilder { private RemoteSessionFactory remoteSessionFactory; private TenantFileSystemDirs tenantFileSystemDirs; private HostValidator<ApplicationId> hostValidator; - private Clock clock = Clock.systemUTC(); private TenantBuilder(GlobalComponentRegistry componentRegistry, TenantName tenant, Path zkPath) { this.componentRegistry = componentRegistry; @@ -89,11 +88,6 @@ public class TenantBuilder { return this; } - public TenantBuilder withClock(Clock clock) { - this.clock = clock; - return this; - } - /** * Create a real tenant from the properties given by this builder. * @@ -103,7 +97,7 @@ public class TenantBuilder { public Tenant build() throws Exception { createTenantRequestHandler(); createApplicationRepo(); - createRemoteSessionFactory(clock); + createRemoteSessionFactory(componentRegistry.getClock()); createRemoteSessionRepo(); createSessionCounter(); createServerDbDirs(); @@ -123,13 +117,15 @@ public class TenantBuilder { private void createLocalSessionRepo() { if (localSessionRepo == null) { - localSessionRepo = new LocalSessionRepo(tenantFileSystemDirs, localSessionLoader, applicationRepo, Clock.systemUTC(), componentRegistry.getConfigserverConfig().sessionLifetime()); + localSessionRepo = new LocalSessionRepo(tenantFileSystemDirs, localSessionLoader, applicationRepo, + componentRegistry.getClock(), componentRegistry.getConfigserverConfig().sessionLifetime()); } } private void createSessionFactory() { if (sessionFactory == null || localSessionLoader == null) { - SessionFactoryImpl impl = new SessionFactoryImpl(componentRegistry, sessionCounter, sessionsPath, applicationRepo, tenantFileSystemDirs, hostValidator, tenant); + SessionFactoryImpl impl = new SessionFactoryImpl(componentRegistry, sessionCounter, sessionsPath, + applicationRepo, tenantFileSystemDirs, hostValidator, tenant); if (sessionFactory == null) { sessionFactory = impl; } @@ -194,7 +190,12 @@ public class TenantBuilder { private void createServerDbDirs() { if (tenantFileSystemDirs == null) { - tenantFileSystemDirs = new TenantFileSystemDirs(new File(Defaults.getDefaults().underVespaHome(componentRegistry.getServerDB().getConfigserverConfig().configServerDBDir())), tenant); + tenantFileSystemDirs = new TenantFileSystemDirs(new File( + Defaults.getDefaults().underVespaHome(componentRegistry + .getServerDB() + .getConfigserverConfig() + .configServerDBDir())), + tenant); } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/Tenants.java b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/Tenants.java index 70be29fa80d..3ad920bbff2 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/Tenants.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/Tenants.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.config.server.tenant; import com.google.common.collect.ImmutableSet; import com.google.inject.Inject; +import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.concurrent.ThreadFactoryFactory; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.TenantName; @@ -20,7 +21,6 @@ import org.apache.curator.framework.state.ConnectionStateListener; import org.apache.zookeeper.KeeperException; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.LinkedHashMap; @@ -56,7 +56,6 @@ public class Tenants implements ConnectionStateListener, PathChildrenCacheListen public static final TenantName HOSTED_VESPA_TENANT = TenantName.from("hosted-vespa"); private static final TenantName DEFAULT_TENANT = TenantName.defaultName(); - private static final List<TenantName> SYSTEM_TENANT_NAMES = Arrays.asList(DEFAULT_TENANT, HOSTED_VESPA_TENANT); private static final Path tenantsPath = Path.fromString("/config/v2/tenants/"); private static final Path vespaPath = Path.fromString("/vespa"); @@ -89,7 +88,7 @@ public class Tenants implements ConnectionStateListener, PathChildrenCacheListen curator.framework().getConnectionStateListenable().addListener(this); curator.create(tenantsPath); - createSystemTenants(); + createSystemTenants(globalComponentRegistry.getConfigserverConfig()); curator.create(vespaPath); this.directoryCache = curator.createDirectoryCache(tenantsPath.getAbsolute(), false, false, pathChildrenExecutor); @@ -213,12 +212,16 @@ public class Tenants implements ConnectionStateListener, PathChildrenCacheListen } /** - * Writes the default tenant into ZooKeeper. Will not fail if the node already exists, - * as this is OK and might happen when several config servers start at the same time and - * try to call this method. + * Writes the tenants that should always be present into ZooKeeper. Will not fail if the node + * already exists, as this is OK and might happen when several config servers start at the + * same time and try to call this method. */ - public synchronized void createSystemTenants() { - for (final TenantName tenantName : SYSTEM_TENANT_NAMES) { + private synchronized void createSystemTenants(ConfigserverConfig configserverConfig) { + List<TenantName> systemTenants = new ArrayList<>(); + systemTenants.add(DEFAULT_TENANT); + if (configserverConfig.hostedVespa()) systemTenants.add(HOSTED_VESPA_TENANT); + + for (final TenantName tenantName : systemTenants) { try { writeTenantPath(tenantName); } catch (RuntimeException e) { diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/ConfigServerBootstrapTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/ConfigServerBootstrapTest.java index 49085971609..36de7d0221a 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/ConfigServerBootstrapTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/ConfigServerBootstrapTest.java @@ -29,6 +29,7 @@ import org.junit.rules.TemporaryFolder; import java.io.File; import java.io.FileReader; +import java.time.Clock; import java.util.ArrayList; import java.util.Optional; @@ -56,12 +57,9 @@ public class ConfigServerBootstrapTest extends TestWithTenant { tenants.writeTenantPath(tenant2); applicationRepository = new ApplicationRepository(tenants, - HostProvisionerProvider.withProvisioner(new SessionHandlerTest.MockProvisioner()), + new SessionHandlerTest.MockProvisioner(), curator, - new LogServerLogGrabber(), - new ApplicationConvergenceChecker(), - new HttpProxy(new SimpleHttpFetcher()), - new ConfigserverConfig(new ConfigserverConfig.Builder())); + Clock.systemUTC()); } @Test diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/TestComponentRegistry.java b/configserver/src/test/java/com/yahoo/vespa/config/server/TestComponentRegistry.java index e98236d8a69..816618c3d28 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/TestComponentRegistry.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/TestComponentRegistry.java @@ -23,6 +23,7 @@ import com.yahoo.vespa.curator.mock.MockCurator; import com.yahoo.vespa.config.server.zookeeper.ConfigCurator; import com.yahoo.vespa.model.VespaModelFactory; +import java.time.Clock; import java.util.Collections; import java.util.Optional; @@ -49,6 +50,7 @@ public class TestComponentRegistry implements GlobalComponentRegistry { private final FileDistributionFactory fileDistributionFactory; private final ModelFactoryRegistry modelFactoryRegistry; private final Optional<Provisioner> hostProvisioner; + private final Clock clock; private TestComponentRegistry(Curator curator, ConfigCurator configCurator, Metrics metrics, ModelFactoryRegistry modelFactoryRegistry, @@ -62,7 +64,8 @@ public class TestComponentRegistry implements GlobalComponentRegistry { Optional<Provisioner> hostProvisioner, ConfigDefinitionRepo defRepo, ReloadListener reloadListener, - TenantListener tenantListener) { + TenantListener tenantListener, + Clock clock) { this.curator = curator; this.configCurator = configCurator; this.metrics = metrics; @@ -78,6 +81,7 @@ public class TestComponentRegistry implements GlobalComponentRegistry { this.modelFactoryRegistry = modelFactoryRegistry; this.hostProvisioner = hostProvisioner; this.sessionPreparer = sessionPreparer; + this.clock = clock; } public static class Builder { @@ -96,7 +100,7 @@ public class TestComponentRegistry implements GlobalComponentRegistry { private Optional<FileDistributionFactory> fileDistributionFactory = Optional.empty(); private ModelFactoryRegistry modelFactoryRegistry = new ModelFactoryRegistry(Collections.singletonList(new VespaModelFactory(new NullConfigModelRegistry()))); private Optional<Provisioner> hostProvisioner = Optional.empty(); - + private Clock clock = Clock.systemUTC(); public Builder configServerConfig(ConfigserverConfig configserverConfig) { this.configserverConfig = configserverConfig; @@ -133,6 +137,11 @@ public class TestComponentRegistry implements GlobalComponentRegistry { return this; } + public Builder clock(Clock clock) { + this.clock = clock; + return this; + } + public TestComponentRegistry build() { final PermanentApplicationPackage permApp = this.permanentApplicationPackage .orElse(new PermanentApplicationPackage(configserverConfig)); @@ -152,7 +161,8 @@ public class TestComponentRegistry implements GlobalComponentRegistry { new SuperModelGenerationCounter(curator), new ConfigServerDB(configserverConfig), hostRegistries, configserverConfig, sessionPreparer, - hostProvisioner, defRepo, reloadListener, tenantListener); + hostProvisioner, defRepo, reloadListener, + tenantListener, clock); } } @@ -194,4 +204,7 @@ public class TestComponentRegistry implements GlobalComponentRegistry { } public FileDistributionFactory getFileDistributionFactory() { return fileDistributionFactory; } + + @Override + public Clock getClock() { return clock;} } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployTester.java b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployTester.java index 5fea15a7b10..8a60a439943 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployTester.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployTester.java @@ -28,13 +28,8 @@ import com.yahoo.transaction.NestedTransaction; import com.yahoo.vespa.config.server.ApplicationRepository; import com.yahoo.vespa.config.server.TestComponentRegistry; import com.yahoo.vespa.config.server.TimeoutBudget; -import com.yahoo.vespa.config.server.application.ApplicationConvergenceChecker; -import com.yahoo.vespa.config.server.application.HttpProxy; -import com.yahoo.vespa.config.server.application.LogServerLogGrabber; -import com.yahoo.vespa.config.server.http.SimpleHttpFetcher; import com.yahoo.vespa.config.server.modelfactory.ModelFactoryRegistry; import com.yahoo.vespa.config.server.monitoring.Metrics; -import com.yahoo.vespa.config.server.provision.HostProvisionerProvider; import com.yahoo.vespa.config.server.session.LocalSession; import com.yahoo.vespa.config.server.session.PrepareParams; import com.yahoo.vespa.config.server.session.SilentDeployLogger; @@ -63,6 +58,7 @@ import java.util.Optional; public class DeployTester { private final Curator curator; + private final Clock clock; private final Tenants tenants; private final File testApp; @@ -75,19 +71,34 @@ public class DeployTester { public DeployTester(String appPath, List<ModelFactory> modelFactories) { this(appPath, modelFactories, new ConfigserverConfig(new ConfigserverConfig.Builder() .configServerDBDir(Files.createTempDir() - .getAbsolutePath()))); + .getAbsolutePath())), + Clock.systemUTC()); } public DeployTester(String appPath, ConfigserverConfig configserverConfig) { this(appPath, Collections.singletonList(createModelFactory(Clock.systemUTC())), - configserverConfig); + configserverConfig, + Clock.systemUTC()); + } + + public DeployTester(String appPath, ConfigserverConfig configserverConfig, Clock clock) { + this(appPath, + Collections.singletonList(createModelFactory(clock)), + configserverConfig, + clock); } public DeployTester(String appPath, List<ModelFactory> modelFactories, ConfigserverConfig configserverConfig) { + this(appPath, modelFactories, configserverConfig, Clock.systemUTC()); + } + + public DeployTester(String appPath, List<ModelFactory> modelFactories, ConfigserverConfig configserverConfig, Clock clock) { Metrics metrics = Metrics.createTestMetrics(); this.curator = new MockCurator(); - TestComponentRegistry componentRegistry = createComponentRegistry(curator, metrics, modelFactories, configserverConfig); + this.clock = clock; + TestComponentRegistry componentRegistry = createComponentRegistry(curator, metrics, modelFactories, + configserverConfig, clock); try { this.testApp = new File(appPath); this.tenants = new Tenants(componentRegistry, metrics); @@ -155,13 +166,9 @@ public class DeployTester { public Optional<com.yahoo.config.provision.Deployment> redeployFromLocalActive(ApplicationId id) { ApplicationRepository applicationRepository = new ApplicationRepository(tenants, - HostProvisionerProvider.withProvisioner(createHostProvisioner()), + createHostProvisioner(), curator, - new LogServerLogGrabber(), - new ApplicationConvergenceChecker(), - new HttpProxy(new SimpleHttpFetcher()), - new ConfigserverConfig(new ConfigserverConfig.Builder())); - + clock); return applicationRepository.deployFromLocalActive(id, Duration.ofSeconds(60)); } @@ -171,7 +178,8 @@ public class DeployTester { private TestComponentRegistry createComponentRegistry(Curator curator, Metrics metrics, List<ModelFactory> modelFactories, - ConfigserverConfig configserverConfig) { + ConfigserverConfig configserverConfig, + Clock clock) { TestComponentRegistry.Builder builder = new TestComponentRegistry.Builder(); if (configserverConfig.hostedVespa()) { @@ -181,7 +189,8 @@ public class DeployTester { builder.configServerConfig(configserverConfig) .curator(curator) .modelFactoryRegistry(new ModelFactoryRegistry(modelFactories)) - .metrics(metrics); + .metrics(metrics) + .clock(clock); return builder.build(); } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/RedeployTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/RedeployTest.java index 49e40321321..82696179dae 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/RedeployTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/RedeployTest.java @@ -1,23 +1,33 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.deploy; +import com.google.common.io.Files; +import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.config.model.api.ModelFactory; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ApplicationName; import com.yahoo.config.provision.InstanceName; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.Version; +import com.yahoo.test.ManualClock; +import com.yahoo.vespa.config.server.session.LocalSession; +import com.yahoo.vespa.config.server.session.Session; import org.junit.Test; import java.io.IOException; import java.time.Clock; +import java.time.Duration; import java.time.Instant; import java.util.ArrayList; +import java.util.Collection; import java.util.List; import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotEquals; import static org.junit.Assert.assertTrue; /** @@ -55,5 +65,48 @@ public class RedeployTest { InstanceName.from("default")); assertFalse(tester.redeployFromLocalActive(id).isPresent()); } - + + @Test + public void testRedeployWillPurgeOldNonActiveDeployments() { + ManualClock clock = new ManualClock(Instant.now()); + ConfigserverConfig configserverConfig = new ConfigserverConfig(new ConfigserverConfig.Builder() + .configServerDBDir(Files.createTempDir() + .getAbsolutePath()) + .sessionLifetime(60)); + DeployTester tester = new DeployTester("src/test/apps/app", configserverConfig, clock); + tester.deployApp("myapp", Instant.now()); // session 2 (numbering starts at 2) + + clock.advance(Duration.ofSeconds(10)); + Optional<com.yahoo.config.provision.Deployment> deployment2 = tester.redeployFromLocalActive(); + + assertTrue(deployment2.isPresent()); + deployment2.get().activate(); // session 3 + long activeSessionId = tester.tenant().getLocalSessionRepo().getActiveSession(tester.applicationId()).getSessionId(); + + clock.advance(Duration.ofSeconds(10)); + Optional<com.yahoo.config.provision.Deployment> deployment3 = tester.redeployFromLocalActive(); + assertTrue(deployment3.isPresent()); + deployment3.get().prepare(); // session 4 (not activated) + + LocalSession deployment3session = ((Deployment) deployment3.get()).session(); + assertNotEquals(activeSessionId, deployment3session); + // No change to active session id + assertEquals(activeSessionId, tester.tenant().getLocalSessionRepo().getActiveSession(tester.applicationId()).getSessionId()); + assertEquals(3, tester.tenant().getLocalSessionRepo().listSessions().size()); + + clock.advance(Duration.ofHours(1)); // longer than session lifetime + + // Need another deployment to get old sessions purged + Optional<com.yahoo.config.provision.Deployment> deployment4 = tester.redeployFromLocalActive(); + assertTrue(deployment4.isPresent()); + deployment4.get().activate(); // session 5 + + // Both session 2 (deactivated) and session 4 (never activated) should have been removed + final Collection<LocalSession> sessions = tester.tenant().getLocalSessionRepo().listSessions(); + assertEquals(2, sessions.size()); + final Set<Long> sessionIds = sessions.stream().map(Session::getSessionId).collect(Collectors.toSet()); + assertTrue(sessionIds.contains(3L)); + assertTrue(sessionIds.contains(5L)); + } + } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationContentHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationContentHandlerTest.java index c3ac7c65266..43e974270a0 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationContentHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationContentHandlerTest.java @@ -1,7 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.http.v2; -import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.config.model.application.provider.FilesApplicationPackage; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.Zone; @@ -11,12 +10,7 @@ import com.yahoo.container.logging.AccessLog; import com.yahoo.jdisc.Response; import com.yahoo.config.provision.ApplicationId; import com.yahoo.vespa.config.server.ApplicationRepository; -import com.yahoo.vespa.config.server.application.ApplicationConvergenceChecker; -import com.yahoo.vespa.config.server.application.HttpProxy; -import com.yahoo.vespa.config.server.application.LogServerLogGrabber; import com.yahoo.vespa.config.server.http.ContentHandlerTestBase; -import com.yahoo.vespa.config.server.http.SimpleHttpFetcher; -import com.yahoo.vespa.config.server.provision.HostProvisionerProvider; import com.yahoo.vespa.config.server.session.Session; import com.yahoo.vespa.curator.mock.MockCurator; import org.junit.Before; @@ -24,6 +18,7 @@ import org.junit.Test; import java.io.File; import java.io.IOException; +import java.time.Clock; import static org.hamcrest.Matchers.is; import static org.junit.Assert.assertNotNull; @@ -62,12 +57,9 @@ public class ApplicationContentHandlerTest extends ContentHandlerTestBase { AccessLog.voidAccessLog(), Zone.defaultZone(), new ApplicationRepository(testTenantBuilder.createTenants(), - HostProvisionerProvider.empty(), + new MockProvisioner(), new MockCurator(), - new LogServerLogGrabber(), - new ApplicationConvergenceChecker(), - new HttpProxy(new SimpleHttpFetcher()), - new ConfigserverConfig(new ConfigserverConfig.Builder()))); + Clock.systemUTC())); pathPrefix = createPath(idTenant1, Zone.defaultZone()); baseUrl = baseServer + pathPrefix; } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ListTenantsTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ListTenantsTest.java index 78dfdd62f1c..189314577e4 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ListTenantsTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ListTenantsTest.java @@ -2,7 +2,6 @@ package com.yahoo.vespa.config.server.http.v2; import com.yahoo.config.provision.TenantName; -import com.yahoo.vespa.config.server.tenant.Tenants; import org.junit.Test; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.jdisc.http.HttpRequest.Method; @@ -34,7 +33,6 @@ public class ListTenantsTest extends TenantTest { private static void assertContainsSystemTenants(final Collection<TenantName> tenantNames) { assertTrue(tenantNames.contains(TenantName.defaultName())); - assertTrue(tenantNames.contains(Tenants.HOSTED_VESPA_TENANT)); } @Test diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionActiveHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionActiveHandlerTest.java index 8644948706b..34ef29b443f 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionActiveHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionActiveHandlerTest.java @@ -1,7 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.http.v2; -import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.config.application.api.ApplicationMetaData; import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.model.NullConfigModelRegistry; @@ -26,9 +25,6 @@ import com.yahoo.vespa.config.server.ApplicationRepository; import com.yahoo.vespa.config.server.PathProvider; import com.yahoo.vespa.config.server.SuperModelGenerationCounter; import com.yahoo.vespa.config.server.TestComponentRegistry; -import com.yahoo.vespa.config.server.application.ApplicationConvergenceChecker; -import com.yahoo.vespa.config.server.application.HttpProxy; -import com.yahoo.vespa.config.server.application.LogServerLogGrabber; import com.yahoo.vespa.config.server.application.MemoryTenantApplications; import com.yahoo.vespa.config.server.application.TenantApplications; import com.yahoo.vespa.config.server.deploy.TenantFileSystemDirs; @@ -38,9 +34,7 @@ import com.yahoo.vespa.config.server.http.HandlerTest; import com.yahoo.vespa.config.server.http.HttpErrorResponse; import com.yahoo.vespa.config.server.http.SessionHandler; import com.yahoo.vespa.config.server.http.SessionHandlerTest; -import com.yahoo.vespa.config.server.http.SimpleHttpFetcher; import com.yahoo.vespa.config.server.modelfactory.ModelFactoryRegistry; -import com.yahoo.vespa.config.server.provision.HostProvisionerProvider; import com.yahoo.vespa.config.server.session.LocalSession; import com.yahoo.vespa.config.server.session.LocalSessionRepo; import com.yahoo.vespa.config.server.session.MockSessionZKClient; @@ -99,7 +93,7 @@ public class SessionActiveHandlerTest extends SessionHandlerTest { applicationRepo = new MemoryTenantApplications(); curator = new MockCurator(); configCurator = ConfigCurator.create(curator); - localRepo = new LocalSessionRepo(applicationRepo); + localRepo = new LocalSessionRepo(applicationRepo, Clock.systemUTC()); pathPrefix = "/application/v2/tenant/" + tenant + "/session/"; pathProvider = new PathProvider(Path.createRoot()); hostProvisioner = new MockProvisioner(); @@ -384,12 +378,9 @@ public class SessionActiveHandlerTest extends SessionHandlerTest { testTenantBuilder.createTenants(), Zone.defaultZone(), new ApplicationRepository(testTenantBuilder.createTenants(), - HostProvisionerProvider.withProvisioner(hostProvisioner), + hostProvisioner, curator, - new LogServerLogGrabber(), - new ApplicationConvergenceChecker(), - new HttpProxy(new SimpleHttpFetcher()), - new ConfigserverConfig(new ConfigserverConfig.Builder()))); + Clock.systemUTC())); } } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionContentHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionContentHandlerTest.java index 16e7aad6d17..26a35b1e195 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionContentHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionContentHandlerTest.java @@ -2,7 +2,6 @@ package com.yahoo.vespa.config.server.http.v2; import com.google.common.io.Files; -import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.config.model.application.provider.FilesApplicationPackage; import com.yahoo.config.provision.TenantName; import com.yahoo.container.jdisc.HttpResponse; @@ -11,13 +10,8 @@ import com.yahoo.jdisc.Response; import com.yahoo.jdisc.http.HttpRequest; import com.yahoo.text.Utf8; import com.yahoo.vespa.config.server.ApplicationRepository; -import com.yahoo.vespa.config.server.application.ApplicationConvergenceChecker; -import com.yahoo.vespa.config.server.application.HttpProxy; -import com.yahoo.vespa.config.server.application.LogServerLogGrabber; import com.yahoo.vespa.config.server.http.ContentHandlerTestBase; import com.yahoo.vespa.config.server.http.SessionHandlerTest; -import com.yahoo.vespa.config.server.http.SimpleHttpFetcher; -import com.yahoo.vespa.config.server.provision.HostProvisionerProvider; import com.yahoo.vespa.curator.mock.MockCurator; import org.apache.commons.io.FileUtils; import org.junit.Before; @@ -28,6 +22,7 @@ import java.io.ByteArrayInputStream; import java.io.File; import java.io.IOException; import java.io.InputStream; +import java.time.Clock; import java.util.concurrent.Executor; import static org.hamcrest.core.Is.is; @@ -175,11 +170,8 @@ public class SessionContentHandlerTest extends ContentHandlerTestBase { } }, AccessLog.voidAccessLog(), testTenantBuilder.createTenants(), new ApplicationRepository(testTenantBuilder.createTenants(), - HostProvisionerProvider.withProvisioner(new SessionHandlerTest.MockProvisioner()), + new SessionHandlerTest.MockProvisioner(), new MockCurator(), - new LogServerLogGrabber(), - new ApplicationConvergenceChecker(), - new HttpProxy(new SimpleHttpFetcher()), - new ConfigserverConfig(new ConfigserverConfig.Builder()))); + Clock.systemUTC())); } } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionCreateHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionCreateHandlerTest.java index 5dc337bf99e..7914b4e0ddb 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionCreateHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionCreateHandlerTest.java @@ -9,17 +9,12 @@ import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.container.logging.AccessLog; import com.yahoo.config.provision.ApplicationId; import com.yahoo.vespa.config.server.ApplicationRepository; -import com.yahoo.vespa.config.server.application.ApplicationConvergenceChecker; -import com.yahoo.vespa.config.server.application.HttpProxy; -import com.yahoo.vespa.config.server.application.LogServerLogGrabber; import com.yahoo.vespa.config.server.application.MemoryTenantApplications; import com.yahoo.vespa.config.server.application.TenantApplications; import com.yahoo.vespa.config.server.http.CompressedApplicationInputStreamTest; import com.yahoo.vespa.config.server.http.HandlerTest; import com.yahoo.vespa.config.server.http.HttpErrorResponse; import com.yahoo.vespa.config.server.http.SessionHandlerTest; -import com.yahoo.vespa.config.server.http.SimpleHttpFetcher; -import com.yahoo.vespa.config.server.provision.HostProvisionerProvider; import com.yahoo.vespa.config.server.session.*; import com.yahoo.vespa.config.server.tenant.Tenants; import com.yahoo.vespa.curator.mock.MockCurator; @@ -28,10 +23,10 @@ import org.junit.Ignore; import org.junit.Test; import java.io.*; +import java.time.Clock; import java.util.Collections; import java.util.HashMap; import java.util.Map; -import java.util.concurrent.Executor; import static com.yahoo.jdisc.Response.Status.*; import static org.hamcrest.core.Is.is; @@ -64,7 +59,7 @@ public class SessionCreateHandlerTest extends SessionHandlerTest { @Before public void setupRepo() throws Exception { applicationRepo = new MemoryTenantApplications(); - localSessionRepo = new LocalSessionRepo(applicationRepo); + localSessionRepo = new LocalSessionRepo(applicationRepo, Clock.systemUTC()); pathPrefix = "/application/v2/tenant/" + tenant + "/session/"; createdMessage = " for tenant '" + tenant + "' created.\""; tenantMessage = ",\"tenant\":\"test\""; @@ -242,12 +237,9 @@ public class SessionCreateHandlerTest extends SessionHandlerTest { final ConfigserverConfig configserverConfig = new ConfigserverConfig(new ConfigserverConfig.Builder()); return new SessionCreateHandler(Runnable::run, AccessLog.voidAccessLog(), tenants, configserverConfig, new ApplicationRepository(testTenantBuilder.createTenants(), - HostProvisionerProvider.withProvisioner(new SessionHandlerTest.MockProvisioner()), + new SessionHandlerTest.MockProvisioner(), new MockCurator(), - new LogServerLogGrabber(), - new ApplicationConvergenceChecker(), - new HttpProxy(new SimpleHttpFetcher()), - configserverConfig)); + Clock.systemUTC())); } public HttpRequest post() throws FileNotFoundException { diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionPrepareHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionPrepareHandlerTest.java index 3172a396eb0..a03940ca95c 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionPrepareHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionPrepareHandlerTest.java @@ -22,10 +22,7 @@ import com.yahoo.transaction.Transaction; import com.yahoo.vespa.config.server.ApplicationRepository; import com.yahoo.vespa.config.server.PathProvider; import com.yahoo.vespa.config.server.TestComponentRegistry; -import com.yahoo.vespa.config.server.application.ApplicationConvergenceChecker; import com.yahoo.vespa.config.server.application.ApplicationSet; -import com.yahoo.vespa.config.server.application.HttpProxy; -import com.yahoo.vespa.config.server.application.LogServerLogGrabber; import com.yahoo.vespa.config.server.host.HostRegistry; import com.yahoo.vespa.config.server.application.TenantApplications; import com.yahoo.vespa.config.server.application.MemoryTenantApplications; @@ -33,7 +30,6 @@ import com.yahoo.vespa.config.server.configchange.ConfigChangeActions; import com.yahoo.vespa.config.server.configchange.MockRefeedAction; import com.yahoo.vespa.config.server.configchange.MockRestartAction; import com.yahoo.vespa.config.server.http.*; -import com.yahoo.vespa.config.server.provision.HostProvisionerProvider; import com.yahoo.vespa.config.server.session.*; import com.yahoo.vespa.curator.Curator; import com.yahoo.vespa.curator.mock.MockCurator; @@ -81,7 +77,7 @@ public class SessionPrepareHandlerTest extends SessionHandlerTest { public void setupRepo() throws Exception { applicationRepo = new MemoryTenantApplications(); curator = new MockCurator(); - localRepo = new LocalSessionRepo(applicationRepo); + localRepo = new LocalSessionRepo(applicationRepo, Clock.systemUTC()); pathPrefix = "/application/v2/tenant/" + tenant + "/session/"; preparedMessage = " for tenant '" + tenant + "' prepared.\""; tenantMessage = ",\"tenant\":\"" + tenant + "\""; @@ -243,7 +239,7 @@ public class SessionPrepareHandlerTest extends SessionHandlerTest { public void require_that_preparing_with_multiple_tenants_work() throws Exception { // Need different repos for 'default' tenant as opposed to the 'test' tenant TenantApplications applicationRepoDefault = new MemoryTenantApplications(); - LocalSessionRepo localRepoDefault = new LocalSessionRepo(applicationRepoDefault); + LocalSessionRepo localRepoDefault = new LocalSessionRepo(applicationRepoDefault, Clock.systemUTC()); final TenantName tenantName = TenantName.defaultName(); addTenant(tenantName, localRepoDefault, new RemoteSessionRepo(), new MockSessionFactory()); addTestTenant(); @@ -379,12 +375,9 @@ public class SessionPrepareHandlerTest extends SessionHandlerTest { final ConfigserverConfig configserverConfig = new ConfigserverConfig(new ConfigserverConfig.Builder()); return new SessionPrepareHandler(Runnable::run, AccessLog.voidAccessLog(), builder.createTenants(), configserverConfig, new ApplicationRepository(builder.createTenants(), - HostProvisionerProvider.withProvisioner(new MockProvisioner()), + new MockProvisioner(), curator, - new LogServerLogGrabber(), - new ApplicationConvergenceChecker(), - new HttpProxy(new SimpleHttpFetcher()), - configserverConfig)); + Clock.systemUTC())); } private TestTenantBuilder addTenant(TenantName tenantName, diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/TestTenantBuilder.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/TestTenantBuilder.java index 3e3d31bbcc0..4f7fc6d271d 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/TestTenantBuilder.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/TestTenantBuilder.java @@ -35,7 +35,7 @@ public class TestTenantBuilder { MemoryTenantApplications applicationRepo = new MemoryTenantApplications(); TenantBuilder builder = TenantBuilder.create(componentRegistry, tenantName, Path.createRoot().append(tenantName.value())) .withSessionFactory(new SessionCreateHandlerTest.MockSessionFactory()) - .withLocalSessionRepo(new LocalSessionRepo(applicationRepo)) + .withLocalSessionRepo(new LocalSessionRepo(applicationRepo, componentRegistry.getClock())) .withRemoteSessionRepo(new RemoteSessionRepo()) .withApplicationRepo(applicationRepo); tenantMap.put(tenantName, builder); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/session/LocalSessionTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/session/LocalSessionTest.java index be98f41c82a..df011dd9307 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/session/LocalSessionTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/session/LocalSessionTest.java @@ -155,10 +155,10 @@ public class LocalSessionTest { } private LocalSession createSession(TenantName tenant, long sessionId, SessionTest.MockSessionPreparer preparer, Optional<AllocatedHosts> allocatedHosts) throws Exception { - Path appPath = Path.fromString("/" + sessionId); - SessionZooKeeperClient zkc = new MockSessionZKClient(curator, appPath, allocatedHosts); + Path sessionPath = Path.fromString("/" + sessionId); + SessionZooKeeperClient zkc = new MockSessionZKClient(curator, sessionPath, allocatedHosts); zkc.createWriteStatusTransaction(Session.Status.NEW).commit(); - ZooKeeperClient zkClient = new ZooKeeperClient(configCurator, new BaseDeployLogger(), false, appPath); + ZooKeeperClient zkClient = new ZooKeeperClient(configCurator, new BaseDeployLogger(), false, sessionPath); if (allocatedHosts.isPresent()) { zkClient.write(allocatedHosts.get()); } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/session/MockSessionZKClient.java b/configserver/src/test/java/com/yahoo/vespa/config/server/session/MockSessionZKClient.java index 412e7881a26..62b0ecbada2 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/session/MockSessionZKClient.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/session/MockSessionZKClient.java @@ -19,20 +19,20 @@ import java.util.Optional; public class MockSessionZKClient extends SessionZooKeeperClient { private ApplicationPackage app = null; - private Optional<AllocatedHosts> info = null; + private Optional<AllocatedHosts> info = Optional.empty(); private Session.Status sessionStatus; - public MockSessionZKClient(Curator curator, Path rootPath) { - this(curator, rootPath, (ApplicationPackage)null); + public MockSessionZKClient(Curator curator, Path sessionPath) { + this(curator, sessionPath, (ApplicationPackage)null); } - public MockSessionZKClient(Curator curator, Path rootPath, Optional<AllocatedHosts> allocatedHosts) { - this(curator, rootPath); + public MockSessionZKClient(Curator curator, Path sessionPath, Optional<AllocatedHosts> allocatedHosts) { + this(curator, sessionPath); this.info = allocatedHosts; } - public MockSessionZKClient(Curator curator, Path rootPath, ApplicationPackage application) { - super(curator, rootPath); + public MockSessionZKClient(Curator curator, Path sessionPath, ApplicationPackage application) { + super(curator, sessionPath); this.app = application; } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/session/RemoteSessionRepoTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/session/RemoteSessionRepoTest.java index cbea7c286cc..462062ce8a8 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/session/RemoteSessionRepoTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/session/RemoteSessionRepoTest.java @@ -52,9 +52,9 @@ public class RemoteSessionRepoTest extends TestWithCurator { private void createSession(String root, long sessionId, boolean wait) { - Path rootPath = Path.fromString(root).append("sessions"); - curator.create(rootPath); - SessionZooKeeperClient zkc = new SessionZooKeeperClient(curator, rootPath.append(String.valueOf(sessionId))); + Path sessionsPath = Path.fromString(root).append("sessions"); + curator.create(sessionsPath); + SessionZooKeeperClient zkc = new SessionZooKeeperClient(curator, sessionsPath.append(String.valueOf(sessionId))); zkc.createNewSession(System.currentTimeMillis(), TimeUnit.MILLISECONDS); if (wait) { Curator.CompletionWaiter waiter = zkc.getUploadWaiter(); @@ -87,6 +87,10 @@ public class RemoteSessionRepoTest extends TestWithCurator { assertNull(remoteSessionRepo.getSession(0l)); } + // If reading a session throws an exception it should be handled and not prevent other applications + // from loading. In this test we just show that we end up with one session in remote session + // repo even if it had bad data (by making getSessionIdForApplication() in FailingTenantApplications + // throw an exception). @Test public void testBadApplicationRepoOnActivate() throws Exception { TenantApplications applicationRepo = new FailingTenantApplications(); @@ -97,6 +101,7 @@ public class RemoteSessionRepoTest extends TestWithCurator { .withApplicationRepo(applicationRepo) .build(); remoteSessionRepo = tenant.getRemoteSessionRepo(); + assertThat(remoteSessionRepo.listSessions().size(), is(0)); createSession("/mytenant", 2l, true); assertThat(remoteSessionRepo.listSessions().size(), is(1)); } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java index 13815e30bbd..265eaa501a2 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java @@ -54,14 +54,15 @@ import static org.junit.Assert.*; */ public class SessionPreparerTest extends TestWithCurator { - private static final Path appPath = Path.createRoot().append("testapp"); + private static final Path tenantPath = Path.createRoot(); + private static final Path sessionsPath = tenantPath.append("sessions").append("testapp"); private static final File testApp = new File("src/test/apps/app"); private static final File invalidTestApp = new File("src/test/apps/illegalApp"); private SessionPreparer preparer; private TestComponentRegistry componentRegistry; private MockFileDistributionFactory fileDistributionFactory; - private Path tenantPath = appPath; + @Rule public TemporaryFolder folder = new TemporaryFolder(); @@ -116,7 +117,7 @@ public class SessionPreparerTest extends TestWithCurator { preparer.prepare(getContext(getApplicationPackage(testApp)), getLogger(), new PrepareParams.Builder().dryRun(true).timeoutBudget(TimeoutBudgetTest.day()).build(), Optional.empty(), tenantPath, Instant.now()); - assertFalse(configCurator.exists(appPath.append(ConfigCurator.USERAPP_ZK_SUBPATH).append("services.xml").getAbsolute())); + assertFalse(configCurator.exists(sessionsPath.append(ConfigCurator.USERAPP_ZK_SUBPATH).append("services.xml").getAbsolute())); } @Test @@ -136,7 +137,7 @@ public class SessionPreparerTest extends TestWithCurator { assertThat(fileDistributionFactory.mockFileDistributionProvider.getMockFileDBHandler().limitSendingOfDeployedFilesToCalled, is(2)); // Should be called only once no matter how many model versions are built assertThat(fileDistributionFactory.mockFileDistributionProvider.getMockFileDBHandler().reloadDeployFileDistributorCalled, is(1)); - assertTrue(configCurator.exists(appPath.append(ConfigCurator.USERAPP_ZK_SUBPATH).append("services.xml").getAbsolute())); + assertTrue(configCurator.exists(sessionsPath.append(ConfigCurator.USERAPP_ZK_SUBPATH).append("services.xml").getAbsolute())); } @Test @@ -185,8 +186,8 @@ public class SessionPreparerTest extends TestWithCurator { .applicationName("foo").instanceName("quux").build(); PrepareParams params = new PrepareParams.Builder().applicationId(origId).build(); preparer.prepare(getContext(getApplicationPackage(testApp)), getLogger(), params, Optional.empty(), tenantPath, Instant.now()); - SessionZooKeeperClient zkc = new SessionZooKeeperClient(curator, appPath); - assertTrue(configCurator.exists(appPath.append(SessionZooKeeperClient.APPLICATION_ID_PATH).getAbsolute())); + SessionZooKeeperClient zkc = new SessionZooKeeperClient(curator, sessionsPath); + assertTrue(configCurator.exists(sessionsPath.append(SessionZooKeeperClient.APPLICATION_ID_PATH).getAbsolute())); assertThat(zkc.readApplicationId(), is(origId)); } @@ -245,7 +246,7 @@ public class SessionPreparerTest extends TestWithCurator { } private SessionContext getContext(FilesApplicationPackage app) throws IOException { - return new SessionContext(app, new SessionZooKeeperClient(curator, appPath), app.getAppDir(), new MemoryTenantApplications(), new HostRegistry<>(), new SuperModelGenerationCounter(curator)); + return new SessionContext(app, new SessionZooKeeperClient(curator, sessionsPath), app.getAppDir(), new MemoryTenantApplications(), new HostRegistry<>(), new SuperModelGenerationCounter(curator)); } private FilesApplicationPackage getApplicationPackage(File testFile) throws IOException { diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClientTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClientTest.java index 6b733bd7ba1..98ba3d4e178 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClientTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClientTest.java @@ -1,7 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.session; -import com.yahoo.config.provision.TenantName; import com.yahoo.path.Path; import com.yahoo.config.provision.ApplicationId; import com.yahoo.text.Utf8; @@ -105,13 +104,9 @@ public class SessionZooKeeperClientTest extends TestWithCurator { assertThat(zkId.serializedForm(), is(expectedIdString)); } - private SessionZooKeeperClient createSessionZKClient(String generation) { - return createSessionZKClient(generation, 100); - } - - private SessionZooKeeperClient createSessionZKClient(String generation, long createTimeInMillis) { - SessionZooKeeperClient zkc = new SessionZooKeeperClient(curator, Path.fromString(generation)); - zkc.createNewSession(createTimeInMillis, TimeUnit.MILLISECONDS); + private SessionZooKeeperClient createSessionZKClient(String sessionId) { + SessionZooKeeperClient zkc = new SessionZooKeeperClient(curator, Path.fromString(sessionId)); + zkc.createNewSession(100, TimeUnit.MILLISECONDS); return zkc; } diff --git a/container-dev/pom.xml b/container-dev/pom.xml index 1d6f3331461..35417a314f8 100644 --- a/container-dev/pom.xml +++ b/container-dev/pom.xml @@ -97,6 +97,10 @@ <groupId>commons-lang</groupId> <artifactId>commons-lang</artifactId> </exclusion> + <exclusion> + <groupId>org.scala-lang</groupId> + <artifactId>scala-library</artifactId> + </exclusion> </exclusions> </dependency> <dependency> @@ -108,6 +112,10 @@ <groupId>org.ow2.asm</groupId> <artifactId>asm</artifactId> </exclusion> + <exclusion> + <groupId>org.scala-lang</groupId> + <artifactId>scala-library</artifactId> + </exclusion> </exclusions> </dependency> <dependency> @@ -142,6 +150,14 @@ <version>${project.version}</version> <exclusions> <exclusion> + <groupId>com.ibm.icu</groupId> + <artifactId>icu4j</artifactId> + </exclusion> + <exclusion> + <groupId>commons-collections</groupId> + <artifactId>commons-collections</artifactId> + </exclusion> + <exclusion> <groupId>commons-lang</groupId> <artifactId>commons-lang</artifactId> </exclusion> @@ -149,6 +165,10 @@ <groupId>org.antlr</groupId> <artifactId>antlr4-runtime</artifactId> </exclusion> + <exclusion> + <groupId>xerces</groupId> + <artifactId>xercesImpl</artifactId> + </exclusion> </exclusions> </dependency> <dependency> @@ -157,6 +177,10 @@ <version>${project.version}</version> <exclusions> <exclusion> + <groupId>org.scala-lang</groupId> + <artifactId>scala-library</artifactId> + </exclusion> + <exclusion> <!-- TODO: Remove exclusion when scala-xml is excluded in config-bundle pom --> <groupId>org.scala-lang.modules</groupId> <artifactId>scala-xml_${scala.major-version}</artifactId> @@ -166,7 +190,18 @@ <!-- Dependencies below are added explicitly to exclude transitive deps that are not provided runtime by the container, and hence make them invisible to user projects' build classpath. - Excluded artifacts should be added explicitly to the application module to make then visible in users' test classpath. --> + Excluded artifacts should be added explicitly to the application module to make then visible in users' test classpath. --> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>document</artifactId> + <version>${project.version}</version> + <exclusions> + <exclusion> + <groupId>commons-collections</groupId> + <artifactId>commons-collections</artifactId> + </exclusion> + </exclusions> + </dependency> <dependency> <groupId>com.yahoo.vespa</groupId> <artifactId>predicate-search-core</artifactId> diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java index 971438e008c..5cc4c441c3b 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java @@ -141,15 +141,16 @@ public class Application { outstandingChange); } - public Application withJobTriggering(JobType type, Instant triggerTime, Controller controller) { + public Application withJobTriggering(JobType type, Optional<Change> change, Instant triggerTime, Controller controller) { return new Application(id, deploymentSpec, validationOverrides, deployments, - deploymentJobs.withTriggering(type, - determineTriggerVersion(type, controller), - determineTriggerRevision(type, controller), - triggerTime), + deploymentJobs.withTriggering(type, + change, + determineTriggerVersion(type, controller), + determineTriggerRevision(type, controller), + triggerTime), deploying, outstandingChange); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java index 0087627675a..ccfcc0e4b0d 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java @@ -310,7 +310,10 @@ public class ApplicationController { // - For self-triggered applications we don't have any trigger information, so we add it here. // - For all applications, we don't have complete control over which revision is actually built, // so we update it here with what we actually triggered if necessary - application = application.with(application.deploymentJobs().withTriggering(jobType, version, Optional.of(revision), clock.instant())); + application = application.with(application.deploymentJobs() + .withTriggering(jobType, application.deploying(), + version, Optional.of(revision), + clock.instant())); } // Delete zones not listed in DeploymentSpec, if allowed diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentJobs.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentJobs.java index dc1dcb2d5ed..b4ca00243b7 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentJobs.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentJobs.java @@ -24,6 +24,7 @@ import java.util.Optional; * This is immutable. * * @author bratseth + * @author mpolden */ public class DeploymentJobs { @@ -68,14 +69,17 @@ public class DeploymentJobs { return new DeploymentJobs(Optional.of(report.projectId()), status, jiraIssueId, report.selfTriggering()); } - public DeploymentJobs withTriggering(DeploymentJobs.JobType jobType, - Version version, - Optional<ApplicationRevision> revision, + public DeploymentJobs withTriggering(JobType jobType, + Optional<Change> change, + Version version, + Optional<ApplicationRevision> revision, Instant triggerTime) { Map<JobType, JobStatus> status = new LinkedHashMap<>(this.status); status.compute(jobType, (type, job) -> { if (job == null) job = JobStatus.initial(jobType); - return job.withTriggering(version, revision, triggerTime); + return job.withTriggering(version, revision, + change.isPresent() && change.get() instanceof Change.VersionChange, + triggerTime); }); return new DeploymentJobs(projectId, status, jiraIssueId, selfTriggering); } @@ -128,10 +132,13 @@ public class DeploymentJobs { } /** Returns whether change has been deployed completely */ - public boolean isDeployed(Change change) { + public boolean isDeployed(Optional<Change> change) { + if (!change.isPresent()) { + return true; + } return status.values().stream() .filter(status -> status.type().isProduction()) - .allMatch(status -> isSuccessful(change, status.type())); + .allMatch(status -> isSuccessful(change.get(), status.type())); } /** Returns whether job has completed successfully */ diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/JobStatus.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/JobStatus.java index a30998d8517..96e91d41584 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/JobStatus.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/JobStatus.java @@ -13,6 +13,7 @@ import java.util.Optional; * This is immutable. * * @author bratseth + * @author mpolden */ public class JobStatus { @@ -52,17 +53,20 @@ public class JobStatus { return new JobStatus(type, Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty()); } - public JobStatus withTriggering(Version version, Optional<ApplicationRevision> revision, Instant triggerTime) { - return new JobStatus(type, jobError, Optional.of(new JobRun(version, revision, triggerTime)), + public JobStatus withTriggering(Version version, Optional<ApplicationRevision> revision, boolean upgrade, + Instant triggerTime) { + return new JobStatus(type, jobError, Optional.of(new JobRun(version, revision, upgrade, triggerTime)), lastCompleted, firstFailing, lastSuccess); } public JobStatus withCompletion(Optional<DeploymentJobs.JobError> jobError, Instant completionTime, Controller controller) { Version version; Optional<ApplicationRevision> revision; + boolean upgrade; if (type == DeploymentJobs.JobType.component) { // not triggered by us version = controller.systemVersion(); revision = Optional.empty(); + upgrade = false; } else if (! lastTriggered.isPresent()) { throw new IllegalStateException("Got notified about completion of " + this + @@ -72,9 +76,10 @@ public class JobStatus { else { version = lastTriggered.get().version(); revision = lastTriggered.get().revision(); + upgrade = lastTriggered.get().upgrade(); } - JobRun thisCompletion = new JobRun(version, revision, completionTime); + JobRun thisCompletion = new JobRun(version, revision, upgrade, completionTime); Optional<JobRun> firstFailing = this.firstFailing; if (jobError.isPresent() && ! this.firstFailing.isPresent()) @@ -166,15 +171,20 @@ public class JobStatus { private final Version version; private final Optional<ApplicationRevision> revision; private final Instant at; + private final boolean upgrade; - public JobRun(Version version, Optional<ApplicationRevision> revision, Instant at) { + public JobRun(Version version, Optional<ApplicationRevision> revision, boolean upgrade, Instant at) { Objects.requireNonNull(version, "version cannot be null"); Objects.requireNonNull(revision, "revision cannot be null"); Objects.requireNonNull(at, "at cannot be null"); this.version = version; this.revision = revision; + this.upgrade = upgrade; this.at = at; } + + /** Returns whether this job run was a Vespa upgrade */ + public boolean upgrade() { return upgrade; } /** The Vespa version used on this run */ public Version version() { return version; } @@ -184,25 +194,26 @@ public class JobStatus { /** The time if this triggering or completion */ public Instant at() { return at; } - - @Override + + @Override public int hashCode() { - return Objects.hash(version ,revision, at); + return Objects.hash(version, revision, upgrade, at); } @Override public boolean equals(Object o) { - if (o == this) return true; - if ( ! (o instanceof JobRun)) return false; - JobRun other = (JobRun)o; - if ( ! Objects.equals(other.version, this.version)) return false; - if ( ! Objects.equals(this.revision, other.revision)) return false; - if ( ! Objects.equals(this.at, other.at)) return false; - return true; + if (this == o) return true; + if (!(o instanceof JobRun)) return false; + JobRun jobRun = (JobRun) o; + return upgrade == jobRun.upgrade && + Objects.equals(version, jobRun.version) && + Objects.equals(revision, jobRun.revision) && + Objects.equals(at, jobRun.at); } - + @Override - public String toString() { return "job run of version " + version + " " + revision + " at " + at; } + public String toString() { return "job run of version " + (upgrade() ? "upgrade " : "") + version + " " + + revision + " at " + at; } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentOrder.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentOrder.java index b860fab3c7e..82e6ca919e1 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentOrder.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentOrder.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.hosted.controller.deployment; import com.yahoo.config.application.api.DeploymentSpec; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.Controller; +import com.yahoo.vespa.hosted.controller.application.Change; import com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType; import com.yahoo.vespa.hosted.controller.application.JobStatus; @@ -13,7 +14,6 @@ import java.util.Collections; import java.util.List; import java.util.Objects; import java.util.Optional; -import java.util.logging.Level; import java.util.logging.Logger; import java.util.stream.Collectors; @@ -39,6 +39,10 @@ public class DeploymentOrder { /** Returns a list of jobs to trigger after the given job */ public List<JobType> nextAfter(JobType job, Application application) { + if (!application.deploying().isPresent()) { // Change was cancelled + return Collections.emptyList(); + } + // Always trigger system test after component as deployment spec might not be available yet (e.g. if this is a // new application with no previous deployments) if (job == JobType.component) { @@ -48,7 +52,7 @@ public class DeploymentOrder { // At this point we have deployed to system test, so deployment spec is available List<DeploymentSpec.Step> deploymentSteps = deploymentSteps(application); Optional<DeploymentSpec.Step> currentStep = fromJob(job, application); - if ( ! currentStep.isPresent()) { + if (!currentStep.isPresent()) { return Collections.emptyList(); } @@ -59,7 +63,7 @@ public class DeploymentOrder { } // Postpone if step hasn't completed all it's jobs for this change - if (!completedSuccessfully(currentStep.get(), application)) { + if (!completedSuccessfully(currentStep.get(), application.deploying().get(), application)) { return Collections.emptyList(); } @@ -89,12 +93,8 @@ public class DeploymentOrder { } DeploymentSpec.Step lastStep = deploymentSteps.get(deploymentSteps.size() - 1); Optional<DeploymentSpec.Step> step = fromJob(job, application); - if (!step.isPresent()) { - log.log(Level.WARNING, "Could not find deployment step for " + application.id().toShortString() + - " from job " + job); - return false; - } - return step.get().equals(lastStep); + // Step may not exist for all jobs, e.g. component + return step.map(s -> s.equals(lastStep)).orElse(false); } /** Returns jobs for given deployment spec, in the order they are declared */ @@ -112,9 +112,9 @@ public class DeploymentOrder { } /** Returns whether all jobs have completed successfully for given step */ - private boolean completedSuccessfully(DeploymentSpec.Step step, Application application) { + private boolean completedSuccessfully(DeploymentSpec.Step step, Change change, Application application) { return jobsFrom(step).stream() - .allMatch(job -> application.deploymentJobs().isSuccessful(application.deploying().get(), job)); + .allMatch(job -> application.deploymentJobs().isSuccessful(change, job)); } /** Resolve deployment step from job */ diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java index ad48ef5c9cc..d2af5fc50cb 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java @@ -72,7 +72,7 @@ public class DeploymentTrigger { else { // start a new change deployment application = application.withDeploying(Optional.of(Change.ApplicationChange.unknown())); } - } else if (order.isLast(report.jobType(), application) && report.success() && application.deploymentJobs().isDeployed(application.deploying().get())) { + } else if (order.isLast(report.jobType(), application) && report.success() && application.deploymentJobs().isDeployed(application.deploying())) { application = application.withDeploying(Optional.empty()); } @@ -290,7 +290,7 @@ public class DeploymentTrigger { cause)); buildSystem.addJob(application.id(), jobType, first); - return application.withJobTriggering(jobType, clock.instant(), controller); + return application.withJobTriggering(jobType, application.deploying(), clock.instant(), controller); } private Application trigger(List<JobType> jobs, Application application, String cause, Lock lock) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java index c705a848b74..607ad4fd9f0 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java @@ -75,6 +75,7 @@ public class ApplicationSerializer { private final String versionField = "version"; private final String revisionField = "revision"; private final String atField = "at"; + private final String upgradeField = "upgrade"; // ------------------ Serialization @@ -149,6 +150,7 @@ public class ApplicationSerializer { object.setString(versionField, jobRun.get().version().toString()); if ( jobRun.get().revision().isPresent()) toSlime(jobRun.get().revision().get(), object.setObject(revisionField)); + object.setBool(upgradeField, jobRun.get().upgrade()); object.setLong(atField, jobRun.get().at().toEpochMilli()); } @@ -196,7 +198,7 @@ public class ApplicationSerializer { return new Zone(Environment.from(object.field(environmentField).asString()), RegionName.from(object.field(regionField).asString())); } - + private Optional<ApplicationRevision> applicationRevisionFromSlime(Inspector object) { if ( ! object.valid()) return Optional.empty(); String applicationPackageHash = object.field(applicationPackageHashField).asString(); @@ -251,14 +253,15 @@ public class ApplicationSerializer { jobRunFromSlime(object.field(firstFailingField)), jobRunFromSlime(object.field(lastSuccessField))); } - + private Optional<JobStatus.JobRun> jobRunFromSlime(Inspector object) { if ( ! object.valid()) return Optional.empty(); return Optional.of(new JobStatus.JobRun(new Version(object.field(versionField).asString()), applicationRevisionFromSlime(object.field(revisionField)), + object.field(upgradeField).asBool(), Instant.ofEpochMilli(object.field(atField).asLong()))); } - + private Optional<Long> optionalLong(Inspector field) { return field.valid() ? Optional.of(field.asLong()) : Optional.empty(); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java index 6d1edc22b93..9d6aa7d3632 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java @@ -370,7 +370,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler { metricsObject.setDouble("writeServiceQuality", metrics.writeServiceQuality()); } catch (RuntimeException e) { - log.log(Level.WARNING, "Failed getting Yamas metrics", e); + log.log(Level.WARNING, "Failed getting Yamas metrics", Exceptions.toMessageString(e)); } return new SlimeJsonResponse(slime); @@ -439,7 +439,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler { metricsObject.setDouble("writeLatencyMillis", metrics.writeLatencyMillis()); } catch (RuntimeException e) { - log.log(Level.WARNING, "Failed getting Yamas metrics", e); + log.log(Level.WARNING, "Failed getting Yamas metrics", Exceptions.toMessageString(e)); } return new SlimeJsonResponse(slime); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiHandler.java index affd679f2c2..8a5f1e4639a 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiHandler.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.restapi.deployment; +import com.yahoo.config.application.api.DeploymentSpec; import com.yahoo.config.provision.ApplicationId; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; @@ -98,25 +99,37 @@ public class DeploymentApiHandler extends LoggingRequestHandler { if (failingSince == null) continue; // started working just now Cursor applicationObject = failingArray.addObject(); - toSlime(id, applicationObject, request); + toSlime(application.get(), applicationObject, request); applicationObject.setLong("failingSince", failingSince.toEpochMilli()); + } Cursor productionArray = versionObject.setArray("productionApplications"); - for (ApplicationId id : version.statistics().production()) - toSlime(id, productionArray.addObject(), request); + for (ApplicationId id : version.statistics().production()) { + Optional<Application> application = controller.applications().get(id); + if ( ! application.isPresent()) continue; // deleted just now + toSlime(application.get(), productionArray.addObject(), request); + } } return new SlimeJsonResponse(slime); } - - private void toSlime(ApplicationId id, Cursor object, HttpRequest request) { - object.setString("tenant", id.tenant().value()); - object.setString("application", id.application().value()); - object.setString("instance", id.instance().value()); - object.setString("url", new Uri(request.getUri()).withPath("/application/v4" + - "/tenant/" + id.tenant().value() + - "/application/" + id.application().value()) - .toString()); + + private void toSlime(Application application, Cursor object, HttpRequest request) { + object.setString("tenant", application.id().tenant().value()); + object.setString("application", application.id().application().value()); + object.setString("instance", application.id().instance().value()); + object.setString("url", new Uri(request.getUri()).withPath("/application/v4/tenant/" + + application.id().tenant().value() + + "/application/" + + application.id().application().value()).toString()); + object.setString("upgradePolicy", toString(application.deploymentSpec().upgradePolicy())); } - + + private static String toString(DeploymentSpec.UpgradePolicy upgradePolicy) { + if (upgradePolicy == DeploymentSpec.UpgradePolicy.defaultPolicy) { + return "default"; + } + return upgradePolicy.name(); + } + } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java index bef96014e79..948301929cf 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java @@ -128,22 +128,24 @@ public class VersionStatus { List<Application> applications) { Map<Version, DeploymentStatistics> versionMap = new HashMap<>(); - for (Version infrastructureVersion : infrastructureVersions) + for (Version infrastructureVersion : infrastructureVersions) { versionMap.put(infrastructureVersion, DeploymentStatistics.empty(infrastructureVersion)); + } for (Application application : applications) { DeploymentJobs jobs = application.deploymentJobs(); // Note that each version deployed on this application exists - for (Deployment deployment : application.deployments().values()) + for (Deployment deployment : application.deployments().values()) { versionMap.computeIfAbsent(deployment.version(), DeploymentStatistics::empty); + } // List versions which have failing jobs, and versions which are in production - // TODO: Don't count applications which started failing on an application change, not a version change // Failing versions Map<Version, List<JobStatus>> failingJobsByVersion = jobs.jobStatus().values().stream() .filter(jobStatus -> jobStatus.lastCompleted().isPresent()) + .filter(jobStatus -> jobStatus.lastCompleted().get().upgrade()) .filter(jobStatus -> jobStatus.jobError().isPresent()) .filter(jobStatus -> jobStatus.jobError().get() != DeploymentJobs.JobError.outOfCapacity) .collect(Collectors.groupingBy(jobStatus -> jobStatus.lastCompleted().get().version())); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java index 26f16a90c6b..3ab98d31a82 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java @@ -111,12 +111,12 @@ public class ControllerTest { Optional<ApplicationRevision> revision = ((Change.ApplicationChange)tester.controller().applications().require(app1.id()).deploying().get()).revision(); assertTrue("Revision has been set during deployment", revision.isPresent()); assertStatus(JobStatus.initial(stagingTest) - .withTriggering(version1, revision, tester.clock().instant()) + .withTriggering(version1, revision, false, tester.clock().instant()) .withCompletion(Optional.empty(), tester.clock().instant(), tester.controller()), app1.id(), tester.controller()); // Causes first deployment job to be triggered assertStatus(JobStatus.initial(productionCorpUsEast1) - .withTriggering(version1, revision, tester.clock().instant()), app1.id(), tester.controller()); + .withTriggering(version1, revision, false, tester.clock().instant()), app1.id(), tester.controller()); tester.clock().advance(Duration.ofSeconds(1)); // production job (failing) @@ -124,9 +124,9 @@ public class ControllerTest { assertEquals(4, applications.require(app1.id()).deploymentJobs().jobStatus().size()); JobStatus expectedJobStatus = JobStatus.initial(productionCorpUsEast1) - .withTriggering(version1, revision, tester.clock().instant()) // Triggered first without revision info + .withTriggering(version1, revision, false, tester.clock().instant()) // Triggered first without revision info .withCompletion(Optional.of(JobError.unknown), tester.clock().instant(), tester.controller()) - .withTriggering(version1, revision, tester.clock().instant()); // Re-triggering (due to failure) has revision info + .withTriggering(version1, revision, false, tester.clock().instant()); // Re-triggering (due to failure) has revision info assertStatus(expectedJobStatus, app1.id(), tester.controller()); @@ -146,20 +146,20 @@ public class ControllerTest { applications.notifyJobCompletion(mockReport(app1, component, true, false)); tester.deployAndNotify(app1, applicationPackage, true, systemTest); assertStatus(JobStatus.initial(systemTest) - .withTriggering(version1, revision, tester.clock().instant()) + .withTriggering(version1, revision, false, tester.clock().instant()) .withCompletion(Optional.empty(), tester.clock().instant(), tester.controller()), app1.id(), tester.controller()); tester.deployAndNotify(app1, applicationPackage, true, stagingTest); // production job succeeding now tester.deployAndNotify(app1, applicationPackage, true, productionCorpUsEast1); expectedJobStatus = expectedJobStatus - .withTriggering(version1, revision, tester.clock().instant()) + .withTriggering(version1, revision, false, tester.clock().instant()) .withCompletion(Optional.empty(), tester.clock().instant(), tester.controller()); assertStatus(expectedJobStatus, app1.id(), tester.controller()); // causes triggering of next production job assertStatus(JobStatus.initial(productionUsEast3) - .withTriggering( version1, revision, tester.clock().instant()), + .withTriggering(version1, revision, false, tester.clock().instant()), app1.id(), tester.controller()); tester.deployAndNotify(app1, applicationPackage, true, productionUsEast3); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java index 7ed0ad843cc..0c0cc0485c8 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java @@ -254,4 +254,30 @@ public class DeploymentTriggerTest { tester.deployAndNotify(application, newApplicationPackage, true, JobType.productionApNortheast1); assertTrue("All jobs consumed", buildSystem.jobs().isEmpty()); } + + @Test + public void testHandleMultipleNotificationsFromLastJob() { + DeploymentTester tester = new DeploymentTester(); + BuildSystem buildSystem = tester.buildSystem(); + TenantId tenant = tester.controllerTester().createTenant("tenant1", "domain1", 1L); + Application application = tester.controllerTester().createApplication(tenant, "app1", "default", 1L); + ApplicationPackage applicationPackage = new ApplicationPackageBuilder() + .environment(Environment.prod) + .region("corp-us-east-1") + .build(); + + // Component job finishes + tester.notifyJobCompletion(JobType.component, application, true); + + // Application is deployed to all test environments and declared zones + tester.deployAndNotify(application, applicationPackage, true, JobType.systemTest); + tester.deployAndNotify(application, applicationPackage, true, JobType.stagingTest); + tester.deployAndNotify(application, applicationPackage, true, JobType.productionCorpUsEast1); + + // Extra notification for last job + tester.notifyJobCompletion(JobType.productionCorpUsEast1, application, true); + assertFalse("Change has been deployed", + tester.applications().require(application.id()).deploying().isPresent()); + assertTrue("All jobs consumed", buildSystem.jobs().isEmpty()); + } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java index e047a288fb9..f7af3ba7a3f 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java @@ -16,6 +16,7 @@ import org.junit.Test; import java.time.Duration; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; /** @@ -266,6 +267,101 @@ public class UpgraderTest { assertTrue("No more jobs triggered at this time", tester.buildSystem().jobs().isEmpty()); } + @Test + public void testUpgradeCancelledWithDeploymentInProgress() { + DeploymentTester tester = new DeploymentTester(); + Version version = Version.fromString("5.0"); + tester.updateVersionStatus(version); + + // Setup applications + Application canary0 = tester.createAndDeploy("canary0", 0, "canary"); + Application canary1 = tester.createAndDeploy("canary1", 1, "canary"); + Application default0 = tester.createAndDeploy("default0", 2, "default"); + Application default1 = tester.createAndDeploy("default1", 3, "default"); + Application default2 = tester.createAndDeploy("default2", 4, "default"); + Application default3 = tester.createAndDeploy("default3", 5, "default"); + Application default4 = tester.createAndDeploy("default4", 6, "default"); + + // New version is released + version = Version.fromString("5.1"); + tester.updateVersionStatus(version); + assertEquals(version, tester.controller().versionStatus().systemVersion().get().versionNumber()); + tester.upgrader().maintain(); + + // Canaries upgrade and raise confidence + tester.completeUpgrade(canary0, version, "canary"); + tester.completeUpgrade(canary1, version, "canary"); + tester.updateVersionStatus(version); + assertEquals(VespaVersion.Confidence.normal, tester.controller().versionStatus().systemVersion().get().confidence()); + + // Applications with default policy start upgrading + tester.upgrader().maintain(); + assertEquals("Upgrade scheduled for remaining apps", 5, tester.buildSystem().jobs().size()); + + // 4/5 applications fail and lowers confidence + tester.completeUpgradeWithError(default0, version, "default", DeploymentJobs.JobType.systemTest); + tester.completeUpgradeWithError(default1, version, "default", DeploymentJobs.JobType.systemTest); + tester.completeUpgradeWithError(default2, version, "default", DeploymentJobs.JobType.systemTest); + tester.completeUpgradeWithError(default3, version, "default", DeploymentJobs.JobType.systemTest); + tester.updateVersionStatus(version); + assertEquals(VespaVersion.Confidence.broken, tester.controller().versionStatus().systemVersion().get().confidence()); + tester.upgrader().maintain(); + + // 5th app passes system-test, but does not trigger next job as upgrade is cancelled + assertFalse("No change present", tester.applications().require(default4.id()).deploying().isPresent()); + tester.notifyJobCompletion(DeploymentJobs.JobType.systemTest, default4, true); + assertTrue("All jobs consumed", tester.buildSystem().jobs().isEmpty()); + } + + @Test + public void testConfidenceIgnoresFailingApplicationChanges() { + DeploymentTester tester = new DeploymentTester(); + Version version = Version.fromString("5.0"); + tester.updateVersionStatus(version); + + // Setup applications + Application canary0 = tester.createAndDeploy("canary0", 0, "canary"); + Application canary1 = tester.createAndDeploy("canary1", 1, "canary"); + Application default0 = tester.createAndDeploy("default0", 2, "default"); + Application default1 = tester.createAndDeploy("default1", 3, "default"); + Application default2 = tester.createAndDeploy("default2", 4, "default"); + Application default3 = tester.createAndDeploy("default3", 5, "default"); + Application default4 = tester.createAndDeploy("default4", 5, "default"); + + // New version is released + version = Version.fromString("5.1"); + tester.updateVersionStatus(version); + assertEquals(version, tester.controller().versionStatus().systemVersion().get().versionNumber()); + tester.upgrader().maintain(); + + // Canaries upgrade and raise confidence + tester.completeUpgrade(canary0, version, "canary"); + tester.completeUpgrade(canary1, version, "canary"); + tester.updateVersionStatus(version); + assertEquals(VespaVersion.Confidence.normal, tester.controller().versionStatus().systemVersion().get().confidence()); + + // All applications upgrade successfully + tester.upgrader().maintain(); + tester.completeUpgrade(default0, version, "default"); + tester.completeUpgrade(default1, version, "default"); + tester.completeUpgrade(default2, version, "default"); + tester.completeUpgrade(default3, version, "default"); + tester.completeUpgrade(default4, version, "default"); + tester.updateVersionStatus(version); + assertEquals(VespaVersion.Confidence.high, tester.controller().versionStatus().systemVersion().get().confidence()); + + // Multiple application changes are triggered and fail, but does not affect version confidence as upgrade has + // completed successfully + tester.notifyJobCompletion(DeploymentJobs.JobType.component, default0, false); + tester.notifyJobCompletion(DeploymentJobs.JobType.component, default1, false); + tester.notifyJobCompletion(DeploymentJobs.JobType.component, default2, true); + tester.notifyJobCompletion(DeploymentJobs.JobType.component, default3, true); + tester.notifyJobCompletion(DeploymentJobs.JobType.systemTest, default2, false); + tester.notifyJobCompletion(DeploymentJobs.JobType.systemTest, default3, false); + tester.updateVersionStatus(version); + assertEquals(VespaVersion.Confidence.normal, tester.controller().versionStatus().systemVersion().get().confidence()); + } + // TODO: Remove when corp-prod special casing is no longer needed @Test public void upgradesCanariesToControllerVersion() { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java index 20e3aae9114..1aaf41350f2 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java @@ -61,10 +61,10 @@ public class ApplicationSerializerTest { List<JobStatus> statusList = new ArrayList<>(); statusList.add(JobStatus.initial(DeploymentJobs.JobType.systemTest) - .withTriggering(Version.fromString("5.6.7"), Optional.empty(), Instant.ofEpochMilli(7)) + .withTriggering(Version.fromString("5.6.7"), Optional.empty(), true, Instant.ofEpochMilli(7)) .withCompletion(Optional.empty(), Instant.ofEpochMilli(8), tester.controller())); statusList.add(JobStatus.initial(DeploymentJobs.JobType.stagingTest) - .withTriggering(Version.fromString("5.6.6"), Optional.empty(), Instant.ofEpochMilli(5)) + .withTriggering(Version.fromString("5.6.6"), Optional.empty(), true, Instant.ofEpochMilli(5)) .withCompletion(Optional.of(JobError.unknown), Instant.ofEpochMilli(6), tester.controller())); DeploymentJobs deploymentJobs = new DeploymentJobs(projectId, statusList, Optional.empty(), false); @@ -133,6 +133,12 @@ public class ApplicationSerializerTest { assertEquals(JobError.unknown, applicationWithFailingJob.deploymentJobs().jobStatus().get(DeploymentJobs.JobType.systemTest).jobError().get()); } + @Test + public void testLegacySerializationWithoutUpgradeField() { + Application application = applicationSerializer.fromSlime(applicationSlime(false)); + assertFalse(application.deploymentJobs().jobStatus().get(DeploymentJobs.JobType.systemTest).lastCompleted().get().upgrade()); + } + private Slime applicationSlime(boolean error) { return SlimeUtils.jsonToSlime(applicationJson(error).getBytes(StandardCharsets.UTF_8)); } @@ -147,10 +153,19 @@ public class ApplicationSerializerTest { " \"jobStatus\": [\n" + " {\n" + " \"jobType\": \"system-test\",\n" + - " \"version\": \"5.6.7\",\n" + - " \"completionTime\": 7,\n" + (error ? " \"jobError\": \"" + JobError.unknown + "\",\n" : "") + - " \"lastTriggered\": 8\n" + + " \"lastCompleted\": {\n" + + " \"version\": \"6.1\",\n" + + " \"revision\": {\n" + + " \"applicationPackageHash\": \"dead\",\n" + + " \"sourceRevision\": {\n" + + " \"repositoryField\": \"git@git.foo\",\n" + + " \"branchField\": \"origin/master\",\n" + + " \"commitField\": \"cafe\"\n" + + " }\n" + + " },\n" + + " \"at\": 1505725189469\n" + + " }\n" + " }\n" + " ],\n" + " \"selfTriggering\": false\n" + diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerControllerTester.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerControllerTester.java index 72abbaa3477..ef606a0eced 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerControllerTester.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerControllerTester.java @@ -28,9 +28,13 @@ import com.yahoo.vespa.hosted.controller.application.DeploymentJobs; import com.yahoo.vespa.hosted.controller.api.integration.athens.mock.AthensMock; import com.yahoo.vespa.hosted.controller.api.integration.athens.mock.AthensDbMock; import com.yahoo.vespa.hosted.controller.api.integration.athens.mock.ZmsClientFactoryMock; +import com.yahoo.vespa.hosted.controller.maintenance.JobControl; +import com.yahoo.vespa.hosted.controller.maintenance.Upgrader; +import com.yahoo.vespa.hosted.controller.persistence.MockCuratorDb; import java.io.File; import java.io.IOException; +import java.time.Duration; import java.util.Optional; /** @@ -42,24 +46,33 @@ public class ContainerControllerTester { private final ContainerTester containerTester; private final Controller controller; + private final Upgrader upgrader; public ContainerControllerTester(JDisc container, String responseFilePath) { containerTester = new ContainerTester(container, responseFilePath); controller = (Controller)container.components().getComponent("com.yahoo.vespa.hosted.controller.Controller"); + upgrader = new Upgrader(controller, Duration.ofMinutes(2), new JobControl(new MockCuratorDb())); } public Controller controller() { return controller; } + public Upgrader upgrader() { return upgrader; } + /** Returns the wrapped generic container tester */ public ContainerTester containerTester() { return containerTester; } public Application createApplication() { - AthensDomain domain1 = addTenantAthensDomain("domain1", "mytenant"); - controller.tenants().addTenant(Tenant.createAthensTenant(new TenantId("tenant1"), domain1, + return createApplication("domain1","tenant1", + "application1"); + } + + public Application createApplication(String athensDomain, String tenant, String application) { + AthensDomain domain1 = addTenantAthensDomain(athensDomain, "mytenant"); + controller.tenants().addTenant(Tenant.createAthensTenant(new TenantId(tenant), domain1, new Property("property1"), Optional.of(new PropertyId("1234"))), Optional.of(TestIdentities.userNToken)); - ApplicationId app = ApplicationId.from("tenant1", "application1", "default"); + ApplicationId app = ApplicationId.from(tenant, application, "default"); return controller.applications().createApplication(app, Optional.of(TestIdentities.userNToken)); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerTester.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerTester.java index 7a9e74a3c27..4fc6e91039c 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerTester.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerTester.java @@ -5,6 +5,7 @@ import com.yahoo.application.container.JDisc; import com.yahoo.application.container.handler.Request; import com.yahoo.application.container.handler.Response; import com.yahoo.collections.Pair; +import com.yahoo.component.Version; import com.yahoo.io.IOUtils; import com.yahoo.slime.ArrayTraverser; import com.yahoo.slime.Inspector; @@ -46,6 +47,11 @@ public class ContainerTester { controller.updateVersionStatus(VersionStatus.compute(controller)); } + public void updateSystemVersion(Version version) { + Controller controller = (Controller)container.components().getComponent("com.yahoo.vespa.hosted.controller.Controller"); + controller.updateVersionStatus(VersionStatus.compute(controller, version)); + } + public void assertResponse(Request request, File responseFile) throws IOException { assertResponse(request, responseFile, 200); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiTest.java index 26741148d3a..c002c7fb24a 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiTest.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.hosted.controller.restapi.deployment; import com.google.common.collect.ImmutableSet; import com.yahoo.application.container.handler.Request; +import com.yahoo.component.Version; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.Zone; @@ -14,6 +15,7 @@ import com.yahoo.vespa.hosted.controller.restapi.ContainerControllerTester; import com.yahoo.vespa.hosted.controller.restapi.ControllerContainerTest; import com.yahoo.vespa.hosted.controller.versions.VersionStatus; import com.yahoo.vespa.hosted.controller.versions.VespaVersion; +import org.junit.Before; import org.junit.Test; import java.io.File; @@ -22,6 +24,7 @@ import java.util.ArrayList; import java.util.List; import static com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType.component; +import static com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType.productionCorpUsEast1; import static com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType.stagingTest; import static com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType.systemTest; @@ -32,23 +35,42 @@ public class DeploymentApiTest extends ControllerContainerTest { private final static String responseFiles = "src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/responses/"; + private ContainerControllerTester tester; + + @Before + public void before() { + tester = new ContainerControllerTester(container, responseFiles); + } + @Test public void testDeploymentApi() throws IOException { ContainerControllerTester tester = new ContainerControllerTester(container, responseFiles); - tester.containerTester().updateSystemVersion(); + Version version = Version.fromString("5.0"); + tester.containerTester().updateSystemVersion(version); long projectId = 11; - Application app = tester.createApplication(); ApplicationPackage applicationPackage = new ApplicationPackageBuilder() .environment(Environment.prod) .region("corp-us-east-1") .build(); - tester.notifyJobCompletion(app.id(), projectId, true, component); - tester.deploy(app, applicationPackage, new Zone(Environment.test, RegionName.from("us-east-1")), projectId); - tester.notifyJobCompletion(app.id(), projectId, true, systemTest); - tester.deploy(app, applicationPackage, new Zone(Environment.staging, RegionName.from("us-east-3")), projectId); - tester.notifyJobCompletion(app.id(), projectId, false, stagingTest); - tester.controller().updateVersionStatus(censorConfigServers(VersionStatus.compute(tester.controller()), + // 2 applications deploy on current system version + Application failingApplication = tester.createApplication("domain1", "tenant1", + "application1"); + Application productionApplication = tester.createApplication("domain2", "tenant2", + "application2"); + deployCompletely(failingApplication, applicationPackage, projectId, true); + deployCompletely(productionApplication, applicationPackage, projectId, true); + + // New version released + version = Version.fromString("5.1"); + tester.containerTester().updateSystemVersion(version); + + // Applications upgrade, 1/2 succeed + tester.upgrader().maintain(); + deployCompletely(failingApplication, applicationPackage, projectId, false); + deployCompletely(productionApplication, applicationPackage, projectId, true); + + tester.controller().updateVersionStatus(censorConfigServers(VersionStatus.compute(tester.controller()), tester.controller())); tester.assertResponse(new Request("http://localhost:8080/deployment/v1/"), new File("root.json")); @@ -69,4 +91,20 @@ public class DeploymentApiTest extends ControllerContainerTest { return new VersionStatus(censored); } + private void deployCompletely(Application application, ApplicationPackage applicationPackage, long projectId, + boolean success) { + tester.notifyJobCompletion(application.id(), projectId, true, component); + tester.deploy(application, applicationPackage, new Zone(Environment.test, + RegionName.from("us-east-1")), projectId); + tester.notifyJobCompletion(application.id(), projectId, true, systemTest); + tester.deploy(application, applicationPackage, new Zone(Environment.staging, + RegionName.from("us-east-3")), projectId); + tester.notifyJobCompletion(application.id(), projectId, success, stagingTest); + if (success) { + tester.deploy(application, applicationPackage, new Zone(Environment.prod,RegionName.from("corp-us-east-1")), + projectId); + tester.notifyJobCompletion(application.id(), projectId, true, productionCorpUsEast1); + } + } + } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/responses/root.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/responses/root.json index 4ea1359519f..c2e83373cf7 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/responses/root.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/responses/root.json @@ -1,50 +1,80 @@ { - "versions":[ + "versions": [ { - "version":"(ignore)", - "confidence":"normal", - "commit":"(ignore)", - "date":0, - "controllerVersion":false, - "systemVersion":true, - "configServers":[ + "version": "(ignore)", + "confidence": "normal", + "commit": "(ignore)", + "date": 0, + "controllerVersion": false, + "systemVersion": false, + "configServers": [], + "failingApplications": [], + "productionApplications": [ { - "hostname":"config1.test" - }, - { - "hostname":"config2.test" + "tenant": "tenant1", + "application": "application1", + "instance": "default", + "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1", + "upgradePolicy": "default" } - ], - "failingApplications":[ + ] + }, + { + "version": "(ignore)", + "confidence": "normal", + "commit": "(ignore)", + "date": 0, + "controllerVersion": false, + "systemVersion": false, + "configServers": [], + "failingApplications": [ { - "tenant":"tenant1", - "application":"application1", - "instance":"default", - "url":"http://localhost:8080/application/v4/tenant/tenant1/application/application1", + "tenant": "tenant1", + "application": "application1", + "instance": "default", + "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1", + "upgradePolicy": "default", "failingSince": "(ignore)" } ], - "productionApplications":[ - + "productionApplications": [ + { + "tenant": "tenant2", + "application": "application2", + "instance": "default", + "url": "http://localhost:8080/application/v4/tenant/tenant2/application/application2", + "upgradePolicy": "default" + } ] }, { - "version":"(ignore)", - "confidence":"normal", - "commit":"(ignore)", - "date":0, - "controllerVersion":true, - "systemVersion":false, - "configServers":[ - - ], - "failingApplications":[ - + "version": "(ignore)", + "confidence": "normal", + "commit": "(ignore)", + "date": 0, + "controllerVersion": false, + "systemVersion": true, + "configServers": [ + { + "hostname": "config1.test" + }, + { + "hostname": "config2.test" + } ], - "productionApplications":[ - - ] + "failingApplications": [], + "productionApplications": [] + }, + { + "version": "(ignore)", + "confidence": "normal", + "commit": "(ignore)", + "date": 0, + "controllerVersion": true, + "systemVersion": false, + "configServers": [], + "failingApplications": [], + "productionApplications": [] } ] } - diff --git a/dist/.tito/packages/.readme b/dist/.tito/packages/.readme new file mode 100644 index 00000000000..ae0c7fe13cc --- /dev/null +++ b/dist/.tito/packages/.readme @@ -0,0 +1,3 @@ +the .tito/packages directory contains metadata files +named after their packages. Each file has the latest tagged +version and the project's relative directory.
\ No newline at end of file diff --git a/dist/.tito/tito.props b/dist/.tito/tito.props new file mode 100644 index 00000000000..db2c33744dc --- /dev/null +++ b/dist/.tito/tito.props @@ -0,0 +1,6 @@ +[buildconfig] +builder = tito.builder.Builder +tagger = tito.tagger.VersionTagger +tag_format = {component}-{version} +changelog_do_not_remove_cherrypick = 0 +changelog_format = %s (%ae)
\ No newline at end of file diff --git a/dist/CMakeLists.txt b/dist/CMakeLists.txt new file mode 100644 index 00000000000..d06074074df --- /dev/null +++ b/dist/CMakeLists.txt @@ -0,0 +1,2 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +install(CODE "execute_process(COMMAND ${CMAKE_CURRENT_LIST_DIR}/post_install.sh ${CMAKE_INSTALL_PREFIX})") diff --git a/dist/post_install.sh b/dist/post_install.sh index eaa7baff6cd..fb0e19370ee 100755 --- a/dist/post_install.sh +++ b/dist/post_install.sh @@ -35,7 +35,6 @@ mkdir -p $INSTALLPATH/conf/zookeeper/ mkdir -p $INSTALLPATH/libexec/jdisc_core/ mkdir -p $INSTALLPATH/libexec/vespa/modelplugins/ mkdir -p $INSTALLPATH/libexec/vespa/plugins/qrs/ -mkdir -p $INSTALLPATH/libexec/yjava_daemon/bin/ mkdir -p $INSTALLPATH/logs/jdisc_core/ mkdir -p $INSTALLPATH/logs/vespa/ mkdir -p $INSTALLPATH/logs/vespa/ @@ -47,8 +46,6 @@ mkdir -p $INSTALLPATH/share/vespa/schema/version/6.x/schema/ mkdir -p $INSTALLPATH/tmp/vespa/ mkdir -p $INSTALLPATH/var/db/jdisc/logcontrol/ mkdir -p $INSTALLPATH/var/db/vespa/ -mkdir -p $INSTALLPATH/var/db/vespa/config_server/serverdb/configs/ -mkdir -p $INSTALLPATH/var/db/vespa/config_server/serverdb/configs/application/ mkdir -p $INSTALLPATH/var/db/vespa/config_server/serverdb/applications/ mkdir -p $INSTALLPATH/var/db/vespa/logcontrol/ mkdir -p $INSTALLPATH/var/jdisc_container/ diff --git a/dist/release-vespa-rpm.sh b/dist/release-vespa-rpm.sh index 95c0b5ce902..ba9195dff05 100755 --- a/dist/release-vespa-rpm.sh +++ b/dist/release-vespa-rpm.sh @@ -9,7 +9,9 @@ fi readonly VERSION=$1 readonly GITREF=$2 -readonly SPECFILE="dist/vespa.spec" +readonly DIST_DIR="dist" +readonly SPECFILE="${DIST_DIR}/vespa.spec" +readonly TITO_DIR="${DIST_DIR}/.tito" readonly RPM_BRANCH="rpmbuild" readonly CURRENT_BRANCH=$(git branch | grep "^\*" | cut -d' ' -f2) @@ -22,14 +24,14 @@ git push --delete origin $RPM_BRANCH &> /dev/null || true git branch -D $RPM_BRANCH &> /dev/null || true git checkout -b $RPM_BRANCH $GITREF -# Tito expects spec file to be on root +# Tito expects spec file and .tito directory to be on root +git mv $TITO_DIR . git mv $SPECFILE . # Hide pom.xml to avoid tito doing anything to our pom.xml files mv pom.xml pom.xml.hide # Run tito to update spec file and tag -tito init tito tag --use-version=$VERSION --no-auto-changelog # Push changes and tag to branc diff --git a/document/src/tests/gid_filter_test.cpp b/document/src/tests/gid_filter_test.cpp index fea4cdbfe95..7519c8f29dd 100644 --- a/document/src/tests/gid_filter_test.cpp +++ b/document/src/tests/gid_filter_test.cpp @@ -1,5 +1,4 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// Copyright 2016 Yahoo! Technologies Norway AS #include <cppunit/TestFixture.h> #include <cppunit/extensions/HelperMacros.h> diff --git a/document/src/tests/heapdebugger.h b/document/src/tests/heapdebugger.h index 84ce5bd1454..1706b99a0a9 100644 --- a/document/src/tests/heapdebugger.h +++ b/document/src/tests/heapdebugger.h @@ -1,14 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -/** - * @file heapdebugger.h - * - * @author Ove Martin Malm - * @date Creation date: 2000-08-09 - * @version $Id$ - * @file mcheckhooks.h - * - * Copyright (c) : 1997-2000 Fast Search & Transfer ASA - * ALL RIGHTS RESERVED +/* + * Author: Ove Martin Malm */ #pragma once diff --git a/document/src/tests/heapdebuggerlinux.cpp b/document/src/tests/heapdebuggerlinux.cpp index 1b9b2e48e1a..fa3a8e3d64b 100644 --- a/document/src/tests/heapdebuggerlinux.cpp +++ b/document/src/tests/heapdebuggerlinux.cpp @@ -1,12 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -/** - * - * @author Ove Martin Malm - * @date Creation date: 2000-21-08 - * @version $Id$ - * - * Copyright (c) : 1997-2000 Fast Search & Transfer ASA - * ALL RIGHTS RESERVED +/* + * Author: Ove Martin Malm */ diff --git a/document/src/tests/heapdebuggerother.cpp b/document/src/tests/heapdebuggerother.cpp index a98203bd8c8..0aeaad57de2 100644 --- a/document/src/tests/heapdebuggerother.cpp +++ b/document/src/tests/heapdebuggerother.cpp @@ -1,12 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -/** - * - * @author Ove Martin Malm - * @date Creation date: 2000-21-08 - * @version $Id$ - * - * Copyright (c) : 1997-2000 Fast Search & Transfer ASA - * ALL RIGHTS RESERVED +/* + * Author: Ove Martin Malm */ #include <stdlib.h> diff --git a/document/src/vespa/document/config/CMakeLists.txt b/document/src/vespa/document/config/CMakeLists.txt index af9f4672fe3..32bbce210a5 100644 --- a/document/src/vespa/document/config/CMakeLists.txt +++ b/document/src/vespa/document/config/CMakeLists.txt @@ -4,7 +4,7 @@ vespa_add_library(document_documentconfig OBJECT DEPENDS ) vespa_generate_config(document_documentconfig documenttypes.def) -install(FILES - documenttypes.def - documentmanager.def +install(FILES documenttypes.def RENAME document.documenttypes.def + DESTINATION var/db/vespa/config_server/serverdb/classes) +install(FILES documentmanager.def RENAME document.config.documentmanager.def DESTINATION var/db/vespa/config_server/serverdb/classes) diff --git a/document/src/vespa/document/serialization/vespadocumentserializer.cpp b/document/src/vespa/document/serialization/vespadocumentserializer.cpp index 6c9e95a9dd6..919f02a5fee 100644 --- a/document/src/vespa/document/serialization/vespadocumentserializer.cpp +++ b/document/src/vespa/document/serialization/vespadocumentserializer.cpp @@ -96,17 +96,14 @@ void VespaDocumentSerializer::write(const Document &value, VespaDocumentSerializer doc_serializer(doc_stream); doc_serializer.write(value.getId()); - int hasHeader = 0; - int hasBody = 0; - - for (StructuredFieldValue::const_iterator it(value.getFields().begin()), mt(value.getFields().end()); - it != mt; - ++it) - { - if (it.field().isHeaderField()) { - hasHeader = 1; + bool hasHeader = false; + bool hasBody = false; + + for (const Field & field : value.getFields()) { + if (field.isHeaderField()) { + hasHeader = true; } else { - hasBody = 1; + hasBody = true; } if (hasHeader && hasBody) { @@ -115,7 +112,7 @@ void VespaDocumentSerializer::write(const Document &value, } if (mode != COMPLETE) { - hasBody = 0; + hasBody = false; } doc_stream << getContentCode(hasHeader, hasBody); @@ -244,8 +241,8 @@ void VespaDocumentSerializer::write(const StringFieldValue &value) { namespace { void serializeFields(const StructFieldValue &value, nbostream &stream, - vector<pair<uint32_t, uint32_t> > &field_info, - const FieldSet& fieldSet) { + vector<pair<uint32_t, uint32_t> > &field_info, const FieldSet& fieldSet) +{ VespaDocumentSerializer serializer(stream); for (StructuredFieldValue::const_iterator it(value.begin()), e(value.end()); it != e; ++it) { if (!fieldSet.contains(it.field())) { @@ -261,8 +258,7 @@ void serializeFields(const StructFieldValue &value, nbostream &stream, } } -bool compressionSufficient(const CompressionConfig &config, - uint64_t old_size, size_t new_size) +bool compressionSufficient(const CompressionConfig &config, uint64_t old_size, size_t new_size) { return (new_size + 8) < (old_size * config.threshold / 100); } @@ -273,13 +269,14 @@ bool bigEnough(size_t size, const CompressionConfig &config) } vespalib::ConstBufferRef -compressStream(const CompressionConfig &config, nbostream &stream, - vespalib::DataBuffer & compressed_data) +compressStream(const CompressionConfig &config, nbostream &stream, vespalib::DataBuffer & compressed_data) { using vespalib::compression::compress; vespalib::ConstBufferRef buf(stream.c_str(), stream.size()); if (config.useCompression() && bigEnough(stream.size(), config)) { - CompressionConfig::Type compressedType = compress(config, vespalib::ConstBufferRef(stream.c_str(), stream.size()), compressed_data, false); + CompressionConfig::Type compressedType = compress(config, + vespalib::ConstBufferRef(stream.c_str(), stream.size()), + compressed_data, false); if (compressedType != config.type || ! compressionSufficient(config, stream.size(), compressed_data.getDataLen())) { @@ -555,5 +552,4 @@ void VespaDocumentSerializer::write(const RemoveFieldPathUpdate &value) writeFieldPath(_stream, value); } - } // namespace document diff --git a/documentapi/src/tests/messages/error_codes_test.cpp b/documentapi/src/tests/messages/error_codes_test.cpp index 46f90d3fb7b..abf8ca13f41 100644 --- a/documentapi/src/tests/messages/error_codes_test.cpp +++ b/documentapi/src/tests/messages/error_codes_test.cpp @@ -1,5 +1,5 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// Copyright 2015 Yahoo Technologies Norway AS + #include <vespa/vespalib/testkit/testapp.h> #include <vespa/documentapi/messagebus/documentprotocol.h> #include <iostream> diff --git a/documentapi/src/vespa/documentapi/messagebus/policies/CMakeLists.txt b/documentapi/src/vespa/documentapi/messagebus/policies/CMakeLists.txt index 14966f28e66..143310d1f67 100644 --- a/documentapi/src/vespa/documentapi/messagebus/policies/CMakeLists.txt +++ b/documentapi/src/vespa/documentapi/messagebus/policies/CMakeLists.txt @@ -20,4 +20,4 @@ vespa_add_library(documentapi_documentapipolicies OBJECT DEPENDS ) vespa_generate_config(documentapi_documentapipolicies ../../../../main/resources/configdefinitions/documentrouteselectorpolicy.def) -install(FILES ../../../../main/resources/configdefinitions/documentrouteselectorpolicy.def DESTINATION var/db/vespa/config_server/serverdb/classes) +install(FILES ../../../../main/resources/configdefinitions/documentrouteselectorpolicy.def RENAME documentapi.messagebus.protocol.documentrouteselectorpolicy.def DESTINATION var/db/vespa/config_server/serverdb/classes) diff --git a/fastlib/src/vespa/fastlib/io/bufferedfile.h b/fastlib/src/vespa/fastlib/io/bufferedfile.h index fcd297bc414..db96d0d6a42 100644 --- a/fastlib/src/vespa/fastlib/io/bufferedfile.h +++ b/fastlib/src/vespa/fastlib/io/bufferedfile.h @@ -1,14 +1,4 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -//************************************************************************ -/** - * Class definitions for Fast_BufferedInputStream - * - */ -/* - * Creation date : 2001-05-22 - * Copyright (c) : 1997-2002 Fast Search & Transfer ASA - * ALL RIGHTS RESERVED - *************************************************************************/ #pragma once diff --git a/fastlib/src/vespa/fastlib/io/bufferedoutputstream.cpp b/fastlib/src/vespa/fastlib/io/bufferedoutputstream.cpp index fff52b5c363..432e7200e66 100644 --- a/fastlib/src/vespa/fastlib/io/bufferedoutputstream.cpp +++ b/fastlib/src/vespa/fastlib/io/bufferedoutputstream.cpp @@ -1,16 +1,9 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -/** -******************************************************************************* -* -* @author Markus Bjartveit Krüger -* @date Creation date: 2001-10-30 -* -* Generic buffered output stream -* -* Copyright (c) : 2001 Fast Search & Transfer ASA -* ALL RIGHTS RESERVED -* -******************************************************************************/ +/* + * Generic buffered output stream + * + * Author: Markus Bjartveit Krüger + */ #include "bufferedoutputstream.h" #include <cstring> diff --git a/fastlib/src/vespa/fastlib/io/fileinputstream.cpp b/fastlib/src/vespa/fastlib/io/fileinputstream.cpp index c9a7cc753b5..ddb6f12d467 100644 --- a/fastlib/src/vespa/fastlib/io/fileinputstream.cpp +++ b/fastlib/src/vespa/fastlib/io/fileinputstream.cpp @@ -1,16 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -/** -******************************************************************************* -* -* @author Stein Hardy Danielsen -* @date Creation date: 2000-10-07 -* -* FileInputStream class implementation -* -* Copyright (c) : 1997-2000 Fast Search & Transfer ASA -* ALL RIGHTS RESERVED -* -******************************************************************************/ +/* + * Author: Stein Hardy Danielsen + */ #include "fileinputstream.h" diff --git a/fastlib/src/vespa/fastlib/io/fileoutputstream.cpp b/fastlib/src/vespa/fastlib/io/fileoutputstream.cpp index 649a1852ecf..df969434d60 100644 --- a/fastlib/src/vespa/fastlib/io/fileoutputstream.cpp +++ b/fastlib/src/vespa/fastlib/io/fileoutputstream.cpp @@ -1,16 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -/** -******************************************************************************* -* -* @author Stein Hardy Danielsen -* @date Creation date: 2000-10-07 -* -* FileOutputStream class implementation -* -* Copyright (c) : 1997-2000 Fast Search & Transfer ASA -* ALL RIGHTS RESERVED -* -******************************************************************************/ +/* + * Author: Stein Hardy Danielsen + */ #include "fileoutputstream.h" #include <vespa/fastos/file.h> diff --git a/fastlib/src/vespa/fastlib/io/filterinputstream.h b/fastlib/src/vespa/fastlib/io/filterinputstream.h index 55b705eba63..41fbda89bde 100644 --- a/fastlib/src/vespa/fastlib/io/filterinputstream.h +++ b/fastlib/src/vespa/fastlib/io/filterinputstream.h @@ -1,16 +1,9 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -/** -******************************************************************************* -* -* @author Stein Hardy Danielsen -* @date Creation date: 2000-1-14 -* -* Generic filter input stream -* -* Copyright (c) : 1997-1999 Fast Search & Transfer ASA -* ALL RIGHTS RESERVED -* -******************************************************************************/ +/* + * Generic filter input stream + * + * Author: Stein Hardy Danielsen + */ #pragma once #include "inputstream.h" diff --git a/fastlib/src/vespa/fastlib/io/outputstream.h b/fastlib/src/vespa/fastlib/io/outputstream.h index 78eb93fac0f..a783d8786a7 100644 --- a/fastlib/src/vespa/fastlib/io/outputstream.h +++ b/fastlib/src/vespa/fastlib/io/outputstream.h @@ -1,16 +1,9 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -/** -******************************************************************************* -* -* @author Stein Hardy Danielsen -* @date Creation date: 2000-1-14 -* -* Generic output stream interface -* -* Copyright (c) : 1997-1999 Fast Search & Transfer ASA -* ALL RIGHTS RESERVED -* -******************************************************************************/ +/* + * Generic output stream interface + * + * Author Stein Hardy Danielsen + */ #pragma once #include <cstdlib> diff --git a/fastlib/src/vespa/fastlib/net/httpchunkedinputstream.cpp b/fastlib/src/vespa/fastlib/net/httpchunkedinputstream.cpp index 123f34fdd4d..899a5a3ab83 100644 --- a/fastlib/src/vespa/fastlib/net/httpchunkedinputstream.cpp +++ b/fastlib/src/vespa/fastlib/net/httpchunkedinputstream.cpp @@ -1,16 +1,9 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -/** -******************************************************************************* -* -* @author Markus Bjartveit Krüger -* @date Creation date: 2001-11-21 -* -* HTTP chunked input stream. -* -* Copyright (c) : 2001 Fast Search & Transfer ASA -* ALL RIGHTS RESERVED -* -******************************************************************************/ +/* + * HTTP chunked input stream. + * + * Author: Markus Bjartveit Krüger + */ #include "httpchunkedinputstream.h" #include <cstring> diff --git a/fastlib/src/vespa/fastlib/net/httpchunkedoutputstream.cpp b/fastlib/src/vespa/fastlib/net/httpchunkedoutputstream.cpp index 673cbe4eca5..67bcee98bd6 100644 --- a/fastlib/src/vespa/fastlib/net/httpchunkedoutputstream.cpp +++ b/fastlib/src/vespa/fastlib/net/httpchunkedoutputstream.cpp @@ -1,16 +1,9 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -/** -******************************************************************************* -* -* @author Markus Bjartveit Krüger -* @date Creation date: 2001-11-21 -* -* HTTP chunked output stream. -* -* Copyright (c) : 2001 Fast Search & Transfer ASA -* ALL RIGHTS RESERVED -* -******************************************************************************/ +/* + * HTTP chunked output stream. + * + * Author: Markus Bjartveit Krüger + */ #include "httpchunkedoutputstream.h" #include <cassert> diff --git a/fastlib/src/vespa/fastlib/net/httpserver.h b/fastlib/src/vespa/fastlib/net/httpserver.h index 9f320924e34..571a6108181 100644 --- a/fastlib/src/vespa/fastlib/net/httpserver.h +++ b/fastlib/src/vespa/fastlib/net/httpserver.h @@ -1,16 +1,9 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -/** -******************************************************************************* -* -* @author Stein Hardy Danielsen -* @date Creation date: 2000-1-7 -* -* Generic http server and connection classes -* -* Copyright (c) : 1997-1999 Fast Search & Transfer ASA -* ALL RIGHTS RESERVED -* -******************************************************************************/ +/* + * Generic http server and connection classes + * + * Author: Stein Hardy Danielsen + */ #pragma once @@ -30,19 +23,9 @@ class Fast_HTTPServer; #define FASTLIB_SUCCESS (0) #define FASTLIB_FAILURE (1) -/** -******************************************************************************** -* -* Generic HTTP connection class -* @author Stein Hardy Danielsen -* @date Creation date: 2000-1-7 -* -* Generic HTTP connection class -* -* Copyright (c) : 1997-1999 Fast Search & Transfer ASA -* ALL RIGHTS RESERVED -* -******************************************************************************/ +/* + * Generic HTTP connection class + */ // Error codes #define FASTLIB_HTTPSERVER_NEWTHREADFAILED (2) @@ -135,18 +118,9 @@ public: -/** -******************************************************************************** -* -* Generic HTTP server class -* -* @author Stein Hardy Danielsen -* @date Creation date: 2000-1-7 -* -* Copyright (c) : 1997-1999 Fast Search & Transfer ASA -* ALL RIGHTS RESERVED -* -******************************************************************************/ +/* + * Generic HTTP server class + */ class Fast_HTTPServer : public FastOS_Runnable { private: diff --git a/fastlib/src/vespa/fastlib/net/url.cpp b/fastlib/src/vespa/fastlib/net/url.cpp index 4f7fe74ab4e..39a49777225 100644 --- a/fastlib/src/vespa/fastlib/net/url.cpp +++ b/fastlib/src/vespa/fastlib/net/url.cpp @@ -1,15 +1,8 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. /** - * - * @file url.cpp - * @author Michael Susæg - * @date Creation date: 1999-11-23 - * * This file contains different URL string functions * - * Copyright (c) : 1997-2000 Fast Search & Transfer ASA. - * ALL RIGHTS RESERVED - * + * Author: Michael Susæg */ diff --git a/fastlib/src/vespa/fastlib/net/url.h b/fastlib/src/vespa/fastlib/net/url.h index 4cc25f8b58d..2dd4827b0db 100644 --- a/fastlib/src/vespa/fastlib/net/url.h +++ b/fastlib/src/vespa/fastlib/net/url.h @@ -1,15 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -/** - * - * @file url.h - * @author Michael Susæg - * @date Creation date: - * - * This file is the header file for the url class. - * - * Copyright (c) : 1997-2000 Fast Search & Transfer ASA. - * ALL RIGHTS RESERVED - * +/* + * Author: Michael Susæg */ #pragma once diff --git a/fastlib/src/vespa/fastlib/testsuite/suite.h b/fastlib/src/vespa/fastlib/testsuite/suite.h index 94c629f5d2c..94e756387bf 100644 --- a/fastlib/src/vespa/fastlib/testsuite/suite.h +++ b/fastlib/src/vespa/fastlib/testsuite/suite.h @@ -1,13 +1,10 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. /************************************************************************** - * @author BÃ¥rd Kvalheim + * Author: BÃ¥rd Kvalheim * * A test suite. Modified from the suite written by Chuck Allison. * http://www.cuj.com/archive/1809/feature.html * - * @date Creation date: 2000-12-15 - * Copyright (c) : 1997-1999 Fast Search & Transfer ASA - * ALL RIGHTS RESERVED * Licensed to Yahoo, and relicensed under the terms of the Apache 2.0 license * * The usage of suite is simple: diff --git a/fastlib/src/vespa/fastlib/testsuite/test.h b/fastlib/src/vespa/fastlib/testsuite/test.h index 7ee04984da0..ea17e64dba2 100644 --- a/fastlib/src/vespa/fastlib/testsuite/test.h +++ b/fastlib/src/vespa/fastlib/testsuite/test.h @@ -1,14 +1,10 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. /************************************************************************** - * @author BÃ¥rd Kvalheim + * Author: BÃ¥rd Kvalheim * * The test class of the testsuite. Written by Chuck Allison. * http://www.cuj.com/archive/1809/feature.html * - * @date Creation date: 2000-12-15 - * Copyright (c) : 1997-2000 Fast Search & Transfer ASA - * ALL RIGHTS RESERVED - * * Apart for a trick the usage of the test class is very simple: * * mytest.h: diff --git a/fastlib/src/vespa/fastlib/text/apps/extcase.cpp b/fastlib/src/vespa/fastlib/text/apps/extcase.cpp index 5e5f3a04492..b713523af08 100644 --- a/fastlib/src/vespa/fastlib/text/apps/extcase.cpp +++ b/fastlib/src/vespa/fastlib/text/apps/extcase.cpp @@ -1,17 +1,12 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -/** - * @author Tor Egge - * @author Vidar Larsen - * +/* * Extract case information from Unicode property files. - * This program read the UnicodeData-4.0.0.txt file and generates + * This program reads the UnicodeData-4.0.0.txt file and generates * the unicodeutil-lowercase.cpp file that gives a mapping from * unicode characters to their lowercase equivalents. * - * Copyright (C) 2000 - 2003 Fast Search & Transfer ASA - * Copyright (C) 2003 Overture Services Norway AS - * - * ALL RIGHTS RESERVED + * Author: Tor Egge + * Author: Vidar Larsen */ #include <vespa/fastlib/io/bufferedfile.h> diff --git a/fastos/src/tests/prefetchtest.cpp b/fastos/src/tests/prefetchtest.cpp index 38d3eed837c..fba57e4048f 100644 --- a/fastos/src/tests/prefetchtest.cpp +++ b/fastos/src/tests/prefetchtest.cpp @@ -1,16 +1,9 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -//************************************************************************ -/** +/* * FastOS_Prefetch test program. * - * @author Olaf Birkeland - * @version $Id$ + * Author: Olaf Birkeland */ - /* - * Creation date : 2000-12-11 - * Copyright (c) : 1997-2001 Fast Search & Transfer ASA - * ALL RIGHTS RESERVED - *************************************************************************/ #include "tests.h" #include <vespa/fastos/time.h> diff --git a/filedistribution/src/vespa/filedistribution/distributor/CMakeLists.txt b/filedistribution/src/vespa/filedistribution/distributor/CMakeLists.txt index d916b68121a..5345cba1e3f 100644 --- a/filedistribution/src/vespa/filedistribution/distributor/CMakeLists.txt +++ b/filedistribution/src/vespa/filedistribution/distributor/CMakeLists.txt @@ -12,4 +12,4 @@ vespa_add_library(filedistribution_distributor STATIC ) target_compile_options(filedistribution_distributor PRIVATE -DTORRENT_DISABLE_ENCRYPTION -DTORRENT_DISABLE_DHT -DWITH_SHIPPED_GEOIP_H -DBOOST_ASIO_HASH_MAP_BUCKETS=1021 -DBOOST_EXCEPTION_DISABLE -DBOOST_ASIO_ENABLE_CANCELIO -DBOOST_ASIO_DYN_LINK -DTORRENT_LINKING_SHARED) vespa_generate_config(filedistribution_distributor filedistributor.def) -install(FILES filedistributor.def DESTINATION var/db/vespa/config_server/serverdb/classes) +install(FILES filedistributor.def RENAME cloud.config.filedistribution.filedistributor.def DESTINATION var/db/vespa/config_server/serverdb/classes) diff --git a/filedistribution/src/vespa/filedistribution/model/CMakeLists.txt b/filedistribution/src/vespa/filedistribution/model/CMakeLists.txt index f745057e3a3..7136aba475b 100644 --- a/filedistribution/src/vespa/filedistribution/model/CMakeLists.txt +++ b/filedistribution/src/vespa/filedistribution/model/CMakeLists.txt @@ -17,4 +17,4 @@ vespa_add_library(filedistribution_filedistributionmodel STATIC vespa_generate_config(filedistribution_filedistributionmodel filereferences.def) vespa_add_target_external_dependency(filedistribution_filedistributionmodel zookeeper_mt) -install(FILES filereferences.def DESTINATION var/db/vespa/config_server/serverdb/classes) +install(FILES filereferences.def RENAME cloud.config.filedistribution.filereferences.def DESTINATION var/db/vespa/config_server/serverdb/classes) diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/linguistics/LinguisticsAnnotator.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/linguistics/LinguisticsAnnotator.java index 3adffa30725..ef94f67d6e6 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/linguistics/LinguisticsAnnotator.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/linguistics/LinguisticsAnnotator.java @@ -103,7 +103,7 @@ public class LinguisticsAnnotator { } private static void addAnnotationSpan(String input, SpanList parent, Tokenizer tokenizer, Token token, StemMode mode, TermOccurrences termOccurrences) { - if (!token.isSpecialToken()) { + if ( ! token.isSpecialToken()) { if (token.getNumComponents() > 0) { for (int i = 0; i < token.getNumComponents(); ++i) { addAnnotationSpan(input, parent, tokenizer, token.getComponent(i), mode, termOccurrences); @@ -144,9 +144,7 @@ public class LinguisticsAnnotator { for (int i = 0; i < token.getNumStems(); i++) { String stem = token.getStem(i); String lowercasedStem = toLowerCase(stem); - if (! (lowercasedOrig.equals(lowercasedStem) - || lowercasedTerm.equals(lowercasedStem))) - { + if (! (lowercasedOrig.equals(lowercasedStem) || lowercasedTerm.equals(lowercasedStem))) { addAnnotation(where, stem, orig, termOccurrences); } } diff --git a/install_java.cmake b/install_java.cmake index a0fb98a12ae..e0611306b5f 100644 --- a/install_java.cmake +++ b/install_java.cmake @@ -87,36 +87,6 @@ install(FILES install(FILES chain/src/main/resources/configdefinitions/chains.def config-provisioning/src/main/resources/configdefinitions/flavors.def - configdefinitions/src/vespa/application-id.def - configdefinitions/src/vespa/attributes.def - configdefinitions/src/vespa/cluster-info.def - configdefinitions/src/vespa/cluster-list.def - configdefinitions/src/vespa/configserver.def - configdefinitions/src/vespa/fleetcontroller.def - configdefinitions/src/vespa/ilscripts.def - configdefinitions/src/vespa/indexschema.def - configdefinitions/src/vespa/lb-services.def - configdefinitions/src/vespa/load-type.def - configdefinitions/src/vespa/messagetyperouteselectorpolicy.def - configdefinitions/src/vespa/model.def - configdefinitions/src/vespa/persistence.def - configdefinitions/src/vespa/rank-profiles.def - configdefinitions/src/vespa/routing-provider.def - configdefinitions/src/vespa/routing.def - configdefinitions/src/vespa/sentinel.def - configdefinitions/src/vespa/slobroks.def - configdefinitions/src/vespa/specialtokens.def - configdefinitions/src/vespa/stor-devices.def - configdefinitions/src/vespa/stor-distribution.def - configdefinitions/src/vespa/stor-filestor.def - configdefinitions/src/vespa/stor-memfilepersistence.def - configdefinitions/src/vespa/summary.def - configdefinitions/src/vespa/summarymap.def - configdefinitions/src/vespa/upgrading.def - configdefinitions/src/vespa/ymon.def - configdefinitions/src/vespa/zookeeper-server.def - configdefinitions/src/vespa/zookeepers.def - configdefinitions/src/vespa/dispatch.def container-accesslogging/src/main/resources/configdefinitions/access-log.def container-core/src/main/resources/configdefinitions/application-metadata.def container-core/src/main/resources/configdefinitions/container-document.def @@ -177,47 +147,14 @@ install(FILES docproc/src/main/resources/configdefinitions/docproc.def docproc/src/main/resources/configdefinitions/schemamapping.def docproc/src/main/resources/configdefinitions/splitter-joiner-document-processor.def - document/src/vespa/document/config/documentmanager.def - document/src/vespa/document/config/documenttypes.def - documentapi/src/main/resources/configdefinitions/documentrouteselectorpolicy.def fileacquirer/src/main/resources/configdefinitions/filedistributorrpc.def - filedistribution/src/vespa/filedistribution/distributor/filedistributor.def - filedistribution/src/vespa/filedistribution/model/filereferences.def jdisc_http_service/src/main/resources/configdefinitions/jdisc.http.client.http-client.def jdisc_http_service/src/main/resources/configdefinitions/jdisc.http.connector.def jdisc_http_service/src/main/resources/configdefinitions/jdisc.http.server.def jdisc_http_service/src/main/resources/configdefinitions/jdisc.http.servlet-paths.def - logd/src/main/resources/configdefinitions/logd.def - messagebus/src/main/config/messagebus.def - metrics/src/vespa/metrics/metricsmanager.def - orchestrator/src/main/resources/configdefinitions/orchestrator.def persistence/src/main/resources/configdefinitions/persistence-rpc.def - searchcore/src/vespa/searchcore/config/fdispatchrc.def - searchcore/src/vespa/searchcore/config/partitions.def - searchcore/src/vespa/searchcore/config/proton.def - searchcore/src/vespa/searchcore/config/ranking-constants.def - searchlib/src/vespa/searchlib/config/translogserver.def - searchsummary/src/vespa/searchsummary/config/juniperrc.def simplemetrics/src/main/resources/configdefinitions/manager.def statistics/src/main/resources/configdefinitions/statistics.def - storage/src/vespa/storage/bucketdb/stor-bucket-init.def - storage/src/vespa/storage/bucketdb/stor-bucketdb.def - storage/src/vespa/storage/config/rpc-provider.def - storage/src/vespa/storage/config/stor-bouncer.def - storage/src/vespa/storage/config/stor-bucketmover.def - storage/src/vespa/storage/config/stor-communicationmanager.def - storage/src/vespa/storage/config/stor-distributormanager.def - storage/src/vespa/storage/config/stor-integritychecker.def - storage/src/vespa/storage/config/stor-messageforwarder.def - storage/src/vespa/storage/config/stor-opslogger.def - storage/src/vespa/storage/config/stor-prioritymapping.def - storage/src/vespa/storage/config/stor-server.def - storage/src/vespa/storage/config/stor-status.def - storage/src/vespa/storage/config/stor-visitordispatcher.def - storage/src/vespa/storage/visiting/stor-visitor.def - vsm/src/vespa/vsm/config/vsm.def - vsm/src/vespa/vsm/config/vsmfields.def - vsm/src/vespa/vsm/config/vsmsummary.def vespaclient-core/src/main/resources/configdefinitions/feeder.def vespaclient-core/src/main/resources/configdefinitions/spooler.def docker-api/src/main/resources/configdefinitions/docker.def diff --git a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/ConnectorFactory.java b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/ConnectorFactory.java index a8dbf66f537..bcc48ed56ae 100644 --- a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/ConnectorFactory.java +++ b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/ConnectorFactory.java @@ -135,6 +135,19 @@ public class ConnectorFactory { Ssl sslConfig = connectorConfig.ssl(); final SslContextFactory factory = new SslContextFactory(); + switch (sslConfig.clientAuth()) { + case NEED_AUTH: + factory.setNeedClientAuth(true); + break; + case WANT_AUTH: + factory.setWantClientAuth(true); + break; + } + + if (!sslConfig.prng().isEmpty()) { + factory.setSecureRandomAlgorithm(sslConfig.prng()); + } + if (!sslConfig.excludeProtocol().isEmpty()) { final String[] prots = new String[sslConfig.excludeProtocol().size()]; for (int i = 0; i < prots.length; i++) { diff --git a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/HttpRequestFactory.java b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/HttpRequestFactory.java index d137632f1fe..714d75f9d1e 100644 --- a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/HttpRequestFactory.java +++ b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/HttpRequestFactory.java @@ -8,6 +8,7 @@ import com.yahoo.jdisc.service.CurrentContainer; import javax.servlet.http.HttpServletRequest; import java.net.InetSocketAddress; import java.net.URI; +import java.security.cert.X509Certificate; import java.util.Enumeration; import static com.yahoo.jdisc.http.core.HttpServletRequestUtils.getConnection; @@ -19,13 +20,15 @@ import static com.yahoo.jdisc.http.core.HttpServletRequestUtils.getConnection; class HttpRequestFactory { public static HttpRequest newJDiscRequest(CurrentContainer container, HttpServletRequest servletRequest) { - return HttpRequest.newServerRequest( + HttpRequest httpRequest = HttpRequest.newServerRequest( container, getUri(servletRequest), HttpRequest.Method.valueOf(servletRequest.getMethod()), HttpRequest.Version.fromString(servletRequest.getProtocol()), new InetSocketAddress(servletRequest.getRemoteAddr(), servletRequest.getRemotePort()), getConnection(servletRequest).getCreatedTimeStamp()); + httpRequest.context().put("jdisc.request.X509Certificate", getCertChain(servletRequest)); + return httpRequest; } public static URI getUri(HttpServletRequest servletRequest) { @@ -93,4 +96,7 @@ class HttpRequestFactory { } } + private static X509Certificate[] getCertChain(HttpServletRequest servletRequest) { + return (X509Certificate[]) servletRequest.getAttribute("javax.servlet.request.X509Certificate"); + } } diff --git a/jdisc_http_service/src/main/resources/configdefinitions/jdisc.http.connector.def b/jdisc_http_service/src/main/resources/configdefinitions/jdisc.http.connector.def index 00b089ae3f9..1c059fff2e7 100644 --- a/jdisc_http_service/src/main/resources/configdefinitions/jdisc.http.connector.def +++ b/jdisc_http_service/src/main/resources/configdefinitions/jdisc.http.connector.def @@ -77,3 +77,12 @@ ssl.sslKeyManagerFactoryAlgorithm string default="SunX509" # The SSL protocol passed to SSLContext.getInstance() ssl.protocol string default="TLS" + +# Client authentication mode. See SSLEngine.getNeedClientAuth()/getWantClientAuth() for details. +ssl.clientAuth enum { DISABLED, WANT_AUTH, NEED_AUTH } default=DISABLED + +# The SecureRandom implementation passed to SSLEngine.init() +# Java have a default pseudo-random number generator (PRNG) for crypto operations. This default may have performance +# issues on some platform (e.g. NativePRNG in Linux utilizes a global lock). Changing the generator to SHA1PRNG may +# improve performance. Set value to empty string to use the default generator. +ssl.prng string default="" diff --git a/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/server/jetty/HttpRequestFactoryTest.java b/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/server/jetty/HttpRequestFactoryTest.java index 862c85c187e..476718ac906 100644 --- a/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/server/jetty/HttpRequestFactoryTest.java +++ b/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/server/jetty/HttpRequestFactoryTest.java @@ -70,9 +70,14 @@ public class HttpRequestFactoryTest { @Override public Object getAttribute(String name) { - HttpConnection connection = mock(HttpConnection.class); - when(connection.getCreatedTimeStamp()).thenReturn(System.currentTimeMillis()); - return connection; + switch (name) { + case "org.eclipse.jetty.server.HttpConnection": + HttpConnection connection = mock(HttpConnection.class); + when(connection.getCreatedTimeStamp()).thenReturn(System.currentTimeMillis()); + return connection; + default: + return null; + } } @Override diff --git a/juniper/src/test/matchobjectTest.cpp b/juniper/src/test/matchobjectTest.cpp index a0e96a3ca2f..1aa05317616 100644 --- a/juniper/src/test/matchobjectTest.cpp +++ b/juniper/src/test/matchobjectTest.cpp @@ -1,21 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -/** - * Implementation of the automated unit test class for the MatchObject - * class. - * - * @file matchobjectTest.cpp - * - * @author Knut Omang - * - * @date Created 21 Feb 2003 - * - * $Id$ - * - * <pre> - * Copyright (c) : 2003 Fast Search & Transfer ASA - * ALL RIGHTS RESERVED - * </pre> - ***************************************************************************/ +/* + * Author: Knut Omang + */ #include "matchobjectTest.h" #include "fakerewriter.h" diff --git a/juniper/src/test/matchobjectTest.h b/juniper/src/test/matchobjectTest.h index d7cad8f445d..d450281a428 100644 --- a/juniper/src/test/matchobjectTest.h +++ b/juniper/src/test/matchobjectTest.h @@ -1,20 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -/** - * Definition of the automated unit test class for the MatchObject class. - * - * @file matchobjectTest.h - * - * @author Knut Omang - * - * @date Created 21 Feb 2003 - * - * $Id$ - * - * <pre> - * Copyright (c) : 2003 Fast Search & Transfer ASA - * ALL RIGHTS RESERVED - * </pre> - ***************************************************************************/ +/* + * Author: Knut Omang + */ #pragma once #include "testenv.h" diff --git a/juniper/src/test/mcandTest.cpp b/juniper/src/test/mcandTest.cpp index c82f4bc44b8..549397c90f4 100644 --- a/juniper/src/test/mcandTest.cpp +++ b/juniper/src/test/mcandTest.cpp @@ -1,21 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -/** - * Implementation of the automated unit test class for the MatchCandidate - * class. - * - * @file mcandTest.cpp - * - * @author Knut Omang - * - * @date Created 27 Feb 2003 - * - * $Id$ - * - * <pre> - * Copyright (c) : 2003 Fast Search & Transfer ASA - * ALL RIGHTS RESERVED - * </pre> - ***************************************************************************/ +/* + * Author: Knut Omang + */ #include "mcandTest.h" diff --git a/juniper/src/test/mcandTest.h b/juniper/src/test/mcandTest.h index 4341b670346..08b6c4c224b 100644 --- a/juniper/src/test/mcandTest.h +++ b/juniper/src/test/mcandTest.h @@ -1,21 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -/** - * Definition of the automated unit test class for the MatchCandidate - * class. - * - * @file mcandTest.h - * - * @author Knut Omang - * - * @date Created 27 Feb 2003 - * - * $Id$ - * - * <pre> - * Copyright (c) : 2003 Fast Search & Transfer ASA - * ALL RIGHTS RESERVED - * </pre> - ***************************************************************************/ +/* + * Author Knut Omang + */ #pragma once #include <map> diff --git a/juniper/src/test/queryparserTest.cpp b/juniper/src/test/queryparserTest.cpp index feffb5b9fa9..7fcf4a5be48 100644 --- a/juniper/src/test/queryparserTest.cpp +++ b/juniper/src/test/queryparserTest.cpp @@ -1,21 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -/** - * Implementation of the automated unit test class for the QueryParser - * class. - * - * @file queryparserTest.cpp - * - * @author Knut Omang - * - * @date Created 24 Feb 2003 - * - * $Id$ - * - * <pre> - * Copyright (c) : 2003 Fast Search & Transfer ASA - * ALL RIGHTS RESERVED - * </pre> - ***************************************************************************/ +/* + * Author Knut Omang + */ #include "queryparserTest.h" #include "fakerewriter.h" diff --git a/juniper/src/test/queryparserTest.h b/juniper/src/test/queryparserTest.h index a1eeb4d2a46..612c62e56d4 100644 --- a/juniper/src/test/queryparserTest.h +++ b/juniper/src/test/queryparserTest.h @@ -1,20 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -/** - * Definition of the automated unit test class for the QueryParser class. - * - * @file queryparserTest.h - * - * @author Knut Omang - * - * @date Created 24 Feb 2003 - * - * $Id$ - * - * <pre> - * Copyright (c) : 2003 Fast Search & Transfer ASA - * ALL RIGHTS RESERVED - * </pre> - ***************************************************************************/ +/* + * Author Knut Omang + */ #pragma once #include "testenv.h" diff --git a/linguistics/src/main/java/com/yahoo/language/LinguisticsCase.java b/linguistics/src/main/java/com/yahoo/language/LinguisticsCase.java index 0593055eecd..37b9463fddb 100644 --- a/linguistics/src/main/java/com/yahoo/language/LinguisticsCase.java +++ b/linguistics/src/main/java/com/yahoo/language/LinguisticsCase.java @@ -9,7 +9,7 @@ import java.util.Locale; * This class provides a case normalization operation to be used e.g. when * document search should be case insensitive. * - * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + * @author Simon Thoresen */ public class LinguisticsCase { diff --git a/logd/src/logd/CMakeLists.txt b/logd/src/logd/CMakeLists.txt index 6b46ef50b92..03ade1d66c8 100644 --- a/logd/src/logd/CMakeLists.txt +++ b/logd/src/logd/CMakeLists.txt @@ -12,4 +12,4 @@ vespa_add_library(logd STATIC DEPENDS ) vespa_generate_config(logd ../main/resources/configdefinitions/logd.def) -install(FILES ../main/resources/configdefinitions/logd.def DESTINATION var/db/vespa/config_server/serverdb/classes) +install(FILES ../main/resources/configdefinitions/logd.def RENAME cloud.config.log.logd.def DESTINATION var/db/vespa/config_server/serverdb/classes) diff --git a/messagebus/src/main/java/com/yahoo/messagebus/network/rpc/RPCNetwork.java b/messagebus/src/main/java/com/yahoo/messagebus/network/rpc/RPCNetwork.java index 096c0c0b485..96a0a5fecef 100644 --- a/messagebus/src/main/java/com/yahoo/messagebus/network/rpc/RPCNetwork.java +++ b/messagebus/src/main/java/com/yahoo/messagebus/network/rpc/RPCNetwork.java @@ -2,17 +2,28 @@ package com.yahoo.messagebus.network.rpc; import com.yahoo.component.Version; -import com.yahoo.component.VersionSpecification; import com.yahoo.component.Vtag; import com.yahoo.concurrent.ThreadFactoryFactory; -import com.yahoo.jrt.*; +import com.yahoo.jrt.Acceptor; +import com.yahoo.jrt.ListenFailedException; +import com.yahoo.jrt.Method; +import com.yahoo.jrt.MethodHandler; +import com.yahoo.jrt.Request; +import com.yahoo.jrt.Spec; +import com.yahoo.jrt.StringValue; +import com.yahoo.jrt.Supervisor; +import com.yahoo.jrt.Task; +import com.yahoo.jrt.Transport; import com.yahoo.jrt.slobrok.api.IMirror; import com.yahoo.jrt.slobrok.api.Mirror; import com.yahoo.jrt.slobrok.api.Register; import com.yahoo.log.LogLevel; -import com.yahoo.messagebus.*; +import com.yahoo.messagebus.EmptyReply; import com.yahoo.messagebus.Error; import com.yahoo.messagebus.ErrorCode; +import com.yahoo.messagebus.Message; +import com.yahoo.messagebus.Protocol; +import com.yahoo.messagebus.Reply; import com.yahoo.messagebus.network.Identity; import com.yahoo.messagebus.network.Network; import com.yahoo.messagebus.network.NetworkOwner; @@ -22,8 +33,16 @@ import com.yahoo.messagebus.routing.RoutingNode; import java.io.PrintWriter; import java.io.StringWriter; -import java.util.*; -import java.util.concurrent.*; +import java.util.LinkedHashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.TreeMap; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.SynchronousQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.logging.Logger; @@ -44,7 +63,7 @@ public class RPCNetwork implements Network, MethodHandler { private final Acceptor listener; private final Mirror mirror; private final Register register; - private final Map<VersionSpecification, RPCSendAdapter> sendAdapters = new HashMap<>(); + private final TreeMap<Version, RPCSendAdapter> sendAdapters = new TreeMap<>(); private NetworkOwner owner; private final SlobrokConfigSubscriber slobroksConfig; private final LinkedHashMap<String, Route> lruRouteMap = new LinkedHashMap<>(10000, 0.5f, true); @@ -162,9 +181,10 @@ public class RPCNetwork implements Network, MethodHandler { } this.owner = owner; - RPCSendAdapter adapter = new RPCSendV1(); - addSendAdapter(new VersionSpecification(5), adapter); - addSendAdapter(new VersionSpecification(6), adapter); + RPCSendAdapter adapter1 = new RPCSendV1(); + RPCSendAdapter adapter2 = new RPCSendV2(); + addSendAdapter(new Version(5), adapter1); + addSendAdapter(new Version(6,149), adapter2); } @Override @@ -234,11 +254,9 @@ public class RPCNetwork implements Network, MethodHandler { */ private void send(SendContext ctx) { if (destroyed.get()) { - replyError(ctx, ErrorCode.NETWORK_SHUTDOWN, - "Network layer has performed shutdown."); + replyError(ctx, ErrorCode.NETWORK_SHUTDOWN, "Network layer has performed shutdown."); } else if (ctx.hasError) { - replyError(ctx, ErrorCode.HANDSHAKE_FAILED, - "An error occured while resolving version."); + replyError(ctx, ErrorCode.HANDSHAKE_FAILED, "An error occured while resolving version."); } else { sendService.execute(new SendTask(owner.getProtocol(ctx.msg.getProtocol()), ctx)); } @@ -315,7 +333,7 @@ public class RPCNetwork implements Network, MethodHandler { * @param version The version for which to register an adapter. * @param adapter The adapter to register. */ - private void addSendAdapter(VersionSpecification version, RPCSendAdapter adapter) { + private void addSendAdapter(Version version, RPCSendAdapter adapter) { adapter.attach(this); sendAdapters.put(version, adapter); } @@ -327,13 +345,9 @@ public class RPCNetwork implements Network, MethodHandler { * @param version The version for which to return an adapter. * @return The compatible adapter. */ - private RPCSendAdapter getSendAdapter(Version version) { - for (Map.Entry<VersionSpecification, RPCSendAdapter> entry : sendAdapters.entrySet()) { - if (entry.getKey().matches(version)) { - return entry.getValue(); - } - } - return null; + public RPCSendAdapter getSendAdapter(Version version) { + Map.Entry<Version, RPCSendAdapter> lower = sendAdapters.floorEntry(version); + return (lower != null) ? lower.getValue() : null; } /** diff --git a/messagebus/src/main/java/com/yahoo/messagebus/network/rpc/RPCSend.java b/messagebus/src/main/java/com/yahoo/messagebus/network/rpc/RPCSend.java new file mode 100644 index 00000000000..d7b4887bd36 --- /dev/null +++ b/messagebus/src/main/java/com/yahoo/messagebus/network/rpc/RPCSend.java @@ -0,0 +1,269 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.messagebus.network.rpc; + +import com.yahoo.component.Version; + +import com.yahoo.jrt.Method; +import com.yahoo.jrt.MethodHandler; +import com.yahoo.jrt.Request; +import com.yahoo.jrt.RequestWaiter; +import com.yahoo.jrt.Values; +import com.yahoo.messagebus.EmptyReply; +import com.yahoo.messagebus.Error; +import com.yahoo.messagebus.ErrorCode; +import com.yahoo.messagebus.Message; +import com.yahoo.messagebus.Protocol; +import com.yahoo.messagebus.Reply; +import com.yahoo.messagebus.ReplyHandler; +import com.yahoo.messagebus.Routable; +import com.yahoo.messagebus.Trace; +import com.yahoo.messagebus.TraceLevel; +import com.yahoo.messagebus.routing.Hop; +import com.yahoo.messagebus.routing.Route; +import com.yahoo.messagebus.routing.RoutingNode; +import com.yahoo.text.Utf8Array; + +/** + * Implements the request adapter for method "mbus.send1/mbus.slime". + * + * @author baldersheim + */ +public abstract class RPCSend implements MethodHandler, ReplyHandler, RequestWaiter, RPCSendAdapter { + + private RPCNetwork net = null; + private String clientIdent = "client"; + private String serverIdent = "server"; + + protected abstract Method buildMethod(); + protected abstract String getReturnSpec(); + protected abstract Request encodeRequest(Version version, Route route, RPCServiceAddress address, Message msg, + long timeRemaining, byte[] payload, int traceLevel); + protected abstract Reply createReply(Values ret, String serviceName, Trace trace); + protected abstract Params toParams(Values req); + protected abstract void createResponse(Values ret, Reply reply, Version version, byte [] payload); + @Override + public final void attach(RPCNetwork net) { + this.net = net; + String prefix = net.getIdentity().getServicePrefix(); + if (prefix != null && prefix.length() > 0) { + clientIdent = "'" + prefix + "'"; + serverIdent = clientIdent; + } + net.getSupervisor().addMethod(buildMethod()); + } + + @Override + public final void send(RoutingNode recipient, Version version, byte[] payload, long timeRemaining) { + SendContext ctx = new SendContext(recipient, timeRemaining); + RPCServiceAddress address = (RPCServiceAddress)recipient.getServiceAddress(); + Message msg = recipient.getMessage(); + Route route = new Route(recipient.getRoute()); + Hop hop = route.removeHop(0); + + Request req = encodeRequest(version, route, address,msg, timeRemaining, payload, ctx.trace.getLevel()); + + if (ctx.trace.shouldTrace(TraceLevel.SEND_RECEIVE)) { + ctx.trace.trace(TraceLevel.SEND_RECEIVE, + "Sending message (version " + version + ") from " + clientIdent + " to '" + + address.getServiceName() + "' with " + ctx.timeout + " seconds timeout."); + } + + if (hop.getIgnoreResult()) { + address.getTarget().getJRTTarget().invokeVoid(req); + if (ctx.trace.shouldTrace(TraceLevel.SEND_RECEIVE)) { + ctx.trace.trace(TraceLevel.SEND_RECEIVE, + "Not waiting for a reply from '" + address.getServiceName() + "'."); + } + Reply reply = new EmptyReply(); + reply.getTrace().swap(ctx.trace); + net.getOwner().deliverReply(reply, recipient); + } else { + req.setContext(ctx); + address.getTarget().getJRTTarget().invokeAsync(req, ctx.timeout, this); + } + req.discardParameters(); // allow garbage collection of request parameters + } + + protected final Object decode(Utf8Array protocolName, Version version, byte [] payload) { + Protocol protocol = net.getOwner().getProtocol(protocolName); + if (protocol != null) { + Routable routable = protocol.decode(version, payload); + if (routable != null) { + if (routable instanceof Reply) { + return routable; + } else { + return new Error(ErrorCode.DECODE_ERROR, + "Payload decoded to a reply when expecting a message."); + } + } else { + return new Error(ErrorCode.DECODE_ERROR, + "Protocol '" + protocol.getName() + "' failed to decode routable."); + } + } else { + return new Error(ErrorCode.UNKNOWN_PROTOCOL, + "Protocol '" + protocolName + "' is not known by " + serverIdent + "."); + } + } + + @Override + public final void handleRequestDone(Request req) { + SendContext ctx = (SendContext)req.getContext(); + String serviceName = ((RPCServiceAddress)ctx.recipient.getServiceAddress()).getServiceName(); + Reply reply = null; + Error error = null; + if (!req.checkReturnTypes(getReturnSpec())) { + // Map all known JRT errors to the appropriate message bus error. + reply = new EmptyReply(); + switch (req.errorCode()) { + case com.yahoo.jrt.ErrorCode.TIMEOUT: + error = new Error(ErrorCode.TIMEOUT, + "A timeout occured while waiting for '" + serviceName + "' (" + + ctx.timeout + " seconds expired); " + req.errorMessage()); + break; + case com.yahoo.jrt.ErrorCode.CONNECTION: + error = new Error(ErrorCode.CONNECTION_ERROR, + "A connection error occured for '" + serviceName + "'; " + req.errorMessage()); + break; + default: + error = new Error(ErrorCode.NETWORK_ERROR, + "A network error occured for '" + serviceName + "'; " + req.errorMessage()); + } + } else { + reply = createReply(req.returnValues(), serviceName, ctx.trace); + } + if (ctx.trace.shouldTrace(TraceLevel.SEND_RECEIVE)) { + ctx.trace.trace(TraceLevel.SEND_RECEIVE, + "Reply (type " + reply.getType() + ") received at " + clientIdent + "."); + } + reply.getTrace().swap(ctx.trace); + if (error != null) { + reply.addError(error); + } + net.getOwner().deliverReply(reply, ctx.recipient); + } + + protected final class Params { + Version version; + String route; + String session; + boolean retryEnabled; + int retry; + long timeRemaining; + Utf8Array protocolName; + byte [] payload; + int traceLevel; + } + + @Override + public final void invoke(Request request) { + request.detach(); + Params p = toParams(request.parameters()); + + request.discardParameters(); // allow garbage collection of request parameters + + // Make sure that the owner understands the protocol. + Protocol protocol = net.getOwner().getProtocol(p.protocolName); + if (protocol == null) { + replyError(request, p.version, p.traceLevel, + new Error(ErrorCode.UNKNOWN_PROTOCOL, + "Protocol '" + p.protocolName + "' is not known by " + serverIdent + ".")); + return; + } + Routable routable = protocol.decode(p.version, p.payload); + if (routable == null) { + replyError(request, p.version, p.traceLevel, + new Error(ErrorCode.DECODE_ERROR, + "Protocol '" + protocol.getName() + "' failed to decode routable.")); + return; + } + if (routable instanceof Reply) { + replyError(request, p.version, p.traceLevel, + new Error(ErrorCode.DECODE_ERROR, + "Payload decoded to a reply when expecting a message.")); + return; + } + Message msg = (Message)routable; + if (p.route != null && p.route.length() > 0) { + msg.setRoute(net.getRoute(p.route)); + } + msg.setContext(new ReplyContext(request, p.version)); + msg.pushHandler(this); + msg.setRetryEnabled(p.retryEnabled); + msg.setRetry(p.retry); + msg.setTimeReceivedNow(); + msg.setTimeRemaining(p.timeRemaining); + msg.getTrace().setLevel(p.traceLevel); + if (msg.getTrace().shouldTrace(TraceLevel.SEND_RECEIVE)) { + msg.getTrace().trace(TraceLevel.SEND_RECEIVE, + "Message (type " + msg.getType() + ") received at " + serverIdent + " for session '" + p.session + "'."); + } + net.getOwner().deliverMessage(msg, p.session); + } + + @Override + public final void handleReply(Reply reply) { + ReplyContext ctx = (ReplyContext)reply.getContext(); + reply.setContext(null); + + // Add trace information. + if (reply.getTrace().shouldTrace(TraceLevel.SEND_RECEIVE)) { + reply.getTrace().trace(TraceLevel.SEND_RECEIVE, + "Sending reply (version " + ctx.version + ") from " + serverIdent + "."); + } + + // Encode and return the reply through the RPC request. + byte[] payload = new byte[0]; + if (reply.getType() != 0) { + Protocol protocol = net.getOwner().getProtocol(reply.getProtocol()); + if (protocol != null) { + payload = protocol.encode(ctx.version, reply); + } + if (payload == null || payload.length == 0) { + reply.addError(new Error(ErrorCode.ENCODE_ERROR, + "An error occured while encoding the reply.")); + } + } + createResponse(ctx.request.returnValues(), reply, ctx.version, payload); + ctx.request.returnRequest(); + } + + /** + * Send an error reply for a given request. + * + * @param request The JRT request to reply to. + * @param version The version to serialize for. + * @param traceLevel The trace level to set in the reply. + * @param err The error to reply with. + */ + private void replyError(Request request, Version version, int traceLevel, Error err) { + Reply reply = new EmptyReply(); + reply.setContext(new ReplyContext(request, version)); + reply.getTrace().setLevel(traceLevel); + reply.addError(err); + handleReply(reply); + } + + private static class SendContext { + + final RoutingNode recipient; + final Trace trace; + final double timeout; + + SendContext(RoutingNode recipient, long timeRemaining) { + this.recipient = recipient; + trace = new Trace(recipient.getTrace().getLevel()); + timeout = timeRemaining * 0.001; + } + } + + private static class ReplyContext { + + final Request request; + final Version version; + + public ReplyContext(Request request, Version version) { + this.request = request; + this.version = version; + } + } +} diff --git a/messagebus/src/main/java/com/yahoo/messagebus/network/rpc/RPCSendV1.java b/messagebus/src/main/java/com/yahoo/messagebus/network/rpc/RPCSendV1.java index 40cb7fb9ee9..480a716e382 100755 --- a/messagebus/src/main/java/com/yahoo/messagebus/network/rpc/RPCSendV1.java +++ b/messagebus/src/main/java/com/yahoo/messagebus/network/rpc/RPCSendV1.java @@ -2,40 +2,41 @@ package com.yahoo.messagebus.network.rpc; import com.yahoo.component.Version; -import com.yahoo.jrt.*; +import com.yahoo.jrt.DataValue; +import com.yahoo.jrt.DoubleValue; +import com.yahoo.jrt.Int32Array; +import com.yahoo.jrt.Int32Value; +import com.yahoo.jrt.Int64Value; +import com.yahoo.jrt.Int8Value; +import com.yahoo.jrt.Method; +import com.yahoo.jrt.Request; +import com.yahoo.jrt.StringArray; import com.yahoo.jrt.StringValue; -import com.yahoo.messagebus.*; +import com.yahoo.jrt.Values; +import com.yahoo.messagebus.EmptyReply; import com.yahoo.messagebus.Error; -import com.yahoo.messagebus.ErrorCode; -import com.yahoo.messagebus.ReplyHandler; -import com.yahoo.messagebus.routing.Hop; +import com.yahoo.messagebus.Message; +import com.yahoo.messagebus.Reply; +import com.yahoo.messagebus.Trace; +import com.yahoo.messagebus.TraceNode; import com.yahoo.messagebus.routing.Route; -import com.yahoo.messagebus.routing.RoutingNode; import com.yahoo.text.Utf8Array; -import com.yahoo.text.Utf8String; /** * Implements the request adapter for method "mbus.send1". * * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> */ -public class RPCSendV1 implements MethodHandler, ReplyHandler, RequestWaiter, RPCSendAdapter { +public class RPCSendV1 extends RPCSend { private final String METHOD_NAME = "mbus.send1"; private final String METHOD_PARAMS = "sssbilsxi"; private final String METHOD_RETURN = "sdISSsxs"; - private RPCNetwork net = null; - private String clientIdent = "client"; - private String serverIdent = "server"; @Override - public void attach(RPCNetwork net) { - this.net = net; - String prefix = net.getIdentity().getServicePrefix(); - if (prefix != null && prefix.length() > 0) { - clientIdent = "'" + prefix + "'"; - serverIdent = clientIdent; - } + protected String getReturnSpec() { return METHOD_RETURN; } + @Override + protected Method buildMethod() { Method method = new Method(METHOD_NAME, METHOD_PARAMS, METHOD_RETURN, this); method.methodDesc("Send a message bus request and get a reply back."); @@ -56,206 +57,80 @@ public class RPCSendV1 implements MethodHandler, ReplyHandler, RequestWaiter, RP .returnDesc(5, "protocol", "The name of the protocol that knows how to decode this reply.") .returnDesc(6, "payload", "The protocol specific reply payload.") .returnDesc(7, "trace", "A string representation of the trace."); - net.getSupervisor().addMethod(method); + return method; } - @Override - public void send(RoutingNode recipient, Version version, byte[] payload, long timeRemaining) { - SendContext ctx = new SendContext(recipient, timeRemaining); - RPCServiceAddress address = (RPCServiceAddress)recipient.getServiceAddress(); - Message msg = recipient.getMessage(); - Route route = new Route(recipient.getRoute()); - Hop hop = route.removeHop(0); - + protected Request encodeRequest(Version version, Route route, RPCServiceAddress address, Message msg, + long timeRemaining, byte[] payload, int traceLevel) { Request req = new Request(METHOD_NAME); - req.parameters().add(new StringValue(version.toString())); - req.parameters().add(new StringValue(route.toString())); - req.parameters().add(new StringValue(address.getSessionName())); - req.parameters().add(new Int8Value(msg.getRetryEnabled() ? (byte)1 : (byte)0)); - req.parameters().add(new Int32Value(msg.getRetry())); - req.parameters().add(new Int64Value(timeRemaining)); - req.parameters().add(new StringValue(msg.getProtocol())); - req.parameters().add(new DataValue(payload)); - req.parameters().add(new Int32Value(ctx.trace.getLevel())); - - if (ctx.trace.shouldTrace(TraceLevel.SEND_RECEIVE)) { - ctx.trace.trace(TraceLevel.SEND_RECEIVE, - "Sending message (version " + version + ") from " + clientIdent + " to '" + - address.getServiceName() + "' with " + ctx.timeout + " seconds timeout."); - } - - if (hop.getIgnoreResult()) { - address.getTarget().getJRTTarget().invokeVoid(req); - if (ctx.trace.shouldTrace(TraceLevel.SEND_RECEIVE)) { - ctx.trace.trace(TraceLevel.SEND_RECEIVE, - "Not waiting for a reply from '" + address.getServiceName() + "'."); - } - Reply reply = new EmptyReply(); - reply.getTrace().swap(ctx.trace); - net.getOwner().deliverReply(reply, recipient); - } else { - req.setContext(ctx); - address.getTarget().getJRTTarget().invokeAsync(req, ctx.timeout, this); - } - req.discardParameters(); // allow garbage collection of request parameters + Values v = req.parameters(); + v.add(new StringValue(version.toString())); + v.add(new StringValue(route.toString())); + v.add(new StringValue(address.getSessionName())); + v.add(new Int8Value(msg.getRetryEnabled() ? (byte)1 : (byte)0)); + v.add(new Int32Value(msg.getRetry())); + v.add(new Int64Value(timeRemaining)); + v.add(new StringValue(msg.getProtocol())); + v.add(new DataValue(payload)); + v.add(new Int32Value(traceLevel)); + return req; } @Override - public void handleRequestDone(Request req) { - SendContext ctx = (SendContext)req.getContext(); - String serviceName = ((RPCServiceAddress)ctx.recipient.getServiceAddress()).getServiceName(); + protected Reply createReply(Values ret, String serviceName, Trace trace) { + Version version = new Version(ret.get(0).asUtf8Array()); + double retryDelay = ret.get(1).asDouble(); + int[] errorCodes = ret.get(2).asInt32Array(); + String[] errorMessages = ret.get(3).asStringArray(); + String[] errorServices = ret.get(4).asStringArray(); + Utf8Array protocolName = ret.get(5).asUtf8Array(); + byte[] payload = ret.get(6).asData(); + String replyTrace = ret.get(7).asString(); + + // Make sure that the owner understands the protocol. Reply reply = null; Error error = null; - if (!req.checkReturnTypes(METHOD_RETURN)) { - // Map all known JRT errors to the appropriate message bus error. - reply = new EmptyReply(); - switch (req.errorCode()) { - case com.yahoo.jrt.ErrorCode.TIMEOUT: - error = new Error(com.yahoo.messagebus.ErrorCode.TIMEOUT, - "A timeout occured while waiting for '" + serviceName + "' (" + - ctx.timeout + " seconds expired); " + req.errorMessage()); - break; - case com.yahoo.jrt.ErrorCode.CONNECTION: - error = new Error(com.yahoo.messagebus.ErrorCode.CONNECTION_ERROR, - "A connection error occured for '" + serviceName + "'; " + req.errorMessage()); - break; - default: - error = new Error(com.yahoo.messagebus.ErrorCode.NETWORK_ERROR, - "A network error occured for '" + serviceName + "'; " + req.errorMessage()); - } - } else { - // Retrieve all reply components from JRT request object. - Version version = new Version(req.returnValues().get(0).asUtf8Array()); - double retryDelay = req.returnValues().get(1).asDouble(); - int[] errorCodes = req.returnValues().get(2).asInt32Array(); - String[] errorMessages = req.returnValues().get(3).asStringArray(); - String[] errorServices = req.returnValues().get(4).asStringArray(); - Utf8Array protocolName = req.returnValues().get(5).asUtf8Array(); - byte[] payload = req.returnValues().get(6).asData(); - String replyTrace = req.returnValues().get(7).asString(); - - // Make sure that the owner understands the protocol. - if (payload.length > 0) { - Protocol protocol = net.getOwner().getProtocol(protocolName); - if (protocol != null) { - Routable routable = protocol.decode(version, payload); - if (routable != null) { - if (routable instanceof Reply) { - reply = (Reply)routable; - } else { - error = new Error(com.yahoo.messagebus.ErrorCode.DECODE_ERROR, - "Payload decoded to a reply when expecting a message."); - } - } else { - error = new Error(com.yahoo.messagebus.ErrorCode.DECODE_ERROR, - "Protocol '" + protocol.getName() + "' failed to decode routable."); - } - } else { - error = new Error(com.yahoo.messagebus.ErrorCode.UNKNOWN_PROTOCOL, - "Protocol '" + protocolName + "' is not known by " + serverIdent + "."); - } - } - if (reply == null) { - reply = new EmptyReply(); - } - reply.setRetryDelay(retryDelay); - for (int i = 0; i < errorCodes.length && i < errorMessages.length; i++) { - reply.addError(new Error(errorCodes[i], - errorMessages[i], - errorServices[i].length() > 0 ? errorServices[i] : serviceName)); - } - if (ctx.trace.getLevel() > 0) { - ctx.trace.getRoot().addChild(TraceNode.decode(replyTrace)); + if (payload.length > 0) { + Object retval = decode(protocolName, version, payload); + if (retval instanceof Reply) { + reply = (Reply) retval; + } else { + error = (Error) retval; } } - if (ctx.trace.shouldTrace(TraceLevel.SEND_RECEIVE)) { - ctx.trace.trace(TraceLevel.SEND_RECEIVE, - "Reply (type " + reply.getType() + ") received at " + clientIdent + "."); + if (reply == null) { + reply = new EmptyReply(); } - reply.getTrace().swap(ctx.trace); if (error != null) { reply.addError(error); } - net.getOwner().deliverReply(reply, ctx.recipient); - } - - @Override - public void invoke(Request request) { - request.detach(); - Version version = new Version(request.parameters().get(0).asUtf8Array()); - String route = request.parameters().get(1).asString(); - String session = request.parameters().get(2).asString(); - boolean retryEnabled = (request.parameters().get(3).asInt8() != 0); - int retry = request.parameters().get(4).asInt32(); - long timeRemaining = request.parameters().get(5).asInt64(); - Utf8Array protocolName = request.parameters().get(6).asUtf8Array(); - byte[] payload = request.parameters().get(7).asData(); - int traceLevel = request.parameters().get(8).asInt32(); - - request.discardParameters(); // allow garbage collection of request parameters - - // Make sure that the owner understands the protocol. - Protocol protocol = net.getOwner().getProtocol(protocolName); - if (protocol == null) { - replyError(request, version, traceLevel, - new com.yahoo.messagebus.Error(ErrorCode.UNKNOWN_PROTOCOL, - "Protocol '" + protocolName + "' is not known by " + serverIdent + ".")); - return; - } - Routable routable = protocol.decode(version, payload); - if (routable == null) { - replyError(request, version, traceLevel, - new Error(ErrorCode.DECODE_ERROR, - "Protocol '" + protocol.getName() + "' failed to decode routable.")); - return; - } - if (routable instanceof Reply) { - replyError(request, version, traceLevel, - new Error(ErrorCode.DECODE_ERROR, - "Payload decoded to a reply when expecting a message.")); - return; + reply.setRetryDelay(retryDelay); + for (int i = 0; i < errorCodes.length && i < errorMessages.length; i++) { + reply.addError(new Error(errorCodes[i], errorMessages[i], + errorServices[i].length() > 0 ? errorServices[i] : serviceName)); } - Message msg = (Message)routable; - if (route != null && route.length() > 0) { - msg.setRoute(net.getRoute(route)); + if (trace.getLevel() > 0) { + trace.getRoot().addChild(TraceNode.decode(replyTrace)); } - msg.setContext(new ReplyContext(request, version)); - msg.pushHandler(this); - msg.setRetryEnabled(retryEnabled); - msg.setRetry(retry); - msg.setTimeReceivedNow(); - msg.setTimeRemaining(timeRemaining); - msg.getTrace().setLevel(traceLevel); - if (msg.getTrace().shouldTrace(TraceLevel.SEND_RECEIVE)) { - msg.getTrace().trace(TraceLevel.SEND_RECEIVE, - "Message (type " + msg.getType() + ") received at " + serverIdent + " for session '" + session + "'."); - } - net.getOwner().deliverMessage(msg, session); + return reply; } - @Override - public void handleReply(Reply reply) { - ReplyContext ctx = (ReplyContext)reply.getContext(); - reply.setContext(null); - - // Add trace information. - if (reply.getTrace().shouldTrace(TraceLevel.SEND_RECEIVE)) { - reply.getTrace().trace(TraceLevel.SEND_RECEIVE, - "Sending reply (version " + ctx.version + ") from " + serverIdent + "."); - } + protected Params toParams(Values args) { + Params p = new Params(); + p.version = new Version(args.get(0).asUtf8Array()); + p.route = args.get(1).asString(); + p.session = args.get(2).asString(); + p.retryEnabled = (args.get(3).asInt8() != 0); + p.retry = args.get(4).asInt32(); + p.timeRemaining = args.get(5).asInt64(); + p.protocolName = args.get(6).asUtf8Array(); + p.payload = args.get(7).asData(); + p.traceLevel = args.get(8).asInt32(); + return p; + } - // Encode and return the reply through the RPC request. - byte[] payload = new byte[0]; - if (reply.getType() != 0) { - Protocol protocol = net.getOwner().getProtocol(reply.getProtocol()); - if (protocol != null) { - payload = protocol.encode(ctx.version, reply); - } - if (payload == null || payload.length == 0) { - reply.addError(new Error(ErrorCode.ENCODE_ERROR, - "An error occured while encoding the reply.")); - } - } + @Override + protected void createResponse(Values ret, Reply reply, Version version, byte [] payload) { int[] eCodes = new int[reply.getNumErrors()]; String[] eMessages = new String[reply.getNumErrors()]; String[] eServices = new String[reply.getNumErrors()]; @@ -265,57 +140,14 @@ public class RPCSendV1 implements MethodHandler, ReplyHandler, RequestWaiter, RP eMessages[i] = error.getMessage(); eServices[i] = error.getService() != null ? error.getService() : ""; } - ctx.request.returnValues().add(new StringValue(ctx.version.toString())); - ctx.request.returnValues().add(new DoubleValue(reply.getRetryDelay())); - ctx.request.returnValues().add(new Int32Array(eCodes)); - ctx.request.returnValues().add(new StringArray(eMessages)); - ctx.request.returnValues().add(new StringArray(eServices)); - ctx.request.returnValues().add(new StringValue(reply.getProtocol())); - ctx.request.returnValues().add(new DataValue(payload)); - ctx.request.returnValues().add(new StringValue( - reply.getTrace().getRoot() != null ? - reply.getTrace().getRoot().encode() : - "")); - ctx.request.returnRequest(); + ret.add(new StringValue(version.toString())); + ret.add(new DoubleValue(reply.getRetryDelay())); + ret.add(new Int32Array(eCodes)); + ret.add(new StringArray(eMessages)); + ret.add(new StringArray(eServices)); + ret.add(new StringValue(reply.getProtocol())); + ret.add(new DataValue(payload)); + ret.add(new StringValue(reply.getTrace().getRoot() != null ? reply.getTrace().getRoot().encode() : "")); } - /** - * Send an error reply for a given request. - * - * @param request The JRT request to reply to. - * @param version The version to serialize for. - * @param traceLevel The trace level to set in the reply. - * @param err The error to reply with. - */ - private void replyError(Request request, Version version, int traceLevel, Error err) { - Reply reply = new EmptyReply(); - reply.setContext(new ReplyContext(request, version)); - reply.getTrace().setLevel(traceLevel); - reply.addError(err); - handleReply(reply); - } - - private static class SendContext { - - final RoutingNode recipient; - final Trace trace; - final double timeout; - - SendContext(RoutingNode recipient, long timeRemaining) { - this.recipient = recipient; - trace = new Trace(recipient.getTrace().getLevel()); - timeout = timeRemaining * 0.001; - } - } - - private static class ReplyContext { - - final Request request; - final Version version; - - public ReplyContext(Request request, Version version) { - this.request = request; - this.version = version; - } - } } diff --git a/messagebus/src/main/java/com/yahoo/messagebus/network/rpc/RPCSendV2.java b/messagebus/src/main/java/com/yahoo/messagebus/network/rpc/RPCSendV2.java new file mode 100644 index 00000000000..8cc0b73ae30 --- /dev/null +++ b/messagebus/src/main/java/com/yahoo/messagebus/network/rpc/RPCSendV2.java @@ -0,0 +1,209 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.messagebus.network.rpc; + +import com.yahoo.component.Version; +import com.yahoo.compress.CompressionType; +import com.yahoo.compress.Compressor; +import com.yahoo.jrt.DataValue; +import com.yahoo.jrt.Int32Value; +import com.yahoo.jrt.Int8Value; +import com.yahoo.jrt.Method; +import com.yahoo.jrt.Request; +import com.yahoo.jrt.Values; +import com.yahoo.messagebus.EmptyReply; +import com.yahoo.messagebus.Error; +import com.yahoo.messagebus.Message; +import com.yahoo.messagebus.Reply; +import com.yahoo.messagebus.Trace; +import com.yahoo.messagebus.TraceNode; +import com.yahoo.messagebus.routing.Route; +import com.yahoo.slime.BinaryFormat; +import com.yahoo.slime.Cursor; +import com.yahoo.slime.Inspector; +import com.yahoo.slime.Slime; +import com.yahoo.text.Utf8; +import com.yahoo.text.Utf8Array; + +/** + * Implements the request adapter for method "mbus.slime". + * + * @author baldersheim + */ +public class RPCSendV2 extends RPCSend { + + private final static String METHOD_NAME = "mbus.slime"; + private final static String METHOD_PARAMS = "bixbix"; + private final static String METHOD_RETURN = "bixbix"; + private final Compressor compressor = new Compressor(CompressionType.LZ4, 3, 90, 1024); + + @Override + protected String getReturnSpec() { return METHOD_RETURN; } + @Override + protected Method buildMethod() { + + Method method = new Method(METHOD_NAME, METHOD_PARAMS, METHOD_RETURN, this); + method.methodDesc("Send a message bus request and get a reply back."); + method.paramDesc(0, "header_encoding", "Encoding type of header.") + .paramDesc(1, "header_decodedSize", "Number of bytes after header decoding.") + .paramDesc(2, "header_payload", "Slime encoded header payload.") + .paramDesc(3, "body_encoding", "Encoding type of body.") + .paramDesc(4, "body_decoded_ize", "Number of bytes after body decoding.") + .paramDesc(5, "body_payload", "Slime encoded body payload."); + method.returnDesc(0, "header_encoding", "Encoding type of header.") + .returnDesc(1, "header_decoded_size", "Number of bytes after header decoding.") + .returnDesc(2, "header_payload", "Slime encoded header payload.") + .returnDesc(3, "body_encoding", "Encoding type of body.") + .returnDesc(4, "body_encoded_size", "Number of bytes after body decoding.") + .returnDesc(5, "body_payload", "Slime encoded body payload."); + return method; + } + private static final String VERSION_F = new String("version"); + private static final String ROUTE_F = new String("route"); + private static final String SESSION_F = new String("session"); + private static final String PROTOCOL_F = new String("prot"); + private static final String TRACELEVEL_F = new String("tracelevel"); + private static final String TRACE_F = new String("trace"); + private static final String USERETRY_F = new String("useretry"); + private static final String RETRY_F = new String("retry"); + private static final String RETRYDELAY_F = new String("retrydelay"); + private static final String TIMEREMAINING_F = new String("timeleft"); + private static final String ERRORS_F = new String("errors"); + private static final String SERVICE_F = new String("service"); + private static final String CODE_F = new String("code"); + private static final String BLOB_F = new String("msg"); + private static final String MSG_F = new String("msg"); + + @Override + protected Request encodeRequest(Version version, Route route, RPCServiceAddress address, Message msg, + long timeRemaining, byte[] payload, int traceLevel) + { + + Request req = new Request(METHOD_NAME); + Values v = req.parameters(); + + v.add(new Int8Value(CompressionType.NONE.getCode())); + v.add(new Int32Value(0)); + v.add(new DataValue(new byte[0])); + + Slime slime = new Slime(); + Cursor root = slime.setObject(); + + root.setString(VERSION_F, version.toString()); + root.setString(ROUTE_F, route.toString()); + root.setString(SESSION_F, address.getSessionName()); + root.setString(PROTOCOL_F, msg.getProtocol().toString()); + root.setBool(USERETRY_F, msg.getRetryEnabled()); + root.setLong(RETRY_F, msg.getRetry()); + root.setLong(TIMEREMAINING_F, msg.getTimeRemaining()); + root.setLong(TRACELEVEL_F, traceLevel); + root.setData(BLOB_F, payload); + + byte[] serializedSlime = BinaryFormat.encode(slime); + Compressor.Compression compressionResult = compressor.compress(serializedSlime); + + v.add(new Int8Value(compressionResult.type().getCode())); + v.add(new Int32Value(compressionResult.uncompressedSize())); + v.add(new DataValue(compressionResult.data())); + + return req; + } + + @Override + protected Reply createReply(Values ret, String serviceName, Trace trace) { + CompressionType compression = CompressionType.valueOf(ret.get(3).asInt8()); + byte[] slimeBytes = compressor.decompress(ret.get(5).asData(), compression, ret.get(4).asInt32()); + Slime slime = BinaryFormat.decode(slimeBytes); + Inspector root = slime.get(); + + Version version = new Version(root.field(VERSION_F).asString()); + byte[] payload = root.field(BLOB_F).asData(); + + // Make sure that the owner understands the protocol. + Reply reply = null; + Error error = null; + if (payload.length > 0) { + Object retval = decode(new Utf8Array(root.field(PROTOCOL_F).asUtf8()), version, payload); + if (retval instanceof Reply) { + reply = (Reply) retval; + } else { + error = (Error) retval; + } + } + if (reply == null) { + reply = new EmptyReply(); + } + if (error != null) { + reply.addError(error); + } + reply.setRetryDelay(root.field(RETRYDELAY_F).asDouble()); + + Inspector errors = root.field(ERRORS_F); + for (int i = 0; i < errors.entries(); i++) { + Inspector e = errors.entry(i); + String service = e.field(SERVICE_F).asString(); + reply.addError(new Error((int)e.field(CODE_F).asLong(), e.field(MSG_F).asString(), + (service != null && service.length() > 0) ? service : serviceName)); + } + if (trace.getLevel() > 0) { + trace.getRoot().addChild(TraceNode.decode(root.field(TRACE_F).asString())); + } + return reply; + } + + protected Params toParams(Values args) { + CompressionType compression = CompressionType.valueOf(args.get(3).asInt8()); + byte[] slimeBytes = compressor.decompress(args.get(5).asData(), compression, args.get(4).asInt32()); + Slime slime = BinaryFormat.decode(slimeBytes); + Inspector root = slime.get(); + Params p = new Params(); + p.version = new Version(root.field(VERSION_F).asString()); + p.route = root.field(ROUTE_F).asString(); + p.session = root.field(SESSION_F).asString(); + p.retryEnabled = root.field(USERETRY_F).asBool(); + p.retry = (int)root.field(RETRY_F).asLong(); + p.timeRemaining = root.field(TIMEREMAINING_F).asLong(); + p.protocolName = new Utf8Array(Utf8.toBytes(root.field(PROTOCOL_F).asString())); + p.payload = root.field(BLOB_F).asData(); + p.traceLevel = (int)root.field(TRACELEVEL_F).asLong(); + return p; + } + + @Override + protected void createResponse(Values ret, Reply reply, Version version, byte [] payload) { + ret.add(new Int8Value(CompressionType.NONE.getCode())); + ret.add(new Int32Value(0)); + ret.add(new DataValue(new byte[0])); + + Slime slime = new Slime(); + Cursor root = slime.setObject(); + + root.setString(VERSION_F, version.toString()); + root.setDouble(RETRYDELAY_F, reply.getRetryDelay()); + root.setString(PROTOCOL_F, reply.getProtocol().toString()); + root.setData(BLOB_F, payload); + if (reply.getTrace().getLevel() > 0) { + root.setString(TRACE_F, reply.getTrace().getRoot().encode()); + } + + if (reply.getNumErrors() > 0) { + Cursor array = root.setArray(ERRORS_F); + for (int i = 0; i < reply.getNumErrors(); i++) { + Cursor e = array.addObject(); + Error mbusE = reply.getError(i); + e.setLong(CODE_F, mbusE.getCode()); + e.setString(MSG_F, mbusE.getMessage()); + if (mbusE.getService() != null) { + e.setString(SERVICE_F, mbusE.getService()); + } + } + } + + byte[] serializedSlime = BinaryFormat.encode(slime); + Compressor.Compression compressionResult = compressor.compress(serializedSlime); + + ret.add(new Int8Value(compressionResult.type().getCode())); + ret.add(new Int32Value(compressionResult.uncompressedSize())); + ret.add(new DataValue(compressionResult.data())); + } + +} diff --git a/messagebus/src/test/java/com/yahoo/messagebus/network/rpc/SendAdapterTestCase.java b/messagebus/src/test/java/com/yahoo/messagebus/network/rpc/SendAdapterTestCase.java index 5a88db8c19b..c8185aaeaf4 100755 --- a/messagebus/src/test/java/com/yahoo/messagebus/network/rpc/SendAdapterTestCase.java +++ b/messagebus/src/test/java/com/yahoo/messagebus/network/rpc/SendAdapterTestCase.java @@ -79,8 +79,23 @@ public class SendAdapterTestCase { //////////////////////////////////////////////////////////////////////////////// @Test + public void requireCorrectVersionSelection() { + assertNull(srcServer.net.getSendAdapter(new Version(4,999))); + assertTrue(srcServer.net.getSendAdapter(new Version(5,0)) instanceof RPCSendV1); + assertTrue(srcServer.net.getSendAdapter(new Version(6,148)) instanceof RPCSendV1); + assertTrue(srcServer.net.getSendAdapter(new Version(6,149)) instanceof RPCSendV2); + assertTrue(srcServer.net.getSendAdapter(new Version(9,9999)) instanceof RPCSendV2); + } + + @Test public void requireThatMessagesCanBeSentAcrossAllSupportedVersions() throws Exception { - List<Version> versions = Arrays.asList(new Version(5, 0), new Version(5, 1)); + List<Version> versions = Arrays.asList( + new Version(5, 0), + new Version(6, 148), + new Version(6, 149), + new Version(9, 999) + ); + for (Version srcVersion : versions) { for (Version itrVersion : versions) { for (Version dstVersion : versions) { diff --git a/messagebus/src/tests/advancedrouting/advancedrouting.cpp b/messagebus/src/tests/advancedrouting/advancedrouting.cpp index e847fdf3222..8d6b8cd3875 100644 --- a/messagebus/src/tests/advancedrouting/advancedrouting.cpp +++ b/messagebus/src/tests/advancedrouting/advancedrouting.cpp @@ -1,17 +1,15 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/messagebus/emptyreply.h> -#include <vespa/messagebus/errorcode.h> -#include <vespa/messagebus/messagebus.h> -#include <vespa/messagebus/routing/errordirective.h> -#include <vespa/messagebus/routing/retrytransienterrorspolicy.h> + #include <vespa/messagebus/testlib/custompolicy.h> #include <vespa/messagebus/testlib/receptor.h> #include <vespa/messagebus/testlib/simplemessage.h> -#include <vespa/messagebus/testlib/simpleprotocol.h> #include <vespa/messagebus/testlib/slobrok.h> #include <vespa/messagebus/testlib/testserver.h> #include <vespa/vespalib/testkit/testapp.h> #include <vespa/vespalib/util/stringfmt.h> +#include <vespa/messagebus/emptyreply.h> +#include <vespa/messagebus/errorcode.h> +#include <vespa/messagebus/routing/retrytransienterrorspolicy.h> using namespace mbus; @@ -117,7 +115,7 @@ Test::Main() void Test::testAdvanced(TestData &data) { - const double TIMEOUT=60; + const double TIMEOUT = 60; IProtocol::SP protocol(new SimpleProtocol()); SimpleProtocol &simple = static_cast<SimpleProtocol&>(*protocol); simple.addPolicyFactory("Custom", SimpleProtocol::IPolicyFactory::SP(new CustomPolicyFactory(false, ErrorCode::NO_ADDRESS_FOR_SERVICE))); diff --git a/messagebus/src/tests/bucketsequence/bucketsequence.cpp b/messagebus/src/tests/bucketsequence/bucketsequence.cpp index 4073469c253..d98acc4f191 100644 --- a/messagebus/src/tests/bucketsequence/bucketsequence.cpp +++ b/messagebus/src/tests/bucketsequence/bucketsequence.cpp @@ -5,7 +5,6 @@ #include <vespa/messagebus/testlib/receptor.h> #include <vespa/messagebus/testlib/simplemessage.h> #include <vespa/messagebus/testlib/simpleprotocol.h> -#include <vespa/messagebus/testlib/simplereply.h> #include <vespa/messagebus/testlib/slobrok.h> #include <vespa/messagebus/testlib/testserver.h> #include <vespa/vespalib/testkit/testapp.h> diff --git a/messagebus/src/tests/choke/choke.cpp b/messagebus/src/tests/choke/choke.cpp index b8e20eb9074..91f91c84f45 100644 --- a/messagebus/src/tests/choke/choke.cpp +++ b/messagebus/src/tests/choke/choke.cpp @@ -2,7 +2,6 @@ #include <vespa/messagebus/emptyreply.h> #include <vespa/messagebus/errorcode.h> -#include <vespa/messagebus/reply.h> #include <vespa/messagebus/testlib/receptor.h> #include <vespa/messagebus/testlib/simplemessage.h> #include <vespa/messagebus/testlib/simpleprotocol.h> diff --git a/messagebus/src/tests/error/error.cpp b/messagebus/src/tests/error/error.cpp index dbaa869507b..1d8a489a5ed 100644 --- a/messagebus/src/tests/error/error.cpp +++ b/messagebus/src/tests/error/error.cpp @@ -65,14 +65,14 @@ Test::Main() reply = pxy.getReply(); ASSERT_TRUE(reply.get() != 0); - EXPECT_EQUAL(reply->getNumErrors(), 1u); + ASSERT_EQUAL(reply->getNumErrors(), 1u); EXPECT_EQUAL(reply->getError(0).getService(), "test/dst/session"); reply->addError(Error(ErrorCode::APP_FATAL_ERROR, "fatality")); is->forward(std::move(reply)); reply = src.getReply(); ASSERT_TRUE(reply.get() != 0); - EXPECT_EQUAL(reply->getNumErrors(), 2u); + ASSERT_EQUAL(reply->getNumErrors(), 2u); EXPECT_EQUAL(reply->getError(0).getService(), "test/dst/session"); EXPECT_EQUAL(reply->getError(1).getService(), "test/pxy/session"); } diff --git a/messagebus/src/tests/routing/routing.cpp b/messagebus/src/tests/routing/routing.cpp index e907e0d1163..515cbd99fde 100644 --- a/messagebus/src/tests/routing/routing.cpp +++ b/messagebus/src/tests/routing/routing.cpp @@ -1,7 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/messagebus/emptyreply.h> #include <vespa/messagebus/errorcode.h> -#include <vespa/messagebus/messagebus.h> #include <vespa/messagebus/routing/errordirective.h> #include <vespa/messagebus/routing/retrytransienterrorspolicy.h> #include <vespa/messagebus/routing/routingcontext.h> diff --git a/messagebus/src/tests/sendadapter/sendadapter.cpp b/messagebus/src/tests/sendadapter/sendadapter.cpp index 8ae1d0993cb..6860453cf54 100644 --- a/messagebus/src/tests/sendadapter/sendadapter.cpp +++ b/messagebus/src/tests/sendadapter/sendadapter.cpp @@ -1,12 +1,12 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/messagebus/messagebus.h> #include <vespa/messagebus/testlib/receptor.h> -#include <vespa/messagebus/testlib/simplemessage.h> #include <vespa/messagebus/testlib/simpleprotocol.h> #include <vespa/messagebus/testlib/simplereply.h> #include <vespa/messagebus/testlib/slobrok.h> #include <vespa/messagebus/testlib/testserver.h> +#include <vespa/messagebus/network/rpcsendv1.h> +#include <vespa/messagebus/network/rpcsendv2.h> #include <vespa/vespalib/testkit/testapp.h> #include <vespa/log/log.h> @@ -59,21 +59,7 @@ public: bool start(); }; -class Test : public vespalib::TestApp { -private: - static const int TIMEOUT_SECS = 60; - - bool testVersionedSend(TestData &data, - const vespalib::Version &srcVersion, - const vespalib::Version &itrVersion, - const vespalib::Version &dstVersion); - void testSendAdapters(TestData &data); - -public: - int Main() override; -}; - -TEST_APPHOOK(Test); +static const int TIMEOUT_SECS = 6; TestData::TestData() : _slobrok(), @@ -117,55 +103,8 @@ TestData::start() return true; } -int -Test::Main() -{ - TEST_INIT("sendadapter_test"); - - TestData data; - ASSERT_TRUE(data.start()); - - testSendAdapters(data); TEST_FLUSH(); - - TEST_DONE(); -} - -//////////////////////////////////////////////////////////////////////////////// -// -// Tests -// -//////////////////////////////////////////////////////////////////////////////// - -void -Test::testSendAdapters(TestData &data) -{ - std::vector<vespalib::Version> versions; - versions.push_back(vespalib::Version(5, 0)); - versions.push_back(vespalib::Version(5, 1)); - - for (std::vector<vespalib::Version>::const_iterator srcVersion = versions.begin(); - srcVersion != versions.end(); ++srcVersion) - { - for (std::vector<vespalib::Version>::const_iterator itrVersion = versions.begin(); - itrVersion != versions.end(); ++itrVersion) - { - for (std::vector<vespalib::Version>::const_iterator dstVersion = versions.begin(); - dstVersion != versions.end(); ++dstVersion) - { - EXPECT_TRUE(testVersionedSend(data, *srcVersion, *itrVersion, *dstVersion)); - } - } - } -} - -//////////////////////////////////////////////////////////////////////////////// -// -// Utilities -// -//////////////////////////////////////////////////////////////////////////////// - bool -Test::testVersionedSend(TestData &data, +testVersionedSend(TestData &data, const vespalib::Version &srcVersion, const vespalib::Version &itrVersion, const vespalib::Version &dstVersion) @@ -251,3 +190,38 @@ Test::testVersionedSend(TestData &data, } return true; } + + +void +testSendAdapters(TestData &data, const std::vector<vespalib::Version> & versions) +{ + for (vespalib::Version src : versions) { + for (vespalib::Version intermediate : versions) { + for (vespalib::Version dst : versions) { + EXPECT_TRUE(testVersionedSend(data, src, intermediate, dst)); + } + } + } +} + +TEST("test that all known versions are present") { + TestData data; + ASSERT_TRUE(data.start()); + EXPECT_FALSE(data._srcServer.net.getSendAdapter(vespalib::Version(4, 999)) != nullptr); + EXPECT_TRUE(data._srcServer.net.getSendAdapter(vespalib::Version(5, 0)) != nullptr); + EXPECT_TRUE(dynamic_cast<mbus::RPCSendV1 *>(data._srcServer.net.getSendAdapter(vespalib::Version(5, 0))) != nullptr); + EXPECT_TRUE(data._srcServer.net.getSendAdapter(vespalib::Version(6, 148)) != nullptr); + EXPECT_TRUE(dynamic_cast<mbus::RPCSendV1 *>(data._srcServer.net.getSendAdapter(vespalib::Version(6, 148))) != nullptr); + EXPECT_TRUE(data._srcServer.net.getSendAdapter(vespalib::Version(6, 149)) != nullptr); + EXPECT_TRUE(dynamic_cast<mbus::RPCSendV2 *>(data._srcServer.net.getSendAdapter(vespalib::Version(6, 149))) != nullptr); + EXPECT_TRUE(data._srcServer.net.getSendAdapter(vespalib::Version(9, 999)) != nullptr); + EXPECT_TRUE(dynamic_cast<mbus::RPCSendV2 *>(data._srcServer.net.getSendAdapter(vespalib::Version(9, 999))) != nullptr); +} + +TEST("test that we can send between multiple versions") { + TestData data; + ASSERT_TRUE(data.start()); + TEST_DO(testSendAdapters(data, {vespalib::Version(5, 0), vespalib::Version(6, 148), vespalib::Version(6, 149), vespalib::Version(9, 999)})); +} + +TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/messagebus/src/tests/serviceaddress/serviceaddress.cpp b/messagebus/src/tests/serviceaddress/serviceaddress.cpp index 15a2e9fdac8..89edb981716 100644 --- a/messagebus/src/tests/serviceaddress/serviceaddress.cpp +++ b/messagebus/src/tests/serviceaddress/serviceaddress.cpp @@ -3,20 +3,7 @@ #include <vespa/vespalib/testkit/testapp.h> #include <vespa/messagebus/testlib/slobrok.h> #include <vespa/messagebus/testlib/testserver.h> -#include <vespa/messagebus/testlib/receptor.h> -#include <vespa/messagebus/testlib/simplemessage.h> -#include <vespa/messagebus/testlib/simplereply.h> -#include <vespa/messagebus/testlib/simpleprotocol.h> -#include <vespa/messagebus/messagebus.h> -#include <vespa/messagebus/sourcesession.h> -#include <vespa/messagebus/intermediatesession.h> -#include <vespa/messagebus/destinationsession.h> -#include <vespa/messagebus/emptyreply.h> -#include <vespa/messagebus/error.h> -#include <vespa/messagebus/errorcode.h> -#include <vespa/messagebus/routing/routingspec.h> #include <vespa/messagebus/network/rpcservice.h> -#include <vespa/messagebus/sourcesessionparams.h> using namespace mbus; diff --git a/messagebus/src/tests/servicepool/servicepool.cpp b/messagebus/src/tests/servicepool/servicepool.cpp index c98b342b2f3..86a800b600a 100644 --- a/messagebus/src/tests/servicepool/servicepool.cpp +++ b/messagebus/src/tests/servicepool/servicepool.cpp @@ -1,6 +1,8 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/messagebus/network/rpcnetwork.h> +#include <vespa/messagebus/network/rpcnetworkparams.h> +#include <vespa/messagebus/network/rpcservicepool.h> #include <vespa/messagebus/testlib/slobrok.h> #include <vespa/vespalib/testkit/testapp.h> diff --git a/messagebus/src/tests/shutdown/shutdown.cpp b/messagebus/src/tests/shutdown/shutdown.cpp index b5e6cac970e..070b51bbbc2 100644 --- a/messagebus/src/tests/shutdown/shutdown.cpp +++ b/messagebus/src/tests/shutdown/shutdown.cpp @@ -1,9 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/messagebus/emptyreply.h> -#include <vespa/messagebus/errorcode.h> -#include <vespa/messagebus/messagebus.h> -#include <vespa/messagebus/routing/errordirective.h> #include <vespa/messagebus/routing/retrytransienterrorspolicy.h> #include <vespa/messagebus/testlib/receptor.h> #include <vespa/messagebus/testlib/simplemessage.h> diff --git a/messagebus/src/tests/slobrok/slobrok.cpp b/messagebus/src/tests/slobrok/slobrok.cpp index 360705e7eae..6c389c25e70 100644 --- a/messagebus/src/tests/slobrok/slobrok.cpp +++ b/messagebus/src/tests/slobrok/slobrok.cpp @@ -2,10 +2,10 @@ #include <vespa/vespalib/testkit/testapp.h> #include <vespa/messagebus/testlib/slobrok.h> -#include <string> -#include <sstream> #include <vespa/slobrok/sbmirror.h> #include <vespa/messagebus/network/rpcnetwork.h> +#include <vespa/messagebus/network/rpcnetworkparams.h> + #include <vespa/vespalib/util/host_name.h> using slobrok::api::IMirrorAPI; diff --git a/messagebus/src/vespa/messagebus/CMakeLists.txt b/messagebus/src/vespa/messagebus/CMakeLists.txt index 7b714b88033..7c922f36a8c 100644 --- a/messagebus/src/vespa/messagebus/CMakeLists.txt +++ b/messagebus/src/vespa/messagebus/CMakeLists.txt @@ -37,4 +37,4 @@ vespa_add_library(messagebus DEPENDS ) vespa_generate_config(messagebus ../../main/config/messagebus.def) -install(FILES ../../main/config/messagebus.def DESTINATION var/db/vespa/config_server/serverdb/classes) +install(FILES ../../main/config/messagebus.def RENAME messagebus.messagebus.def DESTINATION var/db/vespa/config_server/serverdb/classes) diff --git a/messagebus/src/vespa/messagebus/blob.h b/messagebus/src/vespa/messagebus/blob.h index 1e84222064a..0509d71d77c 100644 --- a/messagebus/src/vespa/messagebus/blob.h +++ b/messagebus/src/vespa/messagebus/blob.h @@ -26,13 +26,13 @@ public: _payload(Alloc::alloc(s)), _sz(s) { } - Blob(Blob && rhs) : + Blob(Blob && rhs) noexcept : _payload(std::move(rhs._payload)), _sz(rhs._sz) { rhs._sz = 0; } - Blob & operator = (Blob && rhs) { + Blob & operator = (Blob && rhs) noexcept { swap(rhs); return *this; } @@ -65,4 +65,3 @@ private: }; } // namespace mbus - diff --git a/messagebus/src/vespa/messagebus/callstack.cpp b/messagebus/src/vespa/messagebus/callstack.cpp index 0ab8658d53f..b7179e14cad 100644 --- a/messagebus/src/vespa/messagebus/callstack.cpp +++ b/messagebus/src/vespa/messagebus/callstack.cpp @@ -13,7 +13,7 @@ CallStack::discard() { while (!_stack.empty()) { const Frame &frame = _stack.back(); - if (frame.discardHandler != NULL) { + if (frame.discardHandler != nullptr) { frame.discardHandler->handleDiscard(frame.ctx); } _stack.pop_back(); diff --git a/messagebus/src/vespa/messagebus/callstack.h b/messagebus/src/vespa/messagebus/callstack.h index 68da598e796..0f9d8c93b29 100644 --- a/messagebus/src/vespa/messagebus/callstack.h +++ b/messagebus/src/vespa/messagebus/callstack.h @@ -72,7 +72,7 @@ public: * @param ctx The context to store. * @param discardHandler The handler for discarded messages. **/ - void push(IReplyHandler &replyHandler, Context ctx, IDiscardHandler *discardHandler = NULL) { + void push(IReplyHandler &replyHandler, Context ctx, IDiscardHandler *discardHandler = nullptr) { _stack.emplace_back(&replyHandler, discardHandler, ctx); } diff --git a/messagebus/src/vespa/messagebus/destinationsessionparams.cpp b/messagebus/src/vespa/messagebus/destinationsessionparams.cpp index 3959666e718..ecbc036ffed 100644 --- a/messagebus/src/vespa/messagebus/destinationsessionparams.cpp +++ b/messagebus/src/vespa/messagebus/destinationsessionparams.cpp @@ -6,7 +6,7 @@ namespace mbus { DestinationSessionParams::DestinationSessionParams() : _name("destination"), _broadcastName(true), - _handler(NULL) + _handler(nullptr) { } } // namespace mbus diff --git a/messagebus/src/vespa/messagebus/messagebus.cpp b/messagebus/src/vespa/messagebus/messagebus.cpp index 4cd19b62419..5a8f510ddcf 100644 --- a/messagebus/src/vespa/messagebus/messagebus.cpp +++ b/messagebus/src/vespa/messagebus/messagebus.cpp @@ -97,7 +97,7 @@ MessageBus::MessageBus(INetwork &net, ProtocolSet protocols) : MessageBusParams params; while (!protocols.empty()) { IProtocol::SP protocol = protocols.extract(); - if (protocol.get() != NULL) { + if (protocol.get() != nullptr) { params.addProtocol(protocol); } } @@ -155,7 +155,7 @@ MessageBus::setup(const MessageBusParams ¶ms) // Start messenger. IRetryPolicy::SP retryPolicy = params.getRetryPolicy(); - if (retryPolicy.get() != NULL) { + if (retryPolicy.get() != nullptr) { _resender.reset(new Resender(retryPolicy)); Messenger::ITask::UP task(new ResenderTask(*_resender)); @@ -271,7 +271,7 @@ MessageBus::sync() void MessageBus::handleMessage(Message::UP msg) { - if (_resender.get() != NULL && msg->hasBucketSequence()) { + if (_resender.get() != nullptr && msg->hasBucketSequence()) { deliverError(std::move(msg), ErrorCode::SEQUENCE_ERROR, "Bucket sequences not supported when resender is enabled."); return; @@ -359,7 +359,7 @@ MessageBus::handleDiscard(Context ctx) void MessageBus::deliverMessage(Message::UP msg, const string &session) { - IMessageHandler *msgHandler = NULL; + IMessageHandler *msgHandler = nullptr; { LockGuard guard(_lock); std::map<string, IMessageHandler*>::iterator it = _sessions.find(session); @@ -367,7 +367,7 @@ MessageBus::deliverMessage(Message::UP msg, const string &session) msgHandler = it->second; } } - if (msgHandler == NULL) { + if (msgHandler == nullptr) { deliverError(std::move(msg), ErrorCode::UNKNOWN_SESSION, make_string("Session '%s' does not exist.", session.c_str())); } else if (!checkPending(*msg)) { diff --git a/messagebus/src/vespa/messagebus/messenger.cpp b/messagebus/src/vespa/messagebus/messenger.cpp index 2d1204ee7b6..4b612b66c31 100644 --- a/messagebus/src/vespa/messagebus/messenger.cpp +++ b/messagebus/src/vespa/messagebus/messenger.cpp @@ -32,7 +32,7 @@ public: } ~MessageTask() { - if (_msg.get() != NULL) { + if (_msg.get() != nullptr) { _msg->discard(); } } @@ -42,7 +42,7 @@ public: } uint8_t priority() const override { - if (_msg.get() != NULL) { + if (_msg.get() != nullptr) { return _msg->priority(); } @@ -64,7 +64,7 @@ public: } ~ReplyTask() { - if (_reply.get() != NULL) { + if (_reply.get() != nullptr) { _reply->discard(); } } @@ -74,7 +74,7 @@ public: } uint8_t priority() const override { - if (_reply.get() != NULL) { + if (_reply.get() != nullptr) { return _reply->priority(); } @@ -205,7 +205,7 @@ Messenger::Run(FastOS_ThreadInterface *thread, void *arg) _queue.pop(); } } - if (task.get() != NULL) { + if (task.get() != nullptr) { try { task->run(); } catch (const std::exception &e) { diff --git a/messagebus/src/vespa/messagebus/network/CMakeLists.txt b/messagebus/src/vespa/messagebus/network/CMakeLists.txt index e8992034622..750ff20240f 100644 --- a/messagebus/src/vespa/messagebus/network/CMakeLists.txt +++ b/messagebus/src/vespa/messagebus/network/CMakeLists.txt @@ -6,7 +6,9 @@ vespa_add_library(messagebus_network OBJECT oosmanager.cpp rpcnetwork.cpp rpcnetworkparams.cpp + rpcsend.cpp rpcsendv1.cpp + rpcsendv2.cpp rpcservice.cpp rpcserviceaddress.cpp rpcservicepool.cpp diff --git a/messagebus/src/vespa/messagebus/network/oosmanager.cpp b/messagebus/src/vespa/messagebus/network/oosmanager.cpp index eecfe1da447..250df147675 100644 --- a/messagebus/src/vespa/messagebus/network/oosmanager.cpp +++ b/messagebus/src/vespa/messagebus/network/oosmanager.cpp @@ -89,7 +89,7 @@ OOSManager::isOOS(const string &service) return false; } vespalib::LockGuard guard(_lock); - if (_oosSet.get() == NULL) { + if (_oosSet.get() == nullptr) { return false; } if (_oosSet->find(service) == _oosSet->end()) { diff --git a/messagebus/src/vespa/messagebus/network/oosmanager.h b/messagebus/src/vespa/messagebus/network/oosmanager.h index b521520d2b3..eac00b93896 100644 --- a/messagebus/src/vespa/messagebus/network/oosmanager.h +++ b/messagebus/src/vespa/messagebus/network/oosmanager.h @@ -20,14 +20,11 @@ class RPCNetwork; */ class OOSManager : public FNET_Task { public: - /** - * Convenience typedefs. - */ - typedef slobrok::api::IMirrorAPI IMirrorAPI; - typedef IMirrorAPI::SpecList SpecList; - typedef std::vector<OOSClient::SP> ClientList; - typedef std::set<string> StringSet; - typedef std::shared_ptr<StringSet> OOSSet; + using IMirrorAPI = slobrok::api::IMirrorAPI; + using SpecList = IMirrorAPI::SpecList; + using ClientList = std::vector<OOSClient::SP>; + using StringSet = std::set<string>; + using OOSSet = std::shared_ptr<StringSet>; private: FRT_Supervisor &_orb; diff --git a/messagebus/src/vespa/messagebus/network/rpcnetwork.cpp b/messagebus/src/vespa/messagebus/network/rpcnetwork.cpp index 75c5fc3f6c5..e5c292fedd9 100644 --- a/messagebus/src/vespa/messagebus/network/rpcnetwork.cpp +++ b/messagebus/src/vespa/messagebus/network/rpcnetwork.cpp @@ -1,6 +1,11 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include "inetworkowner.h" #include "rpcnetwork.h" +#include "rpcservicepool.h" +#include "oosmanager.h" +#include "rpcsendv1.h" +#include "rpcsendv2.h" +#include "rpctargetpool.h" +#include "rpcnetworkparams.h" #include <vespa/messagebus/errorcode.h> #include <vespa/messagebus/iprotocol.h> #include <vespa/messagebus/tracelevel.h> @@ -11,6 +16,8 @@ #include <vespa/vespalib/component/vtag.h> #include <vespa/vespalib/util/stringfmt.h> #include <vespa/fnet/scheduler.h> +#include <vespa/fnet/transport.h> +#include <vespa/fnet/frt/supervisor.h> #include <vespa/log/log.h> LOG_SETUP(".rpcnetwork"); @@ -52,17 +59,15 @@ public: namespace mbus { RPCNetwork::SendContext::SendContext(RPCNetwork &net, const Message &msg, - const std::vector<RoutingNode*> &recipients) : - _net(net), - _msg(msg), - _traceLevel(msg.getTrace().getLevel()), - _recipients(recipients), - _hasError(false), - _pending(_recipients.size()), - _version(_net.getVersion()) -{ - // empty -} + const std::vector<RoutingNode*> &recipients) + : _net(net), + _msg(msg), + _traceLevel(msg.getTrace().getLevel()), + _recipients(recipients), + _hasError(false), + _pending(_recipients.size()), + _version(_net.getVersion()) +{ } void RPCNetwork::SendContext::handleVersion(const vespalib::Version *version) @@ -70,7 +75,7 @@ RPCNetwork::SendContext::handleVersion(const vespalib::Version *version) bool shouldSend = false; { vespalib::LockGuard guard(_lock); - if (version == NULL) { + if (version == nullptr) { _hasError = true; } else if (*version < _version) { _version = *version; @@ -85,11 +90,9 @@ RPCNetwork::SendContext::handleVersion(const vespalib::Version *version) } } -RPCNetwork::TargetPoolTask::TargetPoolTask( - FNET_Scheduler &scheduler, - RPCTargetPool &pool) : - FNET_Task(&scheduler), - _pool(pool) +RPCNetwork::TargetPoolTask::TargetPoolTask(FNET_Scheduler &scheduler, RPCTargetPool &pool) + : FNET_Task(&scheduler), + _pool(pool) { ScheduleNow(); } @@ -104,24 +107,26 @@ RPCNetwork::TargetPoolTask::PerformTask() RPCNetwork::RPCNetwork(const RPCNetworkParams ¶ms) : _owner(0), _ident(params.getIdentity()), - _threadPool(128000, 0), - _transport(), - _orb(&_transport, NULL), - _scheduler(*_transport.GetScheduler()), - _targetPool(params.getConnectionExpireSecs()), - _targetPoolTask(_scheduler, _targetPool), - _servicePool(*this, 4096), - _slobrokCfgFactory(params.getSlobrokConfig()), - _mirror(std::make_unique<slobrok::api::MirrorAPI>(_orb, _slobrokCfgFactory)), - _regAPI(std::make_unique<slobrok::api::RegisterAPI>(_orb, _slobrokCfgFactory)), - _oosManager(_orb, *_mirror, params.getOOSServerPattern()), + _threadPool(std::make_unique<FastOS_ThreadPool>(128000, 0)), + _transport(std::make_unique<FNET_Transport>()), + _orb(std::make_unique<FRT_Supervisor>(_transport.get(), nullptr)), + _scheduler(*_transport->GetScheduler()), + _targetPool(std::make_unique<RPCTargetPool>(params.getConnectionExpireSecs())), + _targetPoolTask(_scheduler, *_targetPool), + _servicePool(std::make_unique<RPCServicePool>(*this, 4096)), + _slobrokCfgFactory(std::make_unique<slobrok::ConfiguratorFactory>(params.getSlobrokConfig())), + _mirror(std::make_unique<slobrok::api::MirrorAPI>(*_orb, *_slobrokCfgFactory)), + _regAPI(std::make_unique<slobrok::api::RegisterAPI>(*_orb, *_slobrokCfgFactory)), + _oosManager(std::make_unique<OOSManager>(*_orb, *_mirror, params.getOOSServerPattern())), _requestedPort(params.getListenPort()), - _sendV1(), - _sendAdapters() + _sendV1(std::make_unique<RPCSendV1>()), + _sendV2(std::make_unique<RPCSendV2>()), + _sendAdapters(), + _compressionConfig(params.getCompressionConfig()) { - _transport.SetDirectWrite(false); - _transport.SetMaxInputBufferSize(params.getMaxInputBufferSize()); - _transport.SetMaxOutputBufferSize(params.getMaxOutputBufferSize()); + _transport->SetDirectWrite(false); + _transport->SetMaxInputBufferSize(params.getMaxInputBufferSize()); + _transport->SetMaxOutputBufferSize(params.getMaxOutputBufferSize()); } RPCNetwork::~RPCNetwork() @@ -132,33 +137,33 @@ RPCNetwork::~RPCNetwork() FRT_RPCRequest * RPCNetwork::allocRequest() { - return _orb.AllocRPCRequest(); + return _orb->AllocRPCRequest(); } RPCTarget::SP RPCNetwork::getTarget(const RPCServiceAddress &address) { - return _targetPool.getTarget(_orb, address); + return _targetPool->getTarget(*_orb, address); } void -RPCNetwork::replyError(const SendContext &ctx, uint32_t errCode, - const string &errMsg) +RPCNetwork::replyError(const SendContext &ctx, uint32_t errCode, const string &errMsg) { - for (std::vector<RoutingNode*>::const_iterator it = ctx._recipients.begin(); - it != ctx._recipients.end(); ++it) - { + for (RoutingNode * rnode : ctx._recipients) { Reply::UP reply(new EmptyReply()); reply->setTrace(Trace(ctx._traceLevel)); reply->addError(Error(errCode, errMsg)); - _owner->deliverReply(std::move(reply), **it); + _owner->deliverReply(std::move(reply), *rnode); } } +int RPCNetwork::getPort() const { return _orb->GetListenPort(); } + + void RPCNetwork::flushTargetPool() { - _targetPool.flushTargets(true); + _targetPool->flushTargets(true); } const vespalib::Version & @@ -173,13 +178,13 @@ RPCNetwork::attach(INetworkOwner &owner) LOG_ASSERT(_owner == 0); _owner = &owner; - _sendV1.attach(*this); - _sendAdapters.insert(SendAdapterMap::value_type(vespalib::VersionSpecification(5), &_sendV1)); - _sendAdapters.insert(SendAdapterMap::value_type(vespalib::VersionSpecification(6), &_sendV1)); + _sendV1->attach(*this); + _sendV2->attach(*this); + _sendAdapters[vespalib::Version(5)] = _sendV1.get(); + _sendAdapters[vespalib::Version(6, 149)] = _sendV2.get(); - FRT_ReflectionBuilder builder(&_orb); - builder.DefineMethod("mbus.getVersion", "", "s", true, - FRT_METHOD(RPCNetwork::invoke), this); + FRT_ReflectionBuilder builder(_orb.get()); + builder.DefineMethod("mbus.getVersion", "", "s", true, FRT_METHOD(RPCNetwork::invoke), this); builder.MethodDesc("Retrieves the message bus version."); builder.ReturnDesc("version", "The message bus version."); } @@ -193,29 +198,25 @@ RPCNetwork::invoke(FRT_RPCRequest *req) const string RPCNetwork::getConnectionSpec() const { - return make_string("tcp/%s:%d", _ident.getHostname().c_str(), _orb.GetListenPort()); + return make_string("tcp/%s:%d", _ident.getHostname().c_str(), _orb->GetListenPort()); } RPCSendAdapter * RPCNetwork::getSendAdapter(const vespalib::Version &version) { - for (SendAdapterMap::iterator it = _sendAdapters.begin(); - it != _sendAdapters.end(); ++it) - { - if (it->first.matches(version)) { - return it->second; - } + if (version < _sendAdapters.begin()->first) { + return nullptr; } - return NULL; + return (--_sendAdapters.upper_bound(version))->second; } bool RPCNetwork::start() { - if (!_orb.Listen(_requestedPort)) { + if (!_orb->Listen(_requestedPort)) { return false; } - if (!_transport.Start(&_threadPool)) { + if (!_transport->Start(_threadPool.get())) { return false; } return true; @@ -227,13 +228,13 @@ bool RPCNetwork::waitUntilReady(double seconds) const { slobrok::api::SlobrokList brokerList; - slobrok::Configurator::UP configurator = _slobrokCfgFactory.create(brokerList); + slobrok::Configurator::UP configurator = _slobrokCfgFactory->create(brokerList); bool hasConfig = false; for (uint32_t i = 0; i < seconds * 100; ++i) { if (configurator->poll()) { hasConfig = true; } - if (_mirror->ready() && _oosManager.isReady()) { + if (_mirror->ready() && _oosManager->isReady()) { return true; } FastOS_Thread::Sleep(10); @@ -244,7 +245,7 @@ RPCNetwork::waitUntilReady(double seconds) const std::string brokers = brokerList.logString(); LOG(error, "mirror (of %s) failed to become ready in %d seconds", brokers.c_str(), (int)seconds); - } else if (! _oosManager.isReady()) { + } else if (! _oosManager->isReady()) { LOG(error, "OOS manager failed to become ready in %d seconds", (int)seconds); } return false; @@ -293,24 +294,23 @@ RPCNetwork::allocServiceAddress(RoutingNode &recipient) Error RPCNetwork::resolveServiceAddress(RoutingNode &recipient, const string &serviceName) { - if (_oosManager.isOOS(serviceName)) { + if (_oosManager->isOOS(serviceName)) { return Error(ErrorCode::SERVICE_OOS, make_string("The service '%s' has been marked as out of service.", serviceName.c_str())); } - RPCServiceAddress::UP ret = _servicePool.resolve(serviceName); - if (ret.get() == NULL) { + RPCServiceAddress::UP ret = _servicePool->resolve(serviceName); + if (ret.get() == nullptr) { return Error(ErrorCode::NO_ADDRESS_FOR_SERVICE, make_string("The address of service '%s' could not be resolved. It is not currently " - "registered with the Vespa name server. " - "The service must be having problems, or the routing configuration is wrong.", - serviceName.c_str())); + "registered with the Vespa name server. " + "The service must be having problems, or the routing configuration is wrong.", + serviceName.c_str())); } - RPCTarget::SP target = _targetPool.getTarget(_orb, *ret); - if (target.get() == NULL) { + RPCTarget::SP target = _targetPool->getTarget(*_orb, *ret); + if (target.get() == nullptr) { return Error(ErrorCode::CONNECTION_ERROR, - make_string("Failed to connect to service '%s'.", - serviceName.c_str())); + make_string("Failed to connect to service '%s'.", serviceName.c_str())); } ret->setTarget(target); // free by freeServiceAddress() recipient.setServiceAddress(IServiceAddress::UP(ret.release())); @@ -330,7 +330,7 @@ RPCNetwork::send(const Message &msg, const std::vector<RoutingNode*> &recipients double timeout = ctx._msg.getTimeRemainingNow() / 1000.0; for (uint32_t i = 0, len = ctx._recipients.size(); i < len; ++i) { RoutingNode *&recipient = ctx._recipients[i]; - LOG_ASSERT(recipient != NULL); + LOG_ASSERT(recipient != nullptr); RPCServiceAddress &address = static_cast<RPCServiceAddress&>(recipient->getServiceAddress()); LOG_ASSERT(address.hasTarget()); @@ -343,13 +343,12 @@ void RPCNetwork::send(RPCNetwork::SendContext &ctx) { if (ctx._hasError) { - replyError(ctx, ErrorCode::HANDSHAKE_FAILED, - "An error occured while resolving version."); + replyError(ctx, ErrorCode::HANDSHAKE_FAILED, "An error occured while resolving version."); } else { uint64_t timeRemaining = ctx._msg.getTimeRemainingNow(); Blob payload = _owner->getProtocol(ctx._msg.getProtocol())->encode(ctx._version, ctx._msg); RPCSendAdapter *adapter = getSendAdapter(ctx._version); - if (adapter == NULL) { + if (adapter == nullptr) { replyError(ctx, ErrorCode::INCOMPATIBLE_VERSION, make_string("Can not send to version '%s' recipient.", ctx._version.toString().c_str())); } else if (timeRemaining == 0) { @@ -378,8 +377,8 @@ RPCNetwork::sync() void RPCNetwork::shutdown() { - _transport.ShutDown(false); - _threadPool.Close(); + _transport->ShutDown(false); + _threadPool->Close(); } void diff --git a/messagebus/src/vespa/messagebus/network/rpcnetwork.h b/messagebus/src/vespa/messagebus/network/rpcnetwork.h index 856f9d0ef64..5e762f1a2a9 100644 --- a/messagebus/src/vespa/messagebus/network/rpcnetwork.h +++ b/messagebus/src/vespa/messagebus/network/rpcnetwork.h @@ -2,27 +2,33 @@ #pragma once #include "inetwork.h" -#include "oosmanager.h" -#include "rpcnetworkparams.h" -#include "rpcsendv1.h" -#include "rpcservicepool.h" -#include "rpctargetpool.h" +#include "rpcsendadapter.h" +#include "rpctarget.h" +#include "identity.h" #include <vespa/messagebus/blob.h> #include <vespa/messagebus/blobref.h> #include <vespa/messagebus/message.h> #include <vespa/messagebus/reply.h> #include <vespa/slobrok/imirrorapi.h> #include <vespa/vespalib/component/versionspecification.h> -#include <vespa/fnet/transport.h> -#include <vespa/fnet/frt/supervisor.h> +#include <vespa/vespalib/util/compressionconfig.h> +#include <vespa/fnet/frt/invokable.h> + +class FNET_Transport; namespace slobrok { - namespace api { - class RegisterAPI; - } + namespace api { class RegisterAPI; } + class ConfiguratorFactory; } namespace mbus { + +class OOSManager; +class RPCServicePool; +class RPCTargetPool; +class RPCNetworkParams; +class RPCServiceAddress; + /** * Network implementation based on RPC. This class is responsible for * keeping track of services and for sending messages to services. @@ -30,6 +36,7 @@ namespace mbus { class RPCNetwork : public INetwork, public FRT_Invokable { private: + using CompressionConfig = vespalib::compression::CompressionConfig; struct SendContext : public RPCTarget::IVersionHandler { vespalib::Lock _lock; RPCNetwork &_net; @@ -51,24 +58,26 @@ private: void PerformTask() override; }; - typedef std::map<vespalib::VersionSpecification, RPCSendAdapter*> SendAdapterMap; - - INetworkOwner *_owner; - Identity _ident; - FastOS_ThreadPool _threadPool; - FNET_Transport _transport; - FRT_Supervisor _orb; - FNET_Scheduler &_scheduler; - RPCTargetPool _targetPool; - TargetPoolTask _targetPoolTask; - RPCServicePool _servicePool; - slobrok::ConfiguratorFactory _slobrokCfgFactory; - std::unique_ptr<slobrok::api::IMirrorAPI> _mirror; - std::unique_ptr<slobrok::api::RegisterAPI> _regAPI; - OOSManager _oosManager; - int _requestedPort; - RPCSendV1 _sendV1; - SendAdapterMap _sendAdapters; + using SendAdapterMap = std::map<vespalib::Version, RPCSendAdapter*>; + + INetworkOwner *_owner; + Identity _ident; + std::unique_ptr<FastOS_ThreadPool> _threadPool; + std::unique_ptr<FNET_Transport> _transport; + std::unique_ptr<FRT_Supervisor> _orb; + FNET_Scheduler &_scheduler; + std::unique_ptr<RPCTargetPool> _targetPool; + TargetPoolTask _targetPoolTask; + std::unique_ptr<RPCServicePool> _servicePool; + std::unique_ptr<slobrok::ConfiguratorFactory> _slobrokCfgFactory; + std::unique_ptr<slobrok::api::IMirrorAPI> _mirror; + std::unique_ptr<slobrok::api::RegisterAPI> _regAPI; + std::unique_ptr<OOSManager> _oosManager; + int _requestedPort; + std::unique_ptr<RPCSendAdapter> _sendV1; + std::unique_ptr<RPCSendAdapter> _sendV2; + SendAdapterMap _sendAdapters; + CompressionConfig _compressionConfig; /** * Resolves and assigns a service address for the given recipient using the @@ -84,15 +93,6 @@ private: Error resolveServiceAddress(RoutingNode &recipient, const string &serviceName); /** - * Determines and returns the send adapter that is compatible with the given - * version. If no adapter can be found, this method returns null. - * - * @param version The version for which to return an adapter. - * @return The compatible adapter. - */ - RPCSendAdapter *getSendAdapter(const vespalib::Version &version); - - /** * This method is a callback invoked after {@link #send(Message, List)} once * the version of all recipients have been resolved. If all versions were * resolved ahead of time, this method is invoked by the same thread as the @@ -159,7 +159,7 @@ public: * * @return port number **/ - int getPort() const { return _orb.GetListenPort(); } + int getPort() const; /** * Allocate a new rpc request object. The caller of this method gets the @@ -191,7 +191,7 @@ public: * * @return internal OOS manager **/ - OOSManager &getOOSManager() { return _oosManager; } + OOSManager &getOOSManager() { return *_oosManager; } /** * Obtain a reference to the internal supervisor. This is used by @@ -199,7 +199,7 @@ public: * * @return The supervisor. */ - FRT_Supervisor &getSupervisor() { return _orb; } + FRT_Supervisor &getSupervisor() { return *_orb; } /** * Deliver an error reply to the recipients of a {@link SendContext} in a @@ -209,8 +209,16 @@ public: * @param errCode The error code to return. * @param errMsg The error string to return. */ - void replyError(const SendContext &ctx, uint32_t errCode, - const string &errMsg); + void replyError(const SendContext &ctx, uint32_t errCode, const string &errMsg); + + /** + * Determines and returns the send adapter that is compatible with the given + * version. If no adapter can be found, this method returns null. + * + * @param version The version for which to return an adapter. + * @return The compatible adapter. + */ + RPCSendAdapter *getSendAdapter(const vespalib::Version &version); void attach(INetworkOwner &owner) override; const string getConnectionSpec() const override; @@ -225,9 +233,8 @@ public: void shutdown() override; void postShutdownHook() override; const slobrok::api::IMirrorAPI &getMirror() const override; - + CompressionConfig getCompressionConfig() { return _compressionConfig; } void invoke(FRT_RPCRequest *req); }; } // namespace mbus - diff --git a/messagebus/src/vespa/messagebus/network/rpcnetworkparams.cpp b/messagebus/src/vespa/messagebus/network/rpcnetworkparams.cpp index e025db0e350..df35d51cb54 100644 --- a/messagebus/src/vespa/messagebus/network/rpcnetworkparams.cpp +++ b/messagebus/src/vespa/messagebus/network/rpcnetworkparams.cpp @@ -11,7 +11,8 @@ RPCNetworkParams::RPCNetworkParams() : _listenPort(0), _maxInputBufferSize(256*1024), _maxOutputBufferSize(256*1024), - _connectionExpireSecs(30) + _connectionExpireSecs(30), + _compressionConfig(CompressionConfig::LZ4, 6, 90, 1024) { } RPCNetworkParams::~RPCNetworkParams() {} diff --git a/messagebus/src/vespa/messagebus/network/rpcnetworkparams.h b/messagebus/src/vespa/messagebus/network/rpcnetworkparams.h index a65248f7299..bfc624a6523 100644 --- a/messagebus/src/vespa/messagebus/network/rpcnetworkparams.h +++ b/messagebus/src/vespa/messagebus/network/rpcnetworkparams.h @@ -3,6 +3,7 @@ #include "identity.h" #include <vespa/slobrok/cfg.h> +#include <vespa/vespalib/util/compressionconfig.h> namespace mbus { @@ -12,13 +13,15 @@ namespace mbus { */ class RPCNetworkParams { private: - Identity _identity; + using CompressionConfig = vespalib::compression::CompressionConfig; + Identity _identity; config::ConfigUri _slobrokConfig; - string _oosServerPattern; - int _listenPort; - uint32_t _maxInputBufferSize; - uint32_t _maxOutputBufferSize; - double _connectionExpireSecs; + string _oosServerPattern; + int _listenPort; + uint32_t _maxInputBufferSize; + uint32_t _maxOutputBufferSize; + double _connectionExpireSecs; + CompressionConfig _compressionConfig; public: RPCNetworkParams(); @@ -177,6 +180,12 @@ public: _maxOutputBufferSize = maxOutputBufferSize; return *this; } + + RPCNetworkParams &setCompressionConfig(CompressionConfig compressionConfig) { + _compressionConfig = compressionConfig; + return *this; + } + CompressionConfig getCompressionConfig() const { return _compressionConfig; } }; } diff --git a/messagebus/src/vespa/messagebus/network/rpcsend.cpp b/messagebus/src/vespa/messagebus/network/rpcsend.cpp new file mode 100644 index 00000000000..705b8648442 --- /dev/null +++ b/messagebus/src/vespa/messagebus/network/rpcsend.cpp @@ -0,0 +1,282 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "rpcsend.h" +#include "rpcsend_private.h" +#include "rpcserviceaddress.h" +#include <vespa/messagebus/network/rpcnetwork.h> +#include <vespa/messagebus/tracelevel.h> +#include <vespa/messagebus/emptyreply.h> +#include <vespa/messagebus/errorcode.h> +#include <vespa/vespalib/util/stringfmt.h> +#include <vespa/fnet/channel.h> +#include <vespa/fnet/frt/reflection.h> + +#include <vespa/vespalib/data/slime/cursor.h> + +using vespalib::make_string; + +namespace mbus { + +using network::internal::ReplyContext; +using network::internal::SendContext; + +namespace { + +class FillByCopy final : public PayLoadFiller +{ +public: + FillByCopy(BlobRef payload) : _payload(payload) { } + void fill(FRT_Values & v) const override { + v.AddData(_payload.data(), _payload.size()); + } + void fill(const vespalib::Memory & name, vespalib::slime::Cursor & v) const override { + v.setData(name, vespalib::Memory(_payload.data(), _payload.size())); + } +private: + BlobRef _payload; +}; + +class FillByHandover final : public PayLoadFiller +{ +public: + FillByHandover(Blob payload) : _payload(std::move(payload)) { } + void fill(FRT_Values & v) const override { + size_t sz = _payload.size(); + v.AddData(std::move(_payload.payload()), sz); + } + void fill(const vespalib::Memory & name, vespalib::slime::Cursor & v) const override { + v.setData(name, vespalib::Memory(_payload.data(), _payload.size())); + } +private: + mutable Blob _payload; +}; + +} + +RPCSend::RPCSend() : + _net(nullptr), + _clientIdent("client"), + _serverIdent("server") +{ } + +RPCSend::~RPCSend() {} + +void +RPCSend::attach(RPCNetwork &net) +{ + _net = &net; + const string &prefix = _net->getIdentity().getServicePrefix(); + if (!prefix.empty()) { + _clientIdent = make_string("'%s'", prefix.c_str()); + _serverIdent = _clientIdent; + } + + FRT_ReflectionBuilder builder(&_net->getSupervisor()); + build(builder); +} + +void +RPCSend::replyError(FRT_RPCRequest *req, const vespalib::Version &version, uint32_t traceLevel, const Error &err) +{ + Reply::UP reply(new EmptyReply()); + reply->setContext(Context(new ReplyContext(*req, version))); + reply->getTrace().setLevel(traceLevel); + reply->addError(err); + handleReply(std::move(reply)); +} + +void +RPCSend::handleDiscard(Context ctx) +{ + ReplyContext::UP tmp(static_cast<ReplyContext*>(ctx.value.PTR)); + FRT_RPCRequest &req = tmp->getRequest(); + FNET_Channel *chn = req.GetContext()._value.CHANNEL; + req.SubRef(); + chn->Free(); +} + +void +RPCSend::sendByHandover(RoutingNode &recipient, const vespalib::Version &version, Blob payload, uint64_t timeRemaining) +{ + send(recipient, version, FillByHandover(std::move(payload)), timeRemaining); +} + +void +RPCSend::send(RoutingNode &recipient, const vespalib::Version &version, BlobRef payload, uint64_t timeRemaining) +{ + send(recipient, version, FillByCopy(payload), timeRemaining); +} + +void +RPCSend::send(RoutingNode &recipient, const vespalib::Version &version, + const PayLoadFiller & payload, uint64_t timeRemaining) +{ + SendContext::UP ctx(new SendContext(recipient, timeRemaining)); + RPCServiceAddress &address = static_cast<RPCServiceAddress&>(recipient.getServiceAddress()); + const Message &msg = recipient.getMessage(); + Route route = recipient.getRoute(); + Hop hop = route.removeHop(0); + + FRT_RPCRequest *req = _net->allocRequest(); + encodeRequest(*req, version, route, address, msg, recipient.getTrace().getLevel(), payload, timeRemaining); + + if (ctx->getTrace().shouldTrace(TraceLevel::SEND_RECEIVE)) { + ctx->getTrace().trace(TraceLevel::SEND_RECEIVE, + make_string("Sending message (version %s) from %s to '%s' with %.2f seconds timeout.", + version.toString().c_str(), _clientIdent.c_str(), + address.getServiceName().c_str(), ctx->getTimeout())); + } + + if (hop.getIgnoreResult()) { + address.getTarget().getFRTTarget().InvokeVoid(req); + if (ctx->getTrace().shouldTrace(TraceLevel::SEND_RECEIVE)) { + ctx->getTrace().trace(TraceLevel::SEND_RECEIVE, + make_string("Not waiting for a reply from '%s'.", address.getServiceName().c_str())); + } + Reply::UP reply(new EmptyReply()); + reply->getTrace().swap(ctx->getTrace()); + _net->getOwner().deliverReply(std::move(reply), recipient); + } else { + SendContext *ptr = ctx.release(); + req->SetContext(FNET_Context(ptr)); + address.getTarget().getFRTTarget().InvokeAsync(req, ptr->getTimeout(), this); + } +} + +void +RPCSend::RequestDone(FRT_RPCRequest *req) +{ + SendContext::UP ctx(static_cast<SendContext*>(req->GetContext()._value.VOIDP)); + const string &serviceName = static_cast<RPCServiceAddress&>(ctx->getRecipient().getServiceAddress()).getServiceName(); + Reply::UP reply; + Error error; + Trace & trace = ctx->getTrace(); + if (!req->CheckReturnTypes(getReturnSpec())) { + reply.reset(new EmptyReply()); + switch (req->GetErrorCode()) { + case FRTE_RPC_TIMEOUT: + error = Error(ErrorCode::TIMEOUT, + make_string("A timeout occured while waiting for '%s' (%g seconds expired); %s", + serviceName.c_str(), ctx->getTimeout(), req->GetErrorMessage())); + break; + case FRTE_RPC_CONNECTION: + error = Error(ErrorCode::CONNECTION_ERROR, + make_string("A connection error occured for '%s'; %s", + serviceName.c_str(), req->GetErrorMessage())); + break; + default: + error = Error(ErrorCode::NETWORK_ERROR, + make_string("A network error occured for '%s'; %s", + serviceName.c_str(), req->GetErrorMessage())); + } + } else { + FRT_Values &ret = *req->GetReturn(); + reply = createReply(ret, serviceName, error, trace.getRoot()); + } + if (trace.shouldTrace(TraceLevel::SEND_RECEIVE)) { + trace.trace(TraceLevel::SEND_RECEIVE, + make_string("Reply (type %d) received at %s.", reply->getType(), _clientIdent.c_str())); + } + reply->getTrace().swap(trace); + if (error.getCode() != ErrorCode::NONE) { + reply->addError(error); + } + _net->getOwner().deliverReply(std::move(reply), ctx->getRecipient()); + req->SubRef(); +} + +std::unique_ptr<Reply> +RPCSend::decode(vespalib::stringref protocolName, const vespalib::Version & version, BlobRef payload, Error & error) const +{ + Reply::UP reply; + IProtocol * protocol = _net->getOwner().getProtocol(protocolName); + if (protocol != nullptr) { + Routable::UP routable = protocol->decode(version, payload); + if (routable) { + if (routable->isReply()) { + reply.reset(static_cast<Reply*>(routable.release())); + } else { + error = Error(ErrorCode::DECODE_ERROR, "Payload decoded to a message when expecting a reply."); + } + } else { + error = Error(ErrorCode::DECODE_ERROR, + make_string("Protocol '%s' failed to decode routable.", protocolName.c_str())); + } + + } else { + error = Error(ErrorCode::UNKNOWN_PROTOCOL, + make_string("Protocol '%s' is not known by %s.", protocolName.c_str(), _serverIdent.c_str())); + } + return reply; +} + +void +RPCSend::handleReply(Reply::UP reply) +{ + ReplyContext::UP ctx(static_cast<ReplyContext*>(reply->getContext().value.PTR)); + FRT_RPCRequest &req = ctx->getRequest(); + string version = ctx->getVersion().toString(); + if (reply->getTrace().shouldTrace(TraceLevel::SEND_RECEIVE)) { + reply->getTrace().trace(TraceLevel::SEND_RECEIVE, make_string("Sending reply (version %s) from %s.", + version.c_str(), _serverIdent.c_str())); + } + Blob payload(0); + if (reply->getType() != 0) { + payload = _net->getOwner().getProtocol(reply->getProtocol())->encode(ctx->getVersion(), *reply); + if (payload.size() == 0) { + reply->addError(Error(ErrorCode::ENCODE_ERROR, "An error occured while encoding the reply, see log.")); + } + } + FRT_Values &ret = *req.GetReturn(); + createResponse(ret, version, *reply, std::move(payload)); + req.Return(); +} + +void +RPCSend::invoke(FRT_RPCRequest *req) +{ + req->Detach(); + FRT_Values &args = *req->GetParams(); + + std::unique_ptr<Params> params = toParams(args); + IProtocol * protocol = _net->getOwner().getProtocol(params->getProtocol()); + if (protocol == nullptr) { + replyError(req, params->getVersion(), params->getTraceLevel(), + Error(ErrorCode::UNKNOWN_PROTOCOL, + make_string("Protocol '%s' is not known by %s.", params->getProtocol().c_str(), _serverIdent.c_str()))); + return; + } + Routable::UP routable = protocol->decode(params->getVersion(), params->getPayload()); + req->DiscardBlobs(); + if ( ! routable ) { + replyError(req, params->getVersion(), params->getTraceLevel(), + Error(ErrorCode::DECODE_ERROR, + make_string("Protocol '%s' failed to decode routable.", params->getProtocol().c_str()))); + return; + } + if (routable->isReply()) { + replyError(req, params->getVersion(), params->getTraceLevel(), + Error(ErrorCode::DECODE_ERROR, "Payload decoded to a reply when expecting a mesage.")); + return; + } + Message::UP msg(static_cast<Message*>(routable.release())); + vespalib::stringref route = params->getRoute(); + if (!route.empty()) { + msg->setRoute(Route::parse(route)); + } + msg->setContext(Context(new ReplyContext(*req, params->getVersion()))); + msg->pushHandler(*this, *this); + msg->setRetryEnabled(params->useRetry()); + msg->setRetry(params->getRetries()); + msg->setTimeReceivedNow(); + msg->setTimeRemaining(params->getRemainingTime()); + msg->getTrace().setLevel(params->getTraceLevel()); + if (msg->getTrace().shouldTrace(TraceLevel::SEND_RECEIVE)) { + msg->getTrace().trace(TraceLevel::SEND_RECEIVE, + make_string("Message (type %d) received at %s for session '%s'.", + msg->getType(), _serverIdent.c_str(), string(params->getSession()).c_str())); + } + _net->getOwner().deliverMessage(std::move(msg), params->getSession()); +} + +} // namespace mbus diff --git a/messagebus/src/vespa/messagebus/network/rpcsend.h b/messagebus/src/vespa/messagebus/network/rpcsend.h new file mode 100644 index 00000000000..c707b47f548 --- /dev/null +++ b/messagebus/src/vespa/messagebus/network/rpcsend.h @@ -0,0 +1,95 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include "rpcsendadapter.h" +#include <vespa/messagebus/idiscardhandler.h> +#include <vespa/messagebus/ireplyhandler.h> +#include <vespa/messagebus/common.h> +#include <vespa/fnet/frt/invokable.h> +#include <vespa/fnet/frt/invoker.h> + +class FRT_ReflectionBuilder; + +namespace vespalib::slime { class Cursor; } +namespace vespalib { class Memory; } +namespace vespalib { class TraceNode; } +namespace mbus { + +class Error; +class Route; +class Message; +class RPCServiceAddress; + +class PayLoadFiller +{ +public: + virtual ~PayLoadFiller() { } + virtual void fill(FRT_Values & v) const = 0; + virtual void fill(const vespalib::Memory & name, vespalib::slime::Cursor & v) const = 0; +}; + +class RPCSend : public RPCSendAdapter, + public FRT_Invokable, + public FRT_IRequestWait, + public IDiscardHandler, + public IReplyHandler +{ +public: + class Params { + public: + virtual ~Params() {} + virtual vespalib::Version getVersion() const = 0; + virtual vespalib::stringref getProtocol() const = 0; + virtual uint32_t getTraceLevel() const = 0; + virtual bool useRetry() const = 0; + virtual uint32_t getRetries() const = 0; + virtual uint64_t getRemainingTime() const = 0; + virtual vespalib::stringref getRoute() const = 0; + virtual vespalib::stringref getSession() const = 0; + virtual BlobRef getPayload() const = 0; + }; +protected: + RPCNetwork *_net; + string _clientIdent; + string _serverIdent; + + virtual void build(FRT_ReflectionBuilder & builder) = 0; + virtual std::unique_ptr<Reply> createReply(const FRT_Values & response, const string & serviceName, + Error & error, vespalib::TraceNode & rootTrace) const = 0; + virtual void encodeRequest(FRT_RPCRequest &req, const vespalib::Version &version, const Route & route, + const RPCServiceAddress & address, const Message & msg, uint32_t traceLevel, + const PayLoadFiller &filler, uint64_t timeRemaining) const = 0; + virtual const char * getReturnSpec() const = 0; + virtual void createResponse(FRT_Values & ret, const string & version, Reply & reply, Blob payload) const = 0; + virtual std::unique_ptr<Params> toParams(const FRT_Values ¶m) const = 0; + + void send(RoutingNode &recipient, const vespalib::Version &version, + const PayLoadFiller & filler, uint64_t timeRemaining); + std::unique_ptr<Reply> decode(vespalib::stringref protocol, const vespalib::Version & version, + BlobRef payload, Error & error) const; + /** + * Send an error reply for a given request. + * + * @param request The FRT request to reply to. + * @param version The version to serialize for. + * @param traceLevel The trace level to set in the reply. + * @param err The error to reply with. + */ + void replyError(FRT_RPCRequest *req, const vespalib::Version &version, uint32_t traceLevel, const Error &err); +public: + RPCSend(); + ~RPCSend(); + + void invoke(FRT_RPCRequest *req); +private: + void attach(RPCNetwork &net) final override; + void handleDiscard(Context ctx) final override; + void sendByHandover(RoutingNode &recipient, const vespalib::Version &version, + Blob payload, uint64_t timeRemaining) final override; + void send(RoutingNode &recipient, const vespalib::Version &version, + BlobRef payload, uint64_t timeRemaining) final override; + void RequestDone(FRT_RPCRequest *req) final override; + void handleReply(std::unique_ptr<Reply> reply) final override; +}; + +} // namespace mbus diff --git a/messagebus/src/vespa/messagebus/network/rpcsend_private.h b/messagebus/src/vespa/messagebus/network/rpcsend_private.h new file mode 100644 index 00000000000..f5867e79856 --- /dev/null +++ b/messagebus/src/vespa/messagebus/network/rpcsend_private.h @@ -0,0 +1,52 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/messagebus/trace.h> +#include <vespa/messagebus/routing/routingnode.h> + +namespace mbus::network::internal { +/** + * Implements a helper class to hold the necessary context to create a reply from + * an rpc return value. This object is held as the context of an FRT_RPCRequest. + */ +class SendContext { +private: + mbus::RoutingNode &_recipient; + mbus::Trace _trace; + double _timeout; + +public: + typedef std::unique_ptr<SendContext> UP; + SendContext(const SendContext &) = delete; + SendContext & operator = (const SendContext &) = delete; + SendContext(mbus::RoutingNode &recipient, uint64_t timeRemaining) + : _recipient(recipient), + _trace(recipient.getTrace().getLevel()), + _timeout(timeRemaining * 0.001) { } + mbus::RoutingNode &getRecipient() { return _recipient; } + mbus::Trace &getTrace() { return _trace; } + double getTimeout() { return _timeout; } +}; + +/** + * Implements a helper class to hold the necessary context to send a reply as an + * rpc return value. This object is held in the callstack of the reply. + */ +class ReplyContext { +private: + FRT_RPCRequest &_request; + vespalib::Version _version; + +public: + typedef std::unique_ptr<ReplyContext> UP; + ReplyContext(const ReplyContext &) = delete; + ReplyContext & operator = (const ReplyContext &) = delete; + + ReplyContext(FRT_RPCRequest &request, const vespalib::Version &version) + : _request(request), _version(version) { } + FRT_RPCRequest &getRequest() { return _request; } + const vespalib::Version &getVersion() { return _version; } +}; + + +} diff --git a/messagebus/src/vespa/messagebus/network/rpcsendv1.cpp b/messagebus/src/vespa/messagebus/network/rpcsendv1.cpp index 4cf45207010..6b89a278b88 100644 --- a/messagebus/src/vespa/messagebus/network/rpcsendv1.cpp +++ b/messagebus/src/vespa/messagebus/network/rpcsendv1.cpp @@ -1,87 +1,40 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include "rpcsendv1.h" #include "rpcnetwork.h" -#include <vespa/messagebus/routing/routingnode.h> +#include "rpcserviceaddress.h" #include <vespa/messagebus/emptyreply.h> -#include <vespa/messagebus/errorcode.h> #include <vespa/messagebus/tracelevel.h> #include <vespa/vespalib/util/stringfmt.h> -#include <vespa/fnet/channel.h> +#include <vespa/fnet/frt/reflection.h> using vespalib::make_string; -namespace { - -/** - * Implements a helper class to hold the necessary context to create a reply from - * an rpc return value. This object is held as the context of an FRT_RPCRequest. - */ -class SendContext { -private: - mbus::RoutingNode &_recipient; - mbus::Trace _trace; - double _timeout; - -public: - typedef std::unique_ptr<SendContext> UP; - SendContext(const SendContext &) = delete; - SendContext & operator = (const SendContext &) = delete; - SendContext(mbus::RoutingNode &recipient, uint64_t timeRemaining) - : _recipient(recipient), - _trace(recipient.getTrace().getLevel()), - _timeout(timeRemaining * 0.001) { } - mbus::RoutingNode &getRecipient() { return _recipient; } - mbus::Trace &getTrace() { return _trace; } - double getTimeout() { return _timeout; } -}; +namespace mbus { -/** - * Implements a helper class to hold the necessary context to send a reply as an - * rpc return value. This object is held in the callstack of the reply. - */ -class ReplyContext { -private: - FRT_RPCRequest &_request; - vespalib::Version _version; +namespace { -public: - typedef std::unique_ptr<ReplyContext> UP; - ReplyContext(const ReplyContext &) = delete; - ReplyContext & operator = (const ReplyContext &) = delete; - - ReplyContext(FRT_RPCRequest &request, const vespalib::Version &version) - : _request(request), _version(version) { } - FRT_RPCRequest &getRequest() { return _request; } - const vespalib::Version &getVersion() { return _version; } -}; +const char *METHOD_NAME = "mbus.send1"; +const char *METHOD_PARAMS = "sssbilsxi"; +const char *METHOD_RETURN = "sdISSsxs"; } -namespace mbus { - -const char *RPCSendV1::METHOD_NAME = "mbus.send1"; -const char *RPCSendV1::METHOD_PARAMS = "sssbilsxi"; -const char *RPCSendV1::METHOD_RETURN = "sdISSsxs"; - -RPCSendV1::RPCSendV1() : - _net(NULL), - _clientIdent("client"), - _serverIdent("server") -{ } +bool RPCSendV1::isCompatible(vespalib::stringref method, vespalib::stringref request, vespalib::stringref respons) +{ + return (method == METHOD_NAME) && + (request == METHOD_PARAMS) && + (respons == METHOD_RETURN); +} -RPCSendV1::~RPCSendV1() {} +const char * +RPCSendV1::getReturnSpec() const { + return METHOD_RETURN; +} void -RPCSendV1::attach(RPCNetwork &net) +RPCSendV1::build(FRT_ReflectionBuilder & builder) { - _net = &net; - const string &prefix = _net->getIdentity().getServicePrefix(); - if (!prefix.empty()) { - _clientIdent = make_string("'%s'", prefix.c_str()); - _serverIdent = _clientIdent; - } - - FRT_ReflectionBuilder builder(&_net->getSupervisor()); builder.DefineMethod(METHOD_NAME, METHOD_PARAMS, METHOD_RETURN, true, FRT_METHOD(RPCSendV1::invoke), this); builder.MethodDesc("Send a message bus request and get a reply back."); builder.ParamDesc("version", "The version of the message."); @@ -103,59 +56,14 @@ RPCSendV1::attach(RPCNetwork &net) builder.ReturnDesc("trace", "A string representation of the trace."); } -namespace { - -class FillByCopy : public PayLoadFiller -{ -public: - FillByCopy(BlobRef payload) : _payload(payload) { } - void fill(FRT_Values & v) const override { - v.AddData(_payload.data(), _payload.size()); - } -private: - BlobRef _payload; -}; - -class FillByHandover : public PayLoadFiller -{ -public: - FillByHandover(Blob payload) : _payload(std::move(payload)) { } - void fill(FRT_Values & v) const override { - v.AddData(std::move(_payload.payload()), _payload.size()); - } -private: - mutable Blob _payload; -}; - -} - void -RPCSendV1::send(RoutingNode &recipient, const vespalib::Version &version, - BlobRef payload, uint64_t timeRemaining) +RPCSendV1::encodeRequest(FRT_RPCRequest &req, const vespalib::Version &version, const Route & route, + const RPCServiceAddress & address, const Message & msg, uint32_t traceLevel, + const PayLoadFiller &filler, uint64_t timeRemaining) const { - send(recipient, version, FillByCopy(payload), timeRemaining); -} - -void -RPCSendV1::sendByHandover(RoutingNode &recipient, const vespalib::Version &version, - Blob payload, uint64_t timeRemaining) -{ - send(recipient, version, FillByHandover(std::move(payload)), timeRemaining); -} -void -RPCSendV1::send(RoutingNode &recipient, const vespalib::Version &version, - const PayLoadFiller & payload, uint64_t timeRemaining) -{ - SendContext::UP ctx(new SendContext(recipient, timeRemaining)); - RPCServiceAddress &address = static_cast<RPCServiceAddress&>(recipient.getServiceAddress()); - const Message &msg = recipient.getMessage(); - Route route = recipient.getRoute(); - Hop hop = route.removeHop(0); - - FRT_RPCRequest *req = _net->allocRequest(); - FRT_Values &args = *req->GetParams(); - req->SetMethodName(METHOD_NAME); + FRT_Values &args = *req.GetParams(); + req.SetMethodName(METHOD_NAME); args.AddString(version.toString().c_str()); args.AddString(route.toString().c_str()); args.AddString(address.getSessionName().c_str()); @@ -163,237 +71,103 @@ RPCSendV1::send(RoutingNode &recipient, const vespalib::Version &version, args.AddInt32(msg.getRetry()); args.AddInt64(timeRemaining); args.AddString(msg.getProtocol().c_str()); - payload.fill(args); - args.AddInt32(recipient.getTrace().getLevel()); - - if (ctx->getTrace().shouldTrace(TraceLevel::SEND_RECEIVE)) { - ctx->getTrace().trace(TraceLevel::SEND_RECEIVE, - make_string("Sending message (version %s) from %s to '%s' with %.2f seconds timeout.", - version.toString().c_str(), _clientIdent.c_str(), - address.getServiceName().c_str(), ctx->getTimeout())); - } - - if (hop.getIgnoreResult()) { - address.getTarget().getFRTTarget().InvokeVoid(req); - if (ctx->getTrace().shouldTrace(TraceLevel::SEND_RECEIVE)) { - ctx->getTrace().trace(TraceLevel::SEND_RECEIVE, - make_string("Not waiting for a reply from '%s'.", address.getServiceName().c_str())); - } - Reply::UP reply(new EmptyReply()); - reply->getTrace().swap(ctx->getTrace()); - _net->getOwner().deliverReply(std::move(reply), recipient); - } else { - SendContext *ptr = ctx.release(); - req->SetContext(FNET_Context(ptr)); - address.getTarget().getFRTTarget().InvokeAsync(req, ptr->getTimeout(), this); - } + filler.fill(args); + args.AddInt32(traceLevel); } -void -RPCSendV1::RequestDone(FRT_RPCRequest *req) -{ - SendContext::UP ctx(static_cast<SendContext*>(req->GetContext()._value.VOIDP)); - const string &serviceName = static_cast<RPCServiceAddress&>( - ctx->getRecipient().getServiceAddress()).getServiceName(); - Reply::UP reply; - Error error; - if (!req->CheckReturnTypes(METHOD_RETURN)) { - reply.reset(new EmptyReply()); - switch (req->GetErrorCode()) { - case FRTE_RPC_TIMEOUT: - error = Error(ErrorCode::TIMEOUT, - make_string("A timeout occured while waiting for '%s' (%g seconds expired); %s", - serviceName.c_str(), ctx->getTimeout(), req->GetErrorMessage())); - break; - case FRTE_RPC_CONNECTION: - error = Error(ErrorCode::CONNECTION_ERROR, - make_string("A connection error occured for '%s'; %s", - serviceName.c_str(), req->GetErrorMessage())); - break; - default: - error = Error(ErrorCode::NETWORK_ERROR, - make_string("A network error occured for '%s'; %s", - serviceName.c_str(), req->GetErrorMessage())); - } - } else { - FRT_Values &ret = *req->GetReturn(); +namespace { - vespalib::Version version = vespalib::Version(ret[0]._string._str); - double retryDelay = ret[1]._double; - uint32_t *errorCodes = ret[2]._int32_array._pt; - uint32_t errorCodesLen = ret[2]._int32_array._len; - FRT_StringValue *errorMessages = ret[3]._string_array._pt; - uint32_t errorMessagesLen = ret[3]._string_array._len; - FRT_StringValue *errorServices = ret[4]._string_array._pt; - uint32_t errorServicesLen = ret[4]._string_array._len; - const char *protocolName = ret[5]._string._str; - const char *payload = ret[6]._data._buf; - uint32_t payloadLen = ret[6]._data._len; - const char *trace = ret[7]._string._str; +class ParamsV1 : public RPCSend::Params +{ +public: + ParamsV1(const FRT_Values &args) : _args(args) { } - if (payloadLen > 0) { - IProtocol * protocol = _net->getOwner().getProtocol(protocolName); - if (protocol != nullptr) { - Routable::UP routable = protocol->decode(version, BlobRef(payload, payloadLen)); - if (routable) { - if (routable->isReply()) { - reply.reset(static_cast<Reply*>(routable.release())); - } else { - error = Error(ErrorCode::DECODE_ERROR, - "Payload decoded to a message when expecting a reply."); - } - } else { - error = Error(ErrorCode::DECODE_ERROR, - make_string("Protocol '%s' failed to decode routable.", protocolName)); - } + uint32_t getTraceLevel() const override { return _args[8]._intval32; } + bool useRetry() const override { return _args[3]._intval8 != 0; } + uint32_t getRetries() const override { return _args[4]._intval32; } + uint64_t getRemainingTime() const override { return _args[5]._intval64; } - } else { - error = Error(ErrorCode::UNKNOWN_PROTOCOL, - make_string("Protocol '%s' is not known by %s.", protocolName, _serverIdent.c_str())); - } - } - if ( ! reply ) { - reply.reset(new EmptyReply()); - } - reply->setRetryDelay(retryDelay); - for (uint32_t i = 0; i < errorCodesLen && i < errorMessagesLen && i < errorServicesLen; ++i) { - reply->addError(Error(errorCodes[i], - errorMessages[i]._str, - errorServices[i]._len > 0 ? errorServices[i]._str : serviceName.c_str())); - } - ctx->getTrace().getRoot().addChild(TraceNode::decode(trace)); + vespalib::Version getVersion() const override { + return vespalib::Version(vespalib::stringref(_args[0]._string._str, _args[0]._string._len)); + } + vespalib::stringref getRoute() const override { + return vespalib::stringref(_args[1]._string._str, _args[1]._string._len); + } + vespalib::stringref getSession() const override { + return vespalib::stringref(_args[2]._string._str, _args[2]._string._len); } - if (ctx->getTrace().shouldTrace(TraceLevel::SEND_RECEIVE)) { - ctx->getTrace().trace(TraceLevel::SEND_RECEIVE, - make_string("Reply (type %d) received at %s.", reply->getType(), _clientIdent.c_str())); + vespalib::stringref getProtocol() const override { + return vespalib::stringref(_args[6]._string._str, _args[6]._string._len); } - reply->getTrace().swap(ctx->getTrace()); - if (error.getCode() != ErrorCode::NONE) { - reply->addError(error); + BlobRef getPayload() const override { + return BlobRef(_args[7]._data._buf, _args[7]._data._len); } - _net->getOwner().deliverReply(std::move(reply), ctx->getRecipient()); - req->SubRef(); +private: + const FRT_Values & _args; +}; + } -void -RPCSendV1::invoke(FRT_RPCRequest *req) +std::unique_ptr<RPCSend::Params> +RPCSendV1::toParams(const FRT_Values &args) const { - req->Detach(); + return std::make_unique<ParamsV1>(args); +} - FRT_Values &args = *req->GetParams(); - vespalib::Version version = vespalib::Version(args[0]._string._str); - const char *route = args[1]._string._str; - const char *session = args[2]._string._str; - bool retryEnabled = args[3]._intval8 != 0; - uint32_t retry = args[4]._intval32; - uint64_t timeRemaining = args[5]._intval64; - const char *protocolName = args[6]._string._str; - const char *payload = args[7]._data._buf; - uint32_t payloadLen = args[7]._data._len; - uint32_t traceLevel = args[8]._intval32; - IProtocol * protocol = _net->getOwner().getProtocol(protocolName); - if (protocol == nullptr) { - replyError(req, version, traceLevel, - Error(ErrorCode::UNKNOWN_PROTOCOL, - make_string("Protocol '%s' is not known by %s.", protocolName, _serverIdent.c_str()))); - return; - } - Routable::UP routable = protocol->decode(version, BlobRef(payload, payloadLen)); - req->DiscardBlobs(); - if ( ! routable ) { - replyError(req, version, traceLevel, - Error(ErrorCode::DECODE_ERROR, - make_string("Protocol '%s' failed to decode routable.", protocolName))); - return; - } - if (routable->isReply()) { - replyError(req, version, traceLevel, - Error(ErrorCode::DECODE_ERROR, - "Payload decoded to a reply when expecting a mesage.")); - return; +std::unique_ptr<Reply> +RPCSendV1::createReply(const FRT_Values & ret, const string & serviceName, Error & error, vespalib::TraceNode & rootTrace) const +{ + vespalib::Version version = vespalib::Version(ret[0]._string._str); + double retryDelay = ret[1]._double; + uint32_t *errorCodes = ret[2]._int32_array._pt; + uint32_t errorCodesLen = ret[2]._int32_array._len; + FRT_StringValue *errorMessages = ret[3]._string_array._pt; + uint32_t errorMessagesLen = ret[3]._string_array._len; + FRT_StringValue *errorServices = ret[4]._string_array._pt; + uint32_t errorServicesLen = ret[4]._string_array._len; + const char *protocolName = ret[5]._string._str; + BlobRef payload(ret[6]._data._buf, ret[6]._data._len); + const char *trace = ret[7]._string._str; + + Reply::UP reply; + if (payload.size() > 0) { + reply = decode(protocolName, version, payload, error); } - Message::UP msg(static_cast<Message*>(routable.release())); - if (strlen(route) > 0) { - msg->setRoute(Route::parse(route)); + if ( ! reply ) { + reply.reset(new EmptyReply()); } - msg->setContext(Context(new ReplyContext(*req, version))); - msg->pushHandler(*this, *this); - msg->setRetryEnabled(retryEnabled); - msg->setRetry(retry); - msg->setTimeReceivedNow(); - msg->setTimeRemaining(timeRemaining); - msg->getTrace().setLevel(traceLevel); - if (msg->getTrace().shouldTrace(TraceLevel::SEND_RECEIVE)) { - msg->getTrace().trace(TraceLevel::SEND_RECEIVE, - make_string("Message (type %d) received at %s for session '%s'.", - msg->getType(), _serverIdent.c_str(), session)); + reply->setRetryDelay(retryDelay); + for (uint32_t i = 0; i < errorCodesLen && i < errorMessagesLen && i < errorServicesLen; ++i) { + reply->addError(Error(errorCodes[i], errorMessages[i]._str, + errorServices[i]._len > 0 ? errorServices[i]._str : serviceName.c_str())); } - _net->getOwner().deliverMessage(std::move(msg), session); + rootTrace.addChild(TraceNode::decode(trace)); + return reply; } void -RPCSendV1::handleReply(Reply::UP reply) -{ - ReplyContext::UP ctx(static_cast<ReplyContext*>(reply->getContext().value.PTR)); - FRT_RPCRequest &req = ctx->getRequest(); - string version = ctx->getVersion().toString(); - if (reply->getTrace().shouldTrace(TraceLevel::SEND_RECEIVE)) { - reply->getTrace().trace(TraceLevel::SEND_RECEIVE, make_string("Sending reply (version %s) from %s.", - version.c_str(), _serverIdent.c_str())); - } - Blob payload(0); - if (reply->getType() != 0) { - payload = _net->getOwner().getProtocol(reply->getProtocol())->encode(ctx->getVersion(), *reply); - if (payload.size() == 0) { - reply->addError(Error(ErrorCode::ENCODE_ERROR, "An error occured while encoding the reply, see log.")); - } - } - FRT_Values &ret = *req.GetReturn(); +RPCSendV1::createResponse(FRT_Values & ret, const string & version, Reply & reply, Blob payload) const { ret.AddString(version.c_str()); - ret.AddDouble(reply->getRetryDelay()); + ret.AddDouble(reply.getRetryDelay()); - uint32_t errorCount = reply->getNumErrors(); + uint32_t errorCount = reply.getNumErrors(); uint32_t *errorCodes = ret.AddInt32Array(errorCount); FRT_StringValue *errorMessages = ret.AddStringArray(errorCount); FRT_StringValue *errorServices = ret.AddStringArray(errorCount); for (uint32_t i = 0; i < errorCount; ++i) { - errorCodes[i] = reply->getError(i).getCode(); - ret.SetString(errorMessages + i, - reply->getError(i).getMessage().c_str()); - ret.SetString(errorServices + i, - reply->getError(i).getService().c_str()); + errorCodes[i] = reply.getError(i).getCode(); + ret.SetString(errorMessages + i, reply.getError(i).getMessage().c_str()); + ret.SetString(errorServices + i, reply.getError(i).getService().c_str()); } - ret.AddString(reply->getProtocol().c_str()); + ret.AddString(reply.getProtocol().c_str()); ret.AddData(std::move(payload.payload()), payload.size()); - if (reply->getTrace().getLevel() > 0) { - ret.AddString(reply->getTrace().getRoot().encode().c_str()); + if (reply.getTrace().getLevel() > 0) { + ret.AddString(reply.getTrace().getRoot().encode().c_str()); } else { ret.AddString(""); } - req.Return(); -} - -void -RPCSendV1::handleDiscard(Context ctx) -{ - ReplyContext::UP tmp(static_cast<ReplyContext*>(ctx.value.PTR)); - FRT_RPCRequest &req = tmp->getRequest(); - FNET_Channel *chn = req.GetContext()._value.CHANNEL; - req.SubRef(); - chn->Free(); -} - -void -RPCSendV1::replyError(FRT_RPCRequest *req, const vespalib::Version &version, - uint32_t traceLevel, const Error &err) -{ - Reply::UP reply(new EmptyReply()); - reply->setContext(Context(new ReplyContext(*req, version))); - reply->getTrace().setLevel(traceLevel); - reply->addError(err); - handleReply(std::move(reply)); } } // namespace mbus diff --git a/messagebus/src/vespa/messagebus/network/rpcsendv1.h b/messagebus/src/vespa/messagebus/network/rpcsendv1.h index b634c1ef1b8..37f23335309 100644 --- a/messagebus/src/vespa/messagebus/network/rpcsendv1.h +++ b/messagebus/src/vespa/messagebus/network/rpcsendv1.h @@ -1,77 +1,24 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include "rpcsendadapter.h" -#include <vespa/messagebus/idiscardhandler.h> -#include <vespa/messagebus/ireplyhandler.h> -#include <vespa/fnet/frt/invokable.h> - +#include "rpcsend.h" namespace mbus { -class Error; - -class PayLoadFiller -{ +class RPCSendV1 : public RPCSend { public: - virtual ~PayLoadFiller() { } - virtual void fill(FRT_Values & v) const = 0; -}; - -/** - * Implements the send adapter for method "mbus.send". - */ -class RPCSendV1 : public RPCSendAdapter, - public FRT_Invokable, - public FRT_IRequestWait, - public IDiscardHandler, - public IReplyHandler { + static bool isCompatible(vespalib::stringref method, vespalib::stringref request, vespalib::stringref respons); private: - RPCNetwork *_net; - string _clientIdent; - string _serverIdent; - - /** - * Send an error reply for a given request. - * - * @param request The FRT request to reply to. - * @param version The version to serialize for. - * @param traceLevel The trace level to set in the reply. - * @param err The error to reply with. - */ - void replyError(FRT_RPCRequest *req, const vespalib::Version &version, - uint32_t traceLevel, const Error &err); - - void send(RoutingNode &recipient, const vespalib::Version &version, - const PayLoadFiller & filler, uint64_t timeRemaining); -public: - /** The name of the rpc method that this adapter registers. */ - static const char *METHOD_NAME; - - /** The parameter string of the rpc method. */ - static const char *METHOD_PARAMS; - - /** The return string of the rpc method. */ - static const char *METHOD_RETURN; - - /** - * Constructs a new instance of this adapter. This object is unusable until - * its attach() method has been called. - */ - RPCSendV1(); - ~RPCSendV1(); - - void attach(RPCNetwork &net) override; - - void send(RoutingNode &recipient, const vespalib::Version &version, - BlobRef payload, uint64_t timeRemaining) override; - void sendByHandover(RoutingNode &recipient, const vespalib::Version &version, - Blob payload, uint64_t timeRemaining) override; - - void handleReply(std::unique_ptr<Reply> reply) override; - void handleDiscard(Context ctx) override; - void invoke(FRT_RPCRequest *req); - void RequestDone(FRT_RPCRequest *req) override; + void build(FRT_ReflectionBuilder & builder) override; + const char * getReturnSpec() const override; + std::unique_ptr<Params> toParams(const FRT_Values ¶m) const override; + void encodeRequest(FRT_RPCRequest &req, const vespalib::Version &version, const Route & route, + const RPCServiceAddress & address, const Message & msg, uint32_t traceLevel, + const PayLoadFiller &filler, uint64_t timeRemaining) const override; + + std::unique_ptr<Reply> createReply(const FRT_Values & response, const string & serviceName, + Error & error, vespalib::TraceNode & rootTrace) const override; + void createResponse(FRT_Values & ret, const string & version, Reply & reply, Blob payload) const override; }; } // namespace mbus diff --git a/messagebus/src/vespa/messagebus/network/rpcsendv2.cpp b/messagebus/src/vespa/messagebus/network/rpcsendv2.cpp new file mode 100644 index 00000000000..1228e08f3b4 --- /dev/null +++ b/messagebus/src/vespa/messagebus/network/rpcsendv2.cpp @@ -0,0 +1,263 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "rpcsendv2.h" +#include "rpcnetwork.h" +#include "rpcserviceaddress.h" +#include <vespa/messagebus/emptyreply.h> +#include <vespa/messagebus/tracelevel.h> +#include <vespa/vespalib/util/stringfmt.h> +#include <vespa/vespalib/data/slime/slime.h> +#include <vespa/vespalib/data/databuffer.h> +#include <vespa/vespalib/util/compressor.h> +#include <vespa/fnet/frt/reflection.h> + +using vespalib::make_string; +using vespalib::compression::CompressionConfig; +using vespalib::compression::decompress; +using vespalib::compression::compress; +using vespalib::DataBuffer; +using vespalib::ConstBufferRef; +using vespalib::stringref; +using vespalib::Memory; +using vespalib::Slime; +using vespalib::Version; +using namespace vespalib::slime; + +namespace mbus { + +namespace { + +const char *METHOD_NAME = "mbus.slime"; +const char *METHOD_PARAMS = "bixbix"; +const char *METHOD_RETURN = "bixbix"; + +Memory VERSION_F("version"); +Memory ROUTE_F("route"); +Memory SESSION_F("session"); +Memory USERETRY_F("useretry"); +Memory RETRYDELAY_F("retrydelay"); +Memory RETRY_F("retry"); +Memory TIMELEFT_F("timeleft"); +Memory PROTOCOL_F("prot"); +Memory TRACELEVEL_F("tracelevel"); +Memory TRACE_F("trace"); +Memory BLOB_F("msg"); +Memory ERRORS_F("errors"); +Memory CODE_F("code"); +Memory MSG_F("msg"); +Memory SERVICE_F("service"); + +} + +bool RPCSendV2::isCompatible(stringref method, stringref request, stringref response) +{ + return (method == METHOD_NAME) && + (request == METHOD_PARAMS) && + (response == METHOD_RETURN); +} + +void +RPCSendV2::build(FRT_ReflectionBuilder & builder) +{ + builder.DefineMethod(METHOD_NAME, METHOD_PARAMS, METHOD_RETURN, true, FRT_METHOD(RPCSendV2::invoke), this); + builder.MethodDesc("Send a message bus slime request and get a reply back."); + builder.ParamDesc("header_encoding", "0=raw, 6=lz4"); + builder.ParamDesc("header_decoded_size", "Uncompressed header blob size"); + builder.ParamDesc("header_payload", "The message header blob in slime"); + builder.ParamDesc("body_encoding", "0=raw, 6=lz4"); + builder.ParamDesc("body_decoded_size", "Uncompressed body blob size"); + builder.ParamDesc("body_payload", "The message body blob in slime"); + builder.ReturnDesc("header_encoding", "0=raw, 6=lz4"); + builder.ReturnDesc("header_decoded_size", "Uncompressed header blob size"); + builder.ReturnDesc("header_payload", "The reply header blob in slime."); + builder.ReturnDesc("body_encoding", "0=raw, 6=lz4"); + builder.ReturnDesc("body_decoded_size", "Uncompressed body blob size"); + builder.ReturnDesc("body_payload", "The reply body blob in slime."); +} + +const char * +RPCSendV2::getReturnSpec() const { + return METHOD_RETURN; +} + +namespace { +class OutputBuf : public vespalib::Output { +public: + OutputBuf(size_t estimatedSize) : _buf(estimatedSize) { } + DataBuffer & getBuf() { return _buf; } +private: + vespalib::WritableMemory reserve(size_t bytes) override { + _buf.ensureFree(bytes); + return vespalib::WritableMemory(_buf.getFree(), _buf.getFreeLen()); + } + Output &commit(size_t bytes) override { + _buf.moveFreeToData(bytes); + return *this; + } + DataBuffer _buf; +}; +} + +void +RPCSendV2::encodeRequest(FRT_RPCRequest &req, const Version &version, const Route & route, + const RPCServiceAddress & address, const Message & msg, uint32_t traceLevel, + const PayLoadFiller &filler, uint64_t timeRemaining) const +{ + FRT_Values &args = *req.GetParams(); + req.SetMethodName(METHOD_NAME); + // Place holder for auxillary data to be transfered later. + args.AddInt8(CompressionConfig::NONE); + args.AddInt32(0); + args.AddData("", 0); + + Slime slime; + Cursor & root = slime.setObject(); + + root.setString(VERSION_F, version.toString()); + root.setString(ROUTE_F, route.toString()); + root.setString(SESSION_F, address.getSessionName()); + root.setBool(USERETRY_F, msg.getRetryEnabled()); + root.setLong(RETRY_F, msg.getRetry()); + root.setLong(TIMELEFT_F, timeRemaining); + root.setString(PROTOCOL_F, msg.getProtocol()); + root.setLong(TRACELEVEL_F, traceLevel); + filler.fill(BLOB_F, root); + + OutputBuf rBuf(8192); + BinaryFormat::encode(slime, rBuf); + ConstBufferRef toCompress(rBuf.getBuf().getData(), rBuf.getBuf().getDataLen()); + DataBuffer buf(vespalib::roundUp2inN(rBuf.getBuf().getDataLen())); + CompressionConfig::Type type = compress(_net->getCompressionConfig(), toCompress, buf, false); + + args.AddInt8(type); + args.AddInt32(toCompress.size()); + args.AddData(buf.stealBuffer(), buf.getDataLen()); +} + +namespace { + +class ParamsV2 : public RPCSend::Params +{ +public: + ParamsV2(const FRT_Values &arg) + : _slime() + { + uint8_t encoding = arg[3]._intval8; + uint32_t uncompressedSize = arg[4]._intval32; + DataBuffer uncompressed(arg[5]._data._buf, arg[5]._data._len); + ConstBufferRef blob(arg[5]._data._buf, arg[5]._data._len); + decompress(CompressionConfig::toType(encoding), uncompressedSize, blob, uncompressed, true); + assert(uncompressedSize == uncompressed.getDataLen()); + BinaryFormat::decode(Memory(uncompressed.getData(), uncompressed.getDataLen()), _slime); + } + + uint32_t getTraceLevel() const override { return _slime.get()[TRACELEVEL_F].asLong(); } + bool useRetry() const override { return _slime.get()[USERETRY_F].asBool(); } + uint32_t getRetries() const override { return _slime.get()[RETRY_F].asLong(); } + uint64_t getRemainingTime() const override { return _slime.get()[TIMELEFT_F].asLong(); } + + Version getVersion() const override { + return Version(_slime.get()[VERSION_F].asString().make_stringref()); + } + stringref getRoute() const override { + return _slime.get()[ROUTE_F].asString().make_stringref(); + } + stringref getSession() const override { + return _slime.get()[SESSION_F].asString().make_stringref(); + } + stringref getProtocol() const override { + return _slime.get()[PROTOCOL_F].asString().make_stringref(); + } + BlobRef getPayload() const override { + Memory m = _slime.get()[BLOB_F].asData(); + return BlobRef(m.data, m.size); + } +private: + Slime _slime; +}; + +} + +std::unique_ptr<RPCSend::Params> +RPCSendV2::toParams(const FRT_Values &args) const +{ + return std::make_unique<ParamsV2>(args); +} + +std::unique_ptr<Reply> +RPCSendV2::createReply(const FRT_Values & ret, const string & serviceName, + Error & error, vespalib::TraceNode & rootTrace) const +{ + uint8_t encoding = ret[3]._intval8; + uint32_t uncompressedSize = ret[4]._intval32; + DataBuffer uncompressed(ret[5]._data._buf, ret[5]._data._len); + ConstBufferRef blob(ret[5]._data._buf, ret[5]._data._len); + decompress(CompressionConfig::toType(encoding), uncompressedSize, blob, uncompressed, true); + assert(uncompressedSize == uncompressed.getDataLen()); + Slime slime; + BinaryFormat::decode(Memory(uncompressed.getData(), uncompressed.getDataLen()), slime); + Inspector & root = slime.get(); + Version version(root[VERSION_F].asString().make_string()); + Memory payload = root[BLOB_F].asData(); + + Reply::UP reply; + if (payload.size > 0) { + reply = decode(root[PROTOCOL_F].asString().make_stringref(), version, BlobRef(payload.data, payload.size), error); + } + if ( ! reply ) { + reply.reset(new EmptyReply()); + } + reply->setRetryDelay(root[RETRYDELAY_F].asDouble()); + Inspector & errors = root[ERRORS_F]; + for (uint32_t i = 0; i < errors.entries(); ++i) { + Inspector & e = errors[i]; + Memory service = e[SERVICE_F].asString(); + reply->addError(Error(e[CODE_F].asLong(), e[MSG_F].asString().make_string(), + (service.size > 0) ? service.make_string() : serviceName)); + } + rootTrace.addChild(TraceNode::decode(root[TRACE_F].asString().make_string())); + return reply; +} + +void +RPCSendV2::createResponse(FRT_Values & ret, const string & version, Reply & reply, Blob payload) const +{ + // Place holder for auxillary data to be transfered later. + ret.AddInt8(CompressionConfig::NONE); + ret.AddInt32(0); + ret.AddData("", 0); + + Slime slime; + Cursor & root = slime.setObject(); + + root.setString(VERSION_F, version); + root.setDouble(RETRYDELAY_F, reply.getRetryDelay()); + root.setString(PROTOCOL_F, reply.getProtocol()); + root.setData(BLOB_F, vespalib::Memory(payload.data(), payload.size())); + if (reply.getTrace().getLevel() > 0) { + root.setString(TRACE_F, reply.getTrace().getRoot().encode()); + } + + if (reply.getNumErrors() > 0) { + Cursor & array = root.setArray(ERRORS_F); + for (uint32_t i = 0; i < reply.getNumErrors(); ++i) { + Cursor & error = array.addObject(); + error.setLong(CODE_F, reply.getError(i).getCode()); + error.setString(MSG_F, reply.getError(i).getMessage()); + error.setString(SERVICE_F, reply.getError(i).getService().c_str()); + } + } + + OutputBuf rBuf(8192); + BinaryFormat::encode(slime, rBuf); + ConstBufferRef toCompress(rBuf.getBuf().getData(), rBuf.getBuf().getDataLen()); + DataBuffer buf(vespalib::roundUp2inN(rBuf.getBuf().getDataLen())); + CompressionConfig::Type type = compress(_net->getCompressionConfig(), toCompress, buf, false); + + ret.AddInt8(type); + ret.AddInt32(toCompress.size()); + ret.AddData(buf.stealBuffer(), buf.getDataLen()); + +} + +} // namespace mbus diff --git a/messagebus/src/vespa/messagebus/network/rpcsendv2.h b/messagebus/src/vespa/messagebus/network/rpcsendv2.h new file mode 100644 index 00000000000..e793868d2aa --- /dev/null +++ b/messagebus/src/vespa/messagebus/network/rpcsendv2.h @@ -0,0 +1,24 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include "rpcsend.h" + +namespace mbus { + +class RPCSendV2 : public RPCSend { +public: + static bool isCompatible(vespalib::stringref method, vespalib::stringref request, vespalib::stringref response); +private: + void build(FRT_ReflectionBuilder & builder) override; + const char * getReturnSpec() const override; + std::unique_ptr<Params> toParams(const FRT_Values ¶m) const override; + void encodeRequest(FRT_RPCRequest &req, const vespalib::Version &version, const Route & route, + const RPCServiceAddress & address, const Message & msg, uint32_t traceLevel, + const PayLoadFiller &filler, uint64_t timeRemaining) const override; + + std::unique_ptr<Reply> createReply(const FRT_Values & response, const string & serviceName, + Error & error, vespalib::TraceNode & rootTrace) const override; + void createResponse(FRT_Values & ret, const string & version, Reply & reply, Blob payload) const override; +}; + +} // namespace mbus diff --git a/messagebus/src/vespa/messagebus/network/rpcserviceaddress.h b/messagebus/src/vespa/messagebus/network/rpcserviceaddress.h index c6856057342..36dde19bd18 100644 --- a/messagebus/src/vespa/messagebus/network/rpcserviceaddress.h +++ b/messagebus/src/vespa/messagebus/network/rpcserviceaddress.h @@ -84,7 +84,7 @@ public: * * @return True if target is set. */ - bool hasTarget() const { return _target.get() != NULL; } + bool hasTarget() const { return _target.get() != nullptr; } }; } // namespace mbus diff --git a/messagebus/src/vespa/messagebus/network/rpctargetpool.cpp b/messagebus/src/vespa/messagebus/network/rpctargetpool.cpp index 20e5a2eb3e3..295814f4a8d 100644 --- a/messagebus/src/vespa/messagebus/network/rpctargetpool.cpp +++ b/messagebus/src/vespa/messagebus/network/rpctargetpool.cpp @@ -36,7 +36,7 @@ RPCTargetPool::flushTargets(bool force) TargetMap::iterator it = _targets.begin(); while (it != _targets.end()) { Entry &entry = it->second; - if (entry._target.get() != NULL) { + if (entry._target.get() != nullptr) { if (entry._target.use_count() > 1) { entry._lastUse = currentTime; ++it; diff --git a/messagebus/src/vespa/messagebus/network/rpctargetpool.h b/messagebus/src/vespa/messagebus/network/rpctargetpool.h index 683982de080..5f858f66993 100644 --- a/messagebus/src/vespa/messagebus/network/rpctargetpool.h +++ b/messagebus/src/vespa/messagebus/network/rpctargetpool.h @@ -1,11 +1,11 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include <map> -#include <vespa/messagebus/itimer.h> -#include <vespa/vespalib/util/sync.h> #include "rpcserviceaddress.h" #include "rpctarget.h" +#include <vespa/messagebus/itimer.h> +#include <vespa/vespalib/util/sync.h> +#include <map> class FRT_Supervisor; diff --git a/messagebus/src/vespa/messagebus/protocolrepository.cpp b/messagebus/src/vespa/messagebus/protocolrepository.cpp index d2661e3ef80..4e2efcfb3b9 100644 --- a/messagebus/src/vespa/messagebus/protocolrepository.cpp +++ b/messagebus/src/vespa/messagebus/protocolrepository.cpp @@ -76,7 +76,7 @@ ProtocolRepository::getRoutingPolicy(const string &protocolName, } catch (const std::exception &e) { LOG(error, "Protocol '%s' threw an exception; %s", protocolName.c_str(), e.what()); } - if (policy.get() == NULL) { + if (policy.get() == nullptr) { LOG(error, "Protocol '%s' failed to create routing policy '%s' with parameter '%s'.", protocolName.c_str(), policyName.c_str(), policyParam.c_str()); return IRoutingPolicy::SP(); diff --git a/messagebus/src/vespa/messagebus/routing/route.cpp b/messagebus/src/vespa/messagebus/routing/route.cpp index af7b5113ac2..b705847c3a5 100644 --- a/messagebus/src/vespa/messagebus/routing/route.cpp +++ b/messagebus/src/vespa/messagebus/routing/route.cpp @@ -69,7 +69,7 @@ Route::toDebugString() const { } Route -Route::parse(const string &route) +Route::parse(vespalib::stringref route) { return RouteParser::createRoute(route); } diff --git a/messagebus/src/vespa/messagebus/routing/route.h b/messagebus/src/vespa/messagebus/routing/route.h index d9932e17d26..a2a01648cfe 100644 --- a/messagebus/src/vespa/messagebus/routing/route.h +++ b/messagebus/src/vespa/messagebus/routing/route.h @@ -1,7 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include <vector> #include "hop.h" namespace mbus { @@ -33,7 +32,7 @@ public: * @param route The string to parse. * @return A route that corresponds to the string. */ - static Route parse(const string &route); + static Route parse(vespalib::stringref route); /** * Create a Route that contains no hops diff --git a/messagebus/src/vespa/messagebus/routing/routeparser.cpp b/messagebus/src/vespa/messagebus/routing/routeparser.cpp index ac52ae3b598..668f14f9801 100644 --- a/messagebus/src/vespa/messagebus/routing/routeparser.cpp +++ b/messagebus/src/vespa/messagebus/routing/routeparser.cpp @@ -71,7 +71,7 @@ RouteParser::createDirective(const stringref &str) } Hop -RouteParser::createHop(const stringref &str) +RouteParser::createHop(stringref str) { if (str.empty()) { return Hop().addDirective(createErrorDirective("Failed to parse empty string.")); @@ -84,7 +84,7 @@ RouteParser::createHop(const stringref &str) } if (len > 4 && str.substr(0, 4) == "tcp/") { IHopDirective::SP tcp = createTcpDirective(str.substr(4)); - if (tcp.get() != NULL) { + if (tcp.get() != nullptr) { return Hop().addDirective(tcp); } } @@ -119,7 +119,7 @@ RouteParser::createHop(const stringref &str) } Route -RouteParser::createRoute(const stringref &str) +RouteParser::createRoute(stringref str) { Route ret; for (size_t from = 0, at = 0, depth = 0; at <= str.size(); ++at) { diff --git a/messagebus/src/vespa/messagebus/routing/routeparser.h b/messagebus/src/vespa/messagebus/routing/routeparser.h index a3f16e49307..8ffba3f6e11 100644 --- a/messagebus/src/vespa/messagebus/routing/routeparser.h +++ b/messagebus/src/vespa/messagebus/routing/routeparser.h @@ -29,7 +29,7 @@ public: * @param str The string to parse as a hop. * @return The created hop. */ - static Hop createHop(const vespalib::stringref &str); + static Hop createHop(vespalib::stringref str); /** * Creates a route from a string representation. @@ -37,7 +37,7 @@ public: * @param str The string to parse as a route. * @return The created route. */ - static Route createRoute(const vespalib::stringref &str); + static Route createRoute(vespalib::stringref str); }; } // mbus diff --git a/messagebus/src/vespa/messagebus/routing/routingnode.cpp b/messagebus/src/vespa/messagebus/routing/routingnode.cpp index 62efda4aeb9..6e100999e1d 100644 --- a/messagebus/src/vespa/messagebus/routing/routingnode.cpp +++ b/messagebus/src/vespa/messagebus/routing/routingnode.cpp @@ -21,7 +21,7 @@ RoutingNode::RoutingNode(MessageBus &mbus, INetwork &net, Resender *resender, : _mbus(mbus), _net(net), _resender(resender), - _parent(NULL), + _parent(nullptr), _recipients(), _children(), _replyHandler(&replyHandler), @@ -45,8 +45,8 @@ RoutingNode::RoutingNode(RoutingNode &parent, const Route &route) _parent(&parent), _recipients(parent._recipients), _children(), - _replyHandler(NULL), - _discardHandler(NULL), + _replyHandler(nullptr), + _discardHandler(nullptr), _trace(parent._trace.getLevel()), _pending(0), _msg(parent._msg), @@ -78,8 +78,8 @@ RoutingNode::clearChildren() void RoutingNode::discard() { - assert(_parent == NULL); - if (_discardHandler != NULL) { + assert(_parent == nullptr); + if (_discardHandler != nullptr) { _discardHandler->handleDiscard(Context()); } } @@ -101,7 +101,7 @@ RoutingNode::prepareForRetry() { _shouldRetry = false; _reply.reset(); - if (_routingContext.get() != NULL && _routingContext->getSelectOnRetry()) { + if (_routingContext.get() != nullptr && _routingContext->getSelectOnRetry()) { clearChildren(); } else if (!_children.empty()) { bool retryingSome = false; @@ -109,7 +109,7 @@ RoutingNode::prepareForRetry() it != _children.end(); ++it) { RoutingNode *child= *it; - if (child->_shouldRetry || child->_reply.get() == NULL) { + if (child->_shouldRetry || child->_reply.get() == nullptr) { child->prepareForRetry(); retryingSome = true; } @@ -126,11 +126,11 @@ RoutingNode::prepareForRetry() void RoutingNode::notifyParent() { - if (_serviceAddress.get() != NULL) { + if (_serviceAddress.get() != nullptr) { _net.freeServiceAddress(*this); } tryIgnoreResult(); - if (_parent != NULL) { + if (_parent != nullptr) { _parent->notifyMerge(); return; } @@ -174,7 +174,7 @@ RoutingNode::addError(uint32_t code, const string &msg) void RoutingNode::addError(const Error &err) { - if (_reply.get() != NULL) { + if (_reply.get() != nullptr) { _reply->getTrace().swap(_trace); _reply->addError(err); _reply->getTrace().swap(_trace); @@ -186,8 +186,8 @@ RoutingNode::addError(const Error &err) void RoutingNode::setReply(Reply::UP reply) { - if (reply.get() != NULL) { - _shouldRetry = _resender != NULL && _resender->shouldRetry(*reply); + if (reply.get() != nullptr) { + _shouldRetry = _resender != nullptr && _resender->shouldRetry(*reply); _trace.getRoot().addChild(reply->getTrace().getRoot()); reply->getTrace().clear(); } @@ -211,7 +211,7 @@ RoutingNode::notifyAbort(const string &msg) mystack.pop(); if (!node->_isActive) { // reply not pending - } else if (node->_reply.get() != NULL) { + } else if (node->_reply.get() != nullptr) { node->notifyParent(); } else if (node->_children.empty()) { node->setError(ErrorCode::SEND_ABORTED, msg); @@ -240,7 +240,7 @@ RoutingNode::notifyTransmit() if (node->hasReply()) { node->notifyParent(); } else { - assert(node->_serviceAddress.get() != NULL); + assert(node->_serviceAddress.get() != nullptr); sendTo.push_back(node); } } else { @@ -296,7 +296,7 @@ RoutingNode::notifyMerge() setError(ErrorCode::POLICY_ERROR, make_string("Policy '%s' threw an exception; %s", dir.getName().c_str(), e.what())); } - if (_reply.get() == NULL) { + if (_reply.get() == nullptr) { setError(ErrorCode::APP_FATAL_ERROR, make_string("Routing policy '%s' failed to merge replies.", dir.getName().c_str())); } @@ -315,12 +315,12 @@ RoutingNode::hasUnconsumedErrors() while (!mystack.empty()) { RoutingNode *node = mystack.top(); mystack.pop(); - if (node->_reply.get() != NULL) { + if (node->_reply.get() != nullptr) { for (uint32_t i = 0; i < node->_reply->getNumErrors(); ++i) { int errorCode = node->_reply->getError(i).getCode(); RoutingNode *it = node; - while (it != NULL) { - if (it->_routingContext.get() != NULL && + while (it != nullptr) { + if (it->_routingContext.get() != nullptr && it->_routingContext->isConsumableError(errorCode)) { errorCode = ErrorCode::NONE; @@ -329,7 +329,7 @@ RoutingNode::hasUnconsumedErrors() it = it->_parent; } if (errorCode != ErrorCode::NONE) { - _shouldRetry = _resender != NULL && _resender->canRetry(errorCode); + _shouldRetry = _resender != nullptr && _resender->canRetry(errorCode); if (!_shouldRetry) { return true; // no need to continue } @@ -374,17 +374,17 @@ RoutingNode::resolve(uint32_t depth) if (executePolicySelect()) { return resolveChildren(depth + 1); } - return _reply.get() != NULL; + return _reply.get() != nullptr; } _net.allocServiceAddress(*this); - return _serviceAddress.get() != NULL || _reply.get() != NULL; + return _serviceAddress.get() != nullptr || _reply.get() != nullptr; } bool RoutingNode::lookupHop() { RoutingTable::SP table = _mbus.getRoutingTable(_msg.getProtocol()); - if (table.get() != NULL) { + if (table.get() != nullptr) { string name = _route.getHop(0).getServiceName(); if (table->hasHop(name)) { const HopBlueprint *hop = table->getHop(name); @@ -404,7 +404,7 @@ RoutingNode::lookupRoute() Hop &hop = _route.getHop(0); if (hop.getDirective(0)->getType() == IHopDirective::TYPE_ROUTE) { RouteDirective &dir = static_cast<RouteDirective&>(*hop.getDirective(0)); - if (table.get() == NULL || !table->hasRoute(dir.getName())) { + if (table.get() == nullptr || !table->hasRoute(dir.getName())) { setError(ErrorCode::ILLEGAL_ROUTE, make_string("Route '%s' does not exist.", dir.getName().c_str())); return false; @@ -415,7 +415,7 @@ RoutingNode::lookupRoute() dir.getName().c_str(), _route.toString().c_str())); return true; } - if (table.get() != NULL) { + if (table.get() != nullptr) { string name = hop.getServiceName(); if (table->hasRoute(name)) { insertRoute(*table->getRoute(name)); @@ -474,7 +474,7 @@ RoutingNode::executePolicySelect() { const PolicyDirective &dir = _routingContext->getDirective(); _policy = _mbus.getRoutingPolicy(_msg.getProtocol(), dir.getName(), dir.getParam()); - if (_policy.get() == NULL) { + if (_policy.get() == nullptr) { setError(ErrorCode::UNKNOWN_POLICY, make_string( "Protocol '%s' could not create routing policy '%s' with parameter '%s'.", _msg.getProtocol().c_str(), dir.getName().c_str(), dir.getParam().c_str())); @@ -489,7 +489,7 @@ RoutingNode::executePolicySelect() return false; } if (_children.empty()) { - if (_reply.get() == NULL) { + if (_reply.get() == nullptr) { setError(ErrorCode::NO_SERVICES_FOR_ROUTE, make_string("Policy '%s' selected no recipients for route '%s'.", dir.getName().c_str(), _route.toString().c_str())); @@ -522,7 +522,7 @@ RoutingNode::resolveChildren(uint32_t childDepth) RoutingNode *child = *it; child->_trace.trace(TraceLevel::SPLIT_MERGE, make_string("Resolving '%s'.", child->_route.toString().c_str())); - child->_isActive = (child->_reply.get() == NULL); + child->_isActive = (child->_reply.get() == nullptr); if (child->_isActive) { ++numActiveChildren; if (!child->resolve(childDepth)) { @@ -562,7 +562,7 @@ RoutingNode::tryIgnoreResult() if (!shouldIgnoreResult()) { return false; } - if (_reply.get() == NULL || !_reply->hasErrors()) { + if (_reply.get() == nullptr || !_reply->hasErrors()) { return false; } setReply(Reply::UP(new EmptyReply())); diff --git a/messagebus/src/vespa/messagebus/routing/routingnode.h b/messagebus/src/vespa/messagebus/routing/routingnode.h index 22ff07c26e5..8951785c621 100644 --- a/messagebus/src/vespa/messagebus/routing/routingnode.h +++ b/messagebus/src/vespa/messagebus/routing/routingnode.h @@ -228,7 +228,7 @@ public: */ RoutingNode(MessageBus &mbus, INetwork &net, Resender *resender, IReplyHandler &replyHandler, Message &msg, - IDiscardHandler *discardHandler = NULL); + IDiscardHandler *discardHandler = nullptr); /** * Destructor. Frees up any allocated resources, namely all child nodes of @@ -348,6 +348,7 @@ public: * @return The message being routed. */ Message &getMessage() { return _msg; } + const Message & getMessage() const { return _msg; } /** * Returns the trace object for this node. Each node has a separate trace @@ -356,6 +357,7 @@ public: * @return The trace object. */ Trace &getTrace() { return _trace; } + const Trace &getTrace() const { return _trace; } /** * Returns the route object as it exists at this point of the tree. @@ -369,7 +371,7 @@ public: * * @return True if this node has a reply. */ - bool hasReply() const { return _reply.get() != NULL; } + bool hasReply() const { return _reply.get() != nullptr; } /** * Returns the reply of this node. @@ -419,7 +421,7 @@ public: * * @return True if an address is set. */ - bool hasServiceAddress() { return _serviceAddress.get() != NULL; } + bool hasServiceAddress() { return _serviceAddress.get() != nullptr; } /** * Returns the service address of this node. This is attached by the network @@ -429,6 +431,7 @@ public: * @return The recipient address. */ IServiceAddress &getServiceAddress() { return *_serviceAddress; } + const IServiceAddress &getServiceAddress() const { return *_serviceAddress; } /** * Sets the service address of this node. This is called by the network diff --git a/messagebus/src/vespa/messagebus/routing/routingtable.cpp b/messagebus/src/vespa/messagebus/routing/routingtable.cpp index 7537605d9fa..58e1881dc90 100644 --- a/messagebus/src/vespa/messagebus/routing/routingtable.cpp +++ b/messagebus/src/vespa/messagebus/routing/routingtable.cpp @@ -54,7 +54,7 @@ const HopBlueprint * RoutingTable::getHop(const string &name) const { std::map<string, HopBlueprint>::const_iterator it = _hops.find(name); - return it != _hops.end() ? &(it->second) : NULL; + return it != _hops.end() ? &(it->second) : nullptr; } bool @@ -67,7 +67,7 @@ const Route * RoutingTable::getRoute(const string &name) const { std::map<string, Route>::const_iterator it = _routes.find(name); - return it != _routes.end() ? &(it->second) : NULL; + return it != _routes.end() ? &(it->second) : nullptr; } } // namespace mbus diff --git a/messagebus/src/vespa/messagebus/rpcmessagebus.cpp b/messagebus/src/vespa/messagebus/rpcmessagebus.cpp index c093efd1106..103c21ee3aa 100644 --- a/messagebus/src/vespa/messagebus/rpcmessagebus.cpp +++ b/messagebus/src/vespa/messagebus/rpcmessagebus.cpp @@ -1,5 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "rpcmessagebus.h" +#include <vespa/config/subscription/configuri.h> namespace mbus { @@ -15,6 +16,10 @@ RPCMessageBus::RPCMessageBus(const MessageBusParams &mbusParams, _subscriber.start(); } +RPCMessageBus::RPCMessageBus(const MessageBusParams &mbusParams, const RPCNetworkParams &rpcParams) + : RPCMessageBus(mbusParams, rpcParams, config::ConfigUri("client")) +{} + RPCMessageBus::RPCMessageBus(const ProtocolSet &protocols, const RPCNetworkParams &rpcParams, const config::ConfigUri &routingCfgUri) : diff --git a/messagebus/src/vespa/messagebus/rpcmessagebus.h b/messagebus/src/vespa/messagebus/rpcmessagebus.h index 3f249965fec..e9cfed32c44 100644 --- a/messagebus/src/vespa/messagebus/rpcmessagebus.h +++ b/messagebus/src/vespa/messagebus/rpcmessagebus.h @@ -7,6 +7,8 @@ #include <vespa/messagebus/network/rpcnetwork.h> #include <vespa/config/helper/legacysubscriber.h> +namespace config {class ConfigUri; } + namespace mbus { /** @@ -40,8 +42,10 @@ public: * @param routingCfgId The config id for message bus routing specs. */ RPCMessageBus(const MessageBusParams &mbusParams, - const RPCNetworkParams &rpcParams = RPCNetworkParams(), - const config::ConfigUri & routingCfgId = config::ConfigUri("client")); + const RPCNetworkParams &rpcParams, + const config::ConfigUri & routingCfgId); + RPCMessageBus(const MessageBusParams &mbusParams, + const RPCNetworkParams &rpcParams); /** @@ -55,8 +59,8 @@ public: * @param routingCfgId The config id for messagebus routing specs. */ RPCMessageBus(const ProtocolSet &protocols, - const RPCNetworkParams &rpcParams = RPCNetworkParams(), - const config::ConfigUri & routingCfgId = config::ConfigUri("client")); + const RPCNetworkParams &rpcParams, + const config::ConfigUri & routingCfgId); /** * Destruct. This will destruct the internal MessageBus and RPCNetwork diff --git a/messagebus/src/vespa/messagebus/sequencer.cpp b/messagebus/src/vespa/messagebus/sequencer.cpp index 79fbd346c16..60fb3bdd39e 100644 --- a/messagebus/src/vespa/messagebus/sequencer.cpp +++ b/messagebus/src/vespa/messagebus/sequencer.cpp @@ -19,7 +19,7 @@ Sequencer::~Sequencer() { for (QueueMap::iterator it = _seqMap.begin(); it != _seqMap.end(); ++it) { MessageQueue *queue = it->second; - if (queue != NULL) { + if (queue != nullptr) { while (queue->size() > 0) { Message *msg = queue->front(); queue->pop(); @@ -40,7 +40,7 @@ Sequencer::filter(Message::UP msg) vespalib::LockGuard guard(_lock); QueueMap::iterator it = _seqMap.find(seqId); if (it != _seqMap.end()) { - if (it->second == NULL) { + if (it->second == nullptr) { it->second = new MessageQueue(); } msg->getTrace().trace(TraceLevel::COMPONENT, @@ -49,7 +49,7 @@ Sequencer::filter(Message::UP msg) msg.release(); return Message::UP(); } - _seqMap[seqId] = NULL; // insert empty queue + _seqMap[seqId] = nullptr; // insert empty queue } return std::move(msg); } @@ -69,7 +69,7 @@ Sequencer::handleMessage(Message::UP msg) { if (msg->hasSequenceId()) { msg = filter(std::move(msg)); - if (msg.get() != NULL) { + if (msg.get() != nullptr) { sequencedSend(std::move(msg)); } } else { @@ -89,8 +89,8 @@ Sequencer::handleReply(Reply::UP reply) QueueMap::iterator it = _seqMap.find(seq); MessageQueue *que = it->second; assert(it != _seqMap.end()); - if (que == NULL || que->size() == 0) { - if (que != NULL) { + if (que == nullptr || que->size() == 0) { + if (que != nullptr) { delete que; } _seqMap.erase(it); @@ -99,7 +99,7 @@ Sequencer::handleReply(Reply::UP reply) que->pop(); } } - if (msg.get() != NULL) { + if (msg.get() != nullptr) { sequencedSend(std::move(msg)); } IReplyHandler &handler = reply->getCallStack().pop(*reply); diff --git a/messagebus/src/vespa/messagebus/sourcesession.cpp b/messagebus/src/vespa/messagebus/sourcesession.cpp index 9a93a4aedf1..1dbdd307e17 100644 --- a/messagebus/src/vespa/messagebus/sourcesession.cpp +++ b/messagebus/src/vespa/messagebus/sourcesession.cpp @@ -41,9 +41,9 @@ SourceSession::send(Message::UP msg, const string &routeName, bool parseIfNotFou { bool found = false; RoutingTable::SP rt = _mbus.getRoutingTable(msg->getProtocol()); - if (rt.get() != NULL) { + if (rt.get() != nullptr) { const Route *route = rt->getRoute(routeName); - if (route != NULL) { + if (route != nullptr) { msg->setRoute(*route); found = true; } else if (!parseIfNotFound) { @@ -79,13 +79,13 @@ SourceSession::send(Message::UP msg) if (_closed) { return Result(Error(ErrorCode::SEND_QUEUE_CLOSED, "Source session is closed."), std::move(msg)); } - if (_throttlePolicy.get() != NULL && !_throttlePolicy->canSend(*msg, _pendingCount)) { + if (_throttlePolicy.get() != nullptr && !_throttlePolicy->canSend(*msg, _pendingCount)) { return Result(Error(ErrorCode::SEND_QUEUE_FULL, make_string("Too much pending data (%d messages).", _pendingCount)), std::move(msg)); } msg->pushHandler(_replyHandler); - if (_throttlePolicy.get() != NULL) { + if (_throttlePolicy.get() != nullptr) { _throttlePolicy->processMessage(*msg); } ++_pendingCount; @@ -108,7 +108,7 @@ SourceSession::handleReply(Reply::UP reply) vespalib::MonitorGuard guard(_monitor); assert(_pendingCount > 0); --_pendingCount; - if (_throttlePolicy.get() != NULL) { + if (_throttlePolicy.get() != nullptr) { _throttlePolicy->processReply(*reply); } done = (_closed && _pendingCount == 0); diff --git a/messagebus/src/vespa/messagebus/sourcesessionparams.cpp b/messagebus/src/vespa/messagebus/sourcesessionparams.cpp index 125b2a9822f..51fe91562ae 100644 --- a/messagebus/src/vespa/messagebus/sourcesessionparams.cpp +++ b/messagebus/src/vespa/messagebus/sourcesessionparams.cpp @@ -6,7 +6,7 @@ namespace mbus { SourceSessionParams::SourceSessionParams() : - _replyHandler(NULL), + _replyHandler(nullptr), _throttlePolicy(new DynamicThrottlePolicy()), _timeout(180.0) { } @@ -40,7 +40,7 @@ SourceSessionParams::setTimeout(double timeout) bool SourceSessionParams::hasReplyHandler() const { - return _replyHandler != NULL; + return _replyHandler != nullptr; } IReplyHandler & diff --git a/messagebus/src/vespa/messagebus/testlib/testserver.cpp b/messagebus/src/vespa/messagebus/testlib/testserver.cpp index e7f3646c72c..dbc741f2dd4 100644 --- a/messagebus/src/vespa/messagebus/testlib/testserver.cpp +++ b/messagebus/src/vespa/messagebus/testlib/testserver.cpp @@ -4,6 +4,7 @@ #include "simpleprotocol.h" #include "slobrok.h" #include "slobrokstate.h" +#include <vespa/messagebus/network/oosmanager.h> #include <vespa/vespalib/component/vtag.h> namespace mbus { @@ -11,9 +12,7 @@ namespace mbus { VersionedRPCNetwork::VersionedRPCNetwork(const RPCNetworkParams ¶ms) : RPCNetwork(params), _version(vespalib::Vtag::currentVersion) -{ - // empty -} +{} void VersionedRPCNetwork::setVersion(const vespalib::Version &version) @@ -97,4 +96,4 @@ TestServer::waitState(const OOSState &oosState) return false; } -} // namespace mbus +} diff --git a/messagebus/src/vespa/messagebus/testlib/testserver.h b/messagebus/src/vespa/messagebus/testlib/testserver.h index 1e2de3c4607..400e2b274c5 100644 --- a/messagebus/src/vespa/messagebus/testlib/testserver.h +++ b/messagebus/src/vespa/messagebus/testlib/testserver.h @@ -2,9 +2,10 @@ #pragma once -#include <memory> #include <vespa/messagebus/messagebus.h> #include <vespa/messagebus/network/rpcnetwork.h> +#include <vespa/messagebus/network/rpcnetworkparams.h> +#include <vespa/fnet/frt/supervisor.h> namespace mbus { diff --git a/messagebus/src/vespa/messagebus/tracenode.h b/messagebus/src/vespa/messagebus/tracenode.h index 95de9e70f55..f582a70a151 100644 --- a/messagebus/src/vespa/messagebus/tracenode.h +++ b/messagebus/src/vespa/messagebus/tracenode.h @@ -5,7 +5,7 @@ namespace mbus { - typedef vespalib::TraceNode TraceNode; + using TraceNode = vespalib::TraceNode; } // namespace mbus diff --git a/messagebus_test/src/tests/error/cpp-client.cpp b/messagebus_test/src/tests/error/cpp-client.cpp index 001b0b14674..44580400e26 100644 --- a/messagebus_test/src/tests/error/cpp-client.cpp +++ b/messagebus_test/src/tests/error/cpp-client.cpp @@ -1,13 +1,11 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/messagebus/messagebus.h> -#include <vespa/messagebus/sourcesession.h> #include <vespa/messagebus/errorcode.h> #include <vespa/messagebus/testlib/simplemessage.h> -#include <vespa/messagebus/testlib/simplereply.h> #include <vespa/messagebus/testlib/simpleprotocol.h> #include <vespa/messagebus/rpcmessagebus.h> -#include <vespa/messagebus/sourcesessionparams.h> +#include <vespa/messagebus/network/rpcnetworkparams.h> #include <vespa/messagebus/testlib/receptor.h> #include <vespa/fastos/app.h> diff --git a/messagebus_test/src/tests/error/cpp-server.cpp b/messagebus_test/src/tests/error/cpp-server.cpp index 68a1d0afc27..af0d6e30132 100644 --- a/messagebus_test/src/tests/error/cpp-server.cpp +++ b/messagebus_test/src/tests/error/cpp-server.cpp @@ -3,6 +3,7 @@ #include <vespa/messagebus/messagebus.h> #include <vespa/messagebus/testlib/simpleprotocol.h> #include <vespa/messagebus/rpcmessagebus.h> +#include <vespa/messagebus/network/rpcnetworkparams.h> #include <vespa/messagebus/emptyreply.h> #include <vespa/messagebus/errorcode.h> #include <vespa/fastos/app.h> diff --git a/messagebus_test/src/tests/speed/cpp-client.cpp b/messagebus_test/src/tests/speed/cpp-client.cpp index d510054c0ab..437cc3ce354 100644 --- a/messagebus_test/src/tests/speed/cpp-client.cpp +++ b/messagebus_test/src/tests/speed/cpp-client.cpp @@ -3,6 +3,7 @@ #include <vespa/messagebus/messagebus.h> #include <vespa/messagebus/routing/retrytransienterrorspolicy.h> #include <vespa/messagebus/rpcmessagebus.h> +#include <vespa/messagebus/network/rpcnetworkparams.h> #include <vespa/messagebus/testlib/simplemessage.h> #include <vespa/messagebus/testlib/simpleprotocol.h> #include <vespa/messagebus/testlib/simplereply.h> diff --git a/messagebus_test/src/tests/speed/cpp-server.cpp b/messagebus_test/src/tests/speed/cpp-server.cpp index ba821f19b02..ddbeac62ce2 100644 --- a/messagebus_test/src/tests/speed/cpp-server.cpp +++ b/messagebus_test/src/tests/speed/cpp-server.cpp @@ -5,6 +5,7 @@ #include <vespa/messagebus/testlib/simplereply.h> #include <vespa/messagebus/testlib/simpleprotocol.h> #include <vespa/messagebus/rpcmessagebus.h> +#include <vespa/messagebus/network/rpcnetworkparams.h> #include <vespa/fastos/app.h> using namespace mbus; diff --git a/messagebus_test/src/tests/trace/cpp-server.cpp b/messagebus_test/src/tests/trace/cpp-server.cpp index 60bd55270a7..c9086ff1758 100644 --- a/messagebus_test/src/tests/trace/cpp-server.cpp +++ b/messagebus_test/src/tests/trace/cpp-server.cpp @@ -3,6 +3,7 @@ #include <vespa/messagebus/messagebus.h> #include <vespa/messagebus/testlib/simpleprotocol.h> #include <vespa/messagebus/rpcmessagebus.h> +#include <vespa/messagebus/network/rpcnetworkparams.h> #include <vespa/messagebus/emptyreply.h> #include <vespa/fastos/app.h> diff --git a/messagebus_test/src/tests/trace/trace.cpp b/messagebus_test/src/tests/trace/trace.cpp index 96759f286b8..a653c4934b3 100644 --- a/messagebus_test/src/tests/trace/trace.cpp +++ b/messagebus_test/src/tests/trace/trace.cpp @@ -1,6 +1,5 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/log/log.h> -LOG_SETUP("trace_test"); + #include <vespa/vespalib/testkit/testapp.h> #include <vespa/vespalib/testkit/test_kit.h> #include <vespa/messagebus/testlib/slobrok.h> @@ -21,6 +20,9 @@ LOG_SETUP("trace_test"); #include <vespa/messagebus/testlib/simpleprotocol.h> #include <iostream> +#include <vespa/log/log.h> +LOG_SETUP("trace_test"); + using namespace mbus; using vespalib::make_string; diff --git a/metrics/src/vespa/metrics/CMakeLists.txt b/metrics/src/vespa/metrics/CMakeLists.txt index 4e7558fcfde..27a1f25e887 100644 --- a/metrics/src/vespa/metrics/CMakeLists.txt +++ b/metrics/src/vespa/metrics/CMakeLists.txt @@ -24,4 +24,4 @@ vespa_add_library(metrics DEPENDS ) vespa_generate_config(metrics metricsmanager.def) -install(FILES metricsmanager.def DESTINATION var/db/vespa/config_server/serverdb/classes) +install(FILES metricsmanager.def RENAME metrics.metricsmanager.def DESTINATION var/db/vespa/config_server/serverdb/classes) diff --git a/node-admin/README.md b/node-admin/README.md index bab4c17ffbf..a2f0ba2c4ec 100644 --- a/node-admin/README.md +++ b/node-admin/README.md @@ -1,111 +1,3 @@ # Node Admin -## Setup - -Set up Docker on your machine according to the instructions in [Linux](README_LINUX.md) or [Mac](README_MAC.md), depending on your hardware. - -You should have the docker daemon running and the following environment variables set: -``` -VESPA_HOME -VESPA_WEB_SERVICE_PORT -``` - -To update `/etc/hosts` with the required hostnames for the local containers, run -``` -sudo ./scripts/etc-hosts.sh -``` - -## Developing - -We will describe how you can build a Docker image for Vespa which will be used -to set up a local Docker container with the Node Admin, and a local container -with the Config Server. - -Then, we'll show how you bring up this local zone. And finally, how you can -deploy a local Vespa application to this zone. - -[RunVespaLocal.java](src/test/java/com/yahoo/vespa/hosted/node/admin/docker/RunVespaLocal.java) -implements all of the basic methods you need to get started. - -### Starting a Local Zone - -To start a local zone, simply run: -``` - DockerImage dockerImage = new DockerImage("docker-registry.domain.tld:8080/vespa/ci:6.111.21"); - Path pathToVespaRoot = Paths.get("/home/valerijf/dev/vespa"); - Path pathToContainerStorage = Paths.get("/home/docker/container-storage"); - - RunVespaLocal runVespaLocal = new RunVespaLocal(pathToVespaRoot); - runVespaLocal.startLocalZoneWithNodes(dockerImage, 5); - runVespaLocal.startNodeAdminAsContainer(dockerImage, pathToContainerStorage); -``` - -### Deploying a Local Application - -Package and deploy vespa application by running: - -``` - Path pathToApp = pathToVespaRoot.resolve("sample-apps/blog-search"); - runVespaLocal.deployApplication(pathToApp); -``` -If the deployment is successful, the final log entry should be something like -``` -INFO: Endpoint http://cnode-1:4080/ is now ready -``` -use this endpoint URL to feed or query your application. - -You can delete application with - -``` - runVespaLocal.deleteApplication(); -``` - -### Feed and search - 1. **Feed** the data that is to be searched - ```sh - - # Feeding two documents - curl -X POST --data-binary @music-data-1.json <endpoint url>/document/v1/music/music/docid/1 | python -m json.tool - curl -X POST --data-binary @music-data-2.json <endpoint url>/document/v1/music/music/docid/2 | python -m json.tool - - ``` - - 2. **Visit documents** - - Since we do not have many documents we can list them all - ```sh - - # All documents - curl <endpoint url>/document/v1/music/music/docid | python -m json.tool - - # Document with id 1 - curl <endpoint url>/document/v1/music/music/docid/1 | python -m json.tool - - ``` - - 3. **Search** - We can also search for documents: - ```sh - - curl '<endpoint url>/search/?query=bad' | python -m json.tool - - - ``` - -## Troubleshooting - -If the container doesn't start, it can be helpful to look at the jdisc log. First, find the container id: -``` -docker ps -a -``` - -Then, find the log files: -``` -docker diff <container id>| grep $VESPA_HOME/logs -``` - -View the log file (`-L` follows the symbolic link): -``` -docker cp -L <container id>:$VESPA_HOME/logs/jdisc_core/jdisc_core.log - | less -``` - +Manages docker containers that run different applications on a host.
\ No newline at end of file diff --git a/node-admin/README_LINUX.md b/node-admin/README_LINUX.md deleted file mode 100644 index 56c1fb60a7e..00000000000 --- a/node-admin/README_LINUX.md +++ /dev/null @@ -1,52 +0,0 @@ -# Setting up Docker on a linux machine - -First, install Docker. With Fedora 21 you should follow -https://docs.docker.com/engine/installation/linux/fedora/, which describes how to install -Docker. - -## Configuring Docker - -Docker will by default download (huge) images to a directory under `/var`. On our -Fedora machines, `/var` is part of the root filesystem, which does not have a lot -of free space. Since docker runs as `root`, it is allowed to completely fill up -the filesystem, and it will happily do so. Fedora works very poorly with a full -root filesystem. You won't even be able to log in and clean up the disk usage -once it's happened. - -So you'll want to store images somewhere else. An obvious choice is `/home`, -which typically has a lot more room. Make Docker use directories in the docker -user's home directory, set the `--graph` option by editing -`/etc/systemd/system/docker.service.d/docker.conf` to: - -``` -[Service] -ExecStart= -ExecStart=/usr/bin/dockerd \ - --debug \ - --graph=/home/docker/data \ - --host=127.0.0.1:2376 \ - --host=unix:///var/run/docker.sock \ - --selinux-enabled \ - --storage-driver=devicemapper \ - --storage-opt=dm.basesize=20G -``` - -Finally, start docker: -``` -sudo systemctl start docker -``` - - - -## Other - -For more information on how to configure the docker daemon, see https://docs.docker.com/engine/admin/systemd/. - -## Upgrade of Docker - -When Docker upgrades it may overwrite /lib/systemd/system/docker.service. The -symptom is that any docker command will fail with the error message "Cannot -connect to the Docker daemon. Is the docker daemon running on this host?". - -Once you have updated docker.service according to this document, and restarted -the Docker daemon, Docker should work again. diff --git a/node-admin/README_MAC.md b/node-admin/README_MAC.md deleted file mode 100644 index 49c723b2cd6..00000000000 --- a/node-admin/README_MAC.md +++ /dev/null @@ -1,39 +0,0 @@ -# Setting up Docker on OS X -Install Docker Toolbox according to the procedure on [https://www.docker.com/products/docker-toolbox](https://www.docker.com/products/docker-toolbox). -Note: We need to use Docker Toolbox instead of Docker for Mac for running node-admin or system tests, since we need to -configure networking per container. - -# Running Vespa on OS X - -## Starting the VM -On OS X the docker daemon is running inside a VM called boot2docker. This VM is running using the -VirtualBox virtualization software. To setup and start the VM for the first time, execute the -following script: - -``` -docker-machine create -d virtualbox default -``` -You should now have a Docker machine up and running. This can be verified with: - -``` -docker-machine ls -``` - -which should list the running ```default``` machine. - -Regular ```docker``` commands works as in Linux when you have the environment variables set. -Look in [README.md](README.md) for useful docker commands. - -## Running Vespa applications or system tests - -Before running any applications you need to make containers visible for your Mac: -``` -sudo route add 172.18.0.0/16 192.168.99.100 -``` - -Follow the instructions in [README.md](README.md) for starting local zones and deploying applications. - -## Issues -* Accessing Vespa from OS X while on a Cisco VPN connection does not work. This is because the VPN client will protect the routing table on OS X. - * Workaround is to use ```docker-machine ssh vespa``` and then execute everything from inside the VM. - diff --git a/node-admin/configserver-app/hosted-vespa/http-server.xml b/node-admin/configserver-app/hosted-vespa/http-server.xml deleted file mode 100644 index 369a1af810b..00000000000 --- a/node-admin/configserver-app/hosted-vespa/http-server.xml +++ /dev/null @@ -1,2 +0,0 @@ -<!-- Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> -<server port="4080" id="configserver-real" /> diff --git a/node-admin/configserver-app/node-flavors.xml b/node-admin/configserver-app/node-flavors.xml deleted file mode 100644 index d1892edddb3..00000000000 --- a/node-admin/configserver-app/node-flavors.xml +++ /dev/null @@ -1,15 +0,0 @@ -<!-- Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> -<config name="config.provisioning.flavors"> - <flavor> - <item> - <!-- Note: This flavor does NOT match the production 'docker' flavor --> - <name>docker</name> - <environment>DOCKER_CONTAINER</environment> - <minCpuCores>1.0</minCpuCores> - <minMainMemoryAvailableGb>3.0</minMainMemoryAvailableGb> - <minDiskAvailableGb>10.0</minDiskAvailableGb> - <description>DOCKER_CONTAINER with 1.0 CPUs, 3.0 Gb memory and 10.0 Gb disk</description> - </item> - </flavor> -</config> - diff --git a/node-admin/configserver-app/start-config-server.sh b/node-admin/configserver-app/start-config-server.sh deleted file mode 100755 index f9cf49ed879..00000000000 --- a/node-admin/configserver-app/start-config-server.sh +++ /dev/null @@ -1,116 +0,0 @@ -#!/bin/bash -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -# BEGIN environment bootstrap section -# Do not edit between here and END as this section should stay identical in all scripts - -findpath () { - myname=${0} - mypath=${myname%/*} - myname=${myname##*/} - if [ "$mypath" ] && [ -d "$mypath" ]; then - return - fi - mypath=$(pwd) - if [ -f "${mypath}/${myname}" ]; then - return - fi - echo "FATAL: Could not figure out the path where $myname lives from $0" - exit 1 -} - -COMMON_ENV=libexec/vespa/common-env.sh - -source_common_env () { - if [ "$VESPA_HOME" ] && [ -d "$VESPA_HOME" ]; then - export VESPA_HOME - common_env=$VESPA_HOME/$COMMON_ENV - if [ -f "$common_env" ]; then - . $common_env - return - fi - fi - return 1 -} - -findroot () { - source_common_env && return - if [ "$VESPA_HOME" ]; then - echo "FATAL: bad VESPA_HOME value '$VESPA_HOME'" - exit 1 - fi - if [ "$ROOT" ] && [ -d "$ROOT" ]; then - VESPA_HOME="$ROOT" - source_common_env && return - fi - findpath - while [ "$mypath" ]; do - VESPA_HOME=${mypath} - source_common_env && return - mypath=${mypath%/*} - done - echo "FATAL: missing VESPA_HOME environment variable" - echo "Could not locate $COMMON_ENV anywhere" - exit 1 -} - -findroot - -# END environment bootstrap section - -export LC_ALL=C - -function VerifyRequiredEnvironmentVariablesAreSet { - if [ -z "$HOSTED_VESPA_REGION" ] - then - Fail "Environment variable HOSTED_VESPA_REGION is not set" - fi - if [ -z "$CONFIG_SERVER_HOSTNAME" ] - then - Fail "Environment variable CONFIG_SERVER_HOSTNAME is not set" - fi - - case "$HOSTED_VESPA_ENVIRONMENT" in - prod|test|dev|staging|perf) : ;; - *) Fail "The HOSTED_VESPA_ENVIRONMENT environment variable must be one of prod, test, dev, staging, or perf" ;; - esac -} - -function InternalMain { - VerifyRequiredEnvironmentVariablesAreSet - - mkdir -p $VESPA_HOME/logs - chmod 1777 $VESPA_HOME/logs - mkdir -p $VESPA_HOME/logs/jdisc_core - - rm -rf $VESPA_HOME/var/vespa/bundlecache/standalone - - yinst set \ - cloudconfig_server.multitenant=true \ - cloudconfig_server.region="$HOSTED_VESPA_REGION" \ - cloudconfig_server.autostart=on \ - cloudconfig_server.default_flavor=docker \ - cloudconfig_server.environment="$HOSTED_VESPA_ENVIRONMENT" \ - cloudconfig_server.hosted_vespa=true \ - services.addr_configserver="$CONFIG_SERVER_HOSTNAME" \ - vespa_zkfacade.restrict="" - - # Can also set jvmargs if necessary: - # set cloudconfig_server.jvmargs=-Dvespa.freezedetector.disable=true -XX:NewRatio=1 -verbose:gc -XX:+PrintGCDateStamps -agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=5005 -Xms6g -Xmx6g - - yinst start cloudconfig_server - - touch $VESPA_HOME/logs/jdisc_core/jdisc_core.log - $VESPA_HOME/bin/vespa-logfmt -N -f $VESPA_HOME/logs/jdisc_core/jdisc_core.log -} - -function Main { - # Prefix each line to stdout/stderr with a timestamp to make it easier to - # understand the progress. - InternalMain |& while read -r - do - printf "%s %s\n" "$(date +%FT%T)" "$REPLY" - done -} - -Main "$@" diff --git a/node-admin/node-admin-zone-app/services.xml b/node-admin/node-admin-zone-app/services.xml deleted file mode 100644 index 1ab9debd225..00000000000 --- a/node-admin/node-admin-zone-app/services.xml +++ /dev/null @@ -1,20 +0,0 @@ -<?xml version="1.0" encoding="utf-8" ?> -<!-- Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> -<services version="1.0" application-type="hosted-infrastructure" xmlns:deploy="vespa" xmlns:preprocess="properties"> - - <jdisc id="node-admin" jetty="true" version="1.0"> - <handler id="com.yahoo.vespa.hosted.node.admin.restapi.RestApiHandler" bundle="node-admin"> - <binding>http://*/rest/*</binding> - </handler> - <component id="node-admin" class="com.yahoo.vespa.hosted.node.admin.provider.ComponentsProviderImpl" bundle="node-admin"/> - <component id="docker-api" class="com.yahoo.vespa.hosted.dockerapi.DockerImpl" bundle="docker-api"/> - <component id="metrics-wrapper" class="com.yahoo.vespa.hosted.dockerapi.metrics.MetricReceiverWrapper" bundle="docker-api"/> - - <config name='vespa.hosted.dockerapi.docker'> - <isRunningLocally>true</isRunningLocally> - </config> - - <nodes jvmargs="-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=localhost:5555 -Dvespa.freezedetector.disable=true" type="host"/> - </jdisc> - -</services> diff --git a/node-admin/src/main/application/services.xml b/node-admin/src/main/application/services.xml deleted file mode 100644 index a60c11fb935..00000000000 --- a/node-admin/src/main/application/services.xml +++ /dev/null @@ -1,22 +0,0 @@ -<?xml version="1.0" encoding="utf-8" ?> -<!-- Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> -<services version="1.0"> - <jdisc version="1.0" jetty="true"> - <!-- Please update container test when changing this file --> - <handler id="com.yahoo.vespa.hosted.node.admin.restapi.RestApiHandler" bundle="node-admin"> - <binding>http://*/rest/*</binding> - </handler> - <component id="node-admin" class="com.yahoo.vespa.hosted.node.admin.provider.ComponentsProviderImpl" bundle="node-admin"/> - <component id="docker-api" class="com.yahoo.vespa.hosted.dockerapi.DockerImpl" bundle="docker-api"/> - <component id="metrics-wrapper" class="com.yahoo.vespa.hosted.dockerapi.metrics.MetricReceiverWrapper" bundle="docker-api"/> - - <config name='vespa.hosted.dockerapi.docker'> - <uri>tcp://localhost:2376</uri> - <isRunningLocally>false</isRunningLocally> - </config> - - <config name='vespa.hosted.node.admin.node-admin'> - <isRunningLocally>false</isRunningLocally> - </config> - </jdisc> -</services> diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImpl.java index b21fc573c0f..f6fd8c3bd18 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImpl.java @@ -22,7 +22,6 @@ import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; -import java.util.regex.Pattern; import java.util.stream.Stream; import static com.yahoo.vespa.defaults.Defaults.getDefaults; @@ -40,8 +39,6 @@ public class DockerOperationsImpl implements DockerOperations { private static final String[] RESTART_VESPA_ON_NODE_COMMAND = new String[]{NODE_PROGRAM, "restart-vespa"}; private static final String[] STOP_NODE_COMMAND = new String[]{NODE_PROGRAM, "stop"}; - private static final Pattern VESPA_VERSION_PATTERN = Pattern.compile("^(\\S*)$", Pattern.MULTILINE); - private static final String MANAGER_NAME = "node-admin"; // Map of directories to mount and whether they should be writable by everyone @@ -116,10 +113,6 @@ public class DockerOperationsImpl implements DockerOperations { .withAddCapability("SYS_PTRACE") // Needed for gcore, pstack etc. .withAddCapability("SYS_ADMIN"); // Needed for perf - if (environment.isRunningLocally()) { - command.withEntrypoint("/usr/local/bin/start-services.sh", "--run-local"); - } - command.withVolume("/etc/hosts", "/etc/hosts"); for (String pathInNode : DIRECTORIES_TO_MOUNT.keySet()) { String pathInHost = environment.pathInHostFromPathInNode(containerName, pathInNode).toString(); diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java index 2450759a5c2..cac924cbf18 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java @@ -319,7 +319,8 @@ public class StorageMaintainer { private String executeMaintainer(String mainClass, String... args) { String[] command = Stream.concat( - Stream.of("sudo", "-E", getDefaults().underVespaHome("libexec/vespa/node-admin/maintenance.sh"), mainClass), + Stream.of("sudo", "VESPA_HOME=" + getDefaults().vespaHome(), + getDefaults().underVespaHome("libexec/vespa/node-admin/maintenance.sh"), mainClass), Stream.of(args)) .toArray(String[]::new); diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdmin.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdmin.java index e02f81e8f30..cf70963eee1 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdmin.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdmin.java @@ -62,5 +62,5 @@ public interface NodeAdmin { /** * Stop the NodeAgent. Will not delete the storage or stop the container. */ - void shutdown(); + void stop(); } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImpl.java index 84679d1dadd..f227a166034 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImpl.java @@ -46,7 +46,7 @@ public class NodeAdminImpl implements NodeAdmin { private final DockerOperations dockerOperations; private final Function<String, NodeAgent> nodeAgentFactory; - private final Optional<StorageMaintainer> storageMaintainer; + private final StorageMaintainer storageMaintainer; private final Clock clock; private boolean previousWantFrozen; @@ -55,22 +55,18 @@ public class NodeAdminImpl implements NodeAdmin { private final Map<ContainerName, NodeAgent> nodeAgents = new ConcurrentHashMap<>(); - private final int nodeAgentScanIntervalMillis; - private final GaugeWrapper numberOfContainersInLoadImageState; private final CounterWrapper numberOfUnhandledExceptionsInNodeAgent; public NodeAdminImpl(final DockerOperations dockerOperations, final Function<String, NodeAgent> nodeAgentFactory, - final Optional<StorageMaintainer> storageMaintainer, - final int nodeAgentScanIntervalMillis, + final StorageMaintainer storageMaintainer, + final AclMaintainer aclMaintainer, final MetricReceiverWrapper metricReceiver, - final Optional<AclMaintainer> aclMaintainer, final Clock clock) { this.dockerOperations = dockerOperations; this.nodeAgentFactory = nodeAgentFactory; this.storageMaintainer = storageMaintainer; - this.nodeAgentScanIntervalMillis = nodeAgentScanIntervalMillis; this.clock = clock; this.previousWantFrozen = true; @@ -89,9 +85,9 @@ public class NodeAdminImpl implements NodeAdmin { } }, 0, 55, TimeUnit.SECONDS); - aclMaintainer.ifPresent(maintainer -> aclScheduler.scheduleWithFixedDelay(() -> { - if (!isFrozen()) maintainer.run(); - }, 30, 60, TimeUnit.SECONDS)); + aclScheduler.scheduleWithFixedDelay(() -> { + if (!isFrozen()) aclMaintainer.run(); + }, 30, 60, TimeUnit.SECONDS); } @Override @@ -101,7 +97,7 @@ public class NodeAdminImpl implements NodeAdmin { .map(container -> container.hostname) .collect(Collectors.toList()); - storageMaintainer.ifPresent(StorageMaintainer::cleanNodeAdmin); + storageMaintainer.cleanNodeAdmin(); synchronizeNodeSpecsToNodeAgents(containersToRunHostnames, existingContainerNames); dockerOperations.deleteUnusedDockerImages(); @@ -183,24 +179,21 @@ public class NodeAdminImpl implements NodeAdmin { } @Override - public void shutdown() { + public void stop() { metricsScheduler.shutdown(); aclScheduler.shutdown(); - try { - boolean metricsSchedulerShutdown = metricsScheduler.awaitTermination(30, TimeUnit.SECONDS); - boolean aclSchedulerShutdown = aclScheduler.awaitTermination(30, TimeUnit.SECONDS); - if (! (metricsSchedulerShutdown && aclSchedulerShutdown)) { - throw new RuntimeException("Failed shutting down all scheduler(s), shutdown status:\n" + - "\tMetrics Scheduler: " + metricsSchedulerShutdown + "\n" + - "\tACL Scheduler: " + aclSchedulerShutdown); - } - } catch (InterruptedException e) { - throw new RuntimeException(e); - } - for (NodeAgent nodeAgent : nodeAgents.values()) { - nodeAgent.stop(); - } + // Stop all node-agents in parallel, will block until the last NodeAgent is stopped + nodeAgents.values().parallelStream().forEach(NodeAgent::stop); + + do { + try { + metricsScheduler.awaitTermination(Long.MAX_VALUE, TimeUnit.NANOSECONDS); + aclScheduler.awaitTermination(Long.MAX_VALUE, TimeUnit.NANOSECONDS); + } catch (InterruptedException e) { + logger.info("Was interrupted while waiting for metricsScheduler and aclScheduler to shutdown"); + } + } while (!metricsScheduler.isTerminated() || !aclScheduler.isTerminated()); } // Set-difference. Returns minuend minus subtrahend. @@ -257,7 +250,7 @@ public class NodeAdminImpl implements NodeAdmin { } final NodeAgent agent = nodeAgentFactory.apply(hostname); - agent.start(nodeAgentScanIntervalMillis); + agent.start(); nodeAgents.put(containerName, agent); try { Thread.sleep(1000); diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdater.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdater.java index 1f5d6ddd300..d1f23b13e6c 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdater.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdater.java @@ -1,7 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.node.admin.nodeadmin; -import com.yahoo.component.AbstractComponent; import com.yahoo.concurrent.ThreadFactoryFactory; import com.yahoo.log.LogLevel; import com.yahoo.vespa.hosted.node.admin.ContainerNodeSpec; @@ -20,7 +19,6 @@ import java.util.ArrayList; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; -import java.util.Optional; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; @@ -37,8 +35,8 @@ import static com.yahoo.vespa.hosted.node.admin.nodeadmin.NodeAdminStateUpdater. * * @author dybis, stiankri */ -public class NodeAdminStateUpdater extends AbstractComponent { - public static final Duration FREEZE_CONVERGENCE_TIMEOUT = Duration.ofMinutes(5); +public class NodeAdminStateUpdater { + static final Duration FREEZE_CONVERGENCE_TIMEOUT = Duration.ofMinutes(5); private final AtomicBoolean terminated = new AtomicBoolean(false); private State currentState = SUSPENDED_NODE_ADMIN; @@ -53,35 +51,31 @@ public class NodeAdminStateUpdater extends AbstractComponent { private Thread loopThread; private final NodeRepository nodeRepository; + private final Orchestrator orchestrator; + private final StorageMaintainer storageMaintainer; private final NodeAdmin nodeAdmin; private final Clock clock; - private final Orchestrator orchestrator; private final String dockerHostHostName; + private final Duration nodeAdminConvergeStateInterval; - private long delaysBetweenEachTickMillis = 30_000; private Instant lastTick; public NodeAdminStateUpdater( - final NodeRepository nodeRepository, - final NodeAdmin nodeAdmin, - Optional<StorageMaintainer> storageMaintainer, - Clock clock, + NodeRepository nodeRepository, Orchestrator orchestrator, - String dockerHostHostName) { - log.log(LogLevel.INFO, objectToString() + ": Creating object"); + StorageMaintainer storageMaintainer, + NodeAdmin nodeAdmin, + String dockerHostHostName, + Clock clock, + Duration nodeAdminConvergeStateInterval) { this.nodeRepository = nodeRepository; - this.nodeAdmin = nodeAdmin; - this.clock = clock; this.orchestrator = orchestrator; + this.storageMaintainer = storageMaintainer; + this.nodeAdmin = nodeAdmin; this.dockerHostHostName = dockerHostHostName; + this.clock = clock; + this.nodeAdminConvergeStateInterval = nodeAdminConvergeStateInterval; this.lastTick = clock.instant(); - - storageMaintainer.ifPresent(maintainer -> specVerifierScheduler.scheduleWithFixedDelay(() -> - updateHardwareDivergence(maintainer), 5, 60, TimeUnit.MINUTES)); - } - - private String objectToString() { - return this.getClass().getSimpleName() + "@" + Integer.toString(System.identityHashCode(this)); } public enum State { RESUMED, SUSPENDED_NODE_ADMIN, SUSPENDED} @@ -134,7 +128,8 @@ public class NodeAdminStateUpdater extends AbstractComponent { State wantedStateCopy; synchronized (monitor) { while (! workToDoNow) { - long remainder = delaysBetweenEachTickMillis - Duration.between(lastTick, clock.instant()).toMillis(); + Duration timeSinceLastConverge = Duration.between(lastTick, clock.instant()); + long remainder = nodeAdminConvergeStateInterval.minus(timeSinceLastConverge).toMillis(); if (remainder > 0) { try { monitor.wait(remainder); @@ -232,7 +227,7 @@ public class NodeAdminStateUpdater extends AbstractComponent { } final List<ContainerNodeSpec> containersToRun; try { - containersToRun = nodeRepository.getContainersToRun(); + containersToRun = nodeRepository.getContainersToRun(dockerHostHostName); } catch (Exception e) { log.log(LogLevel.WARNING, "Failed fetching container info from node repository", e); return; @@ -251,7 +246,7 @@ public class NodeAdminStateUpdater extends AbstractComponent { private List<String> getNodesInActiveState() { try { - return nodeRepository.getContainersToRun() + return nodeRepository.getContainersToRun(dockerHostHostName) .stream() .filter(nodespec -> nodespec.nodeState == Node.State.active) .map(nodespec -> nodespec.hostname) @@ -261,8 +256,7 @@ public class NodeAdminStateUpdater extends AbstractComponent { } } - public void start(long stateConvergeInterval) { - delaysBetweenEachTickMillis = stateConvergeInterval; + public void start() { if (loopThread != null) { throw new RuntimeException("Can not restart NodeAdminStateUpdater"); } @@ -272,24 +266,30 @@ public class NodeAdminStateUpdater extends AbstractComponent { }); loopThread.setName("tick-NodeAdminStateUpdater"); loopThread.start(); + + specVerifierScheduler.scheduleWithFixedDelay(() -> + updateHardwareDivergence(storageMaintainer), 5, 60, TimeUnit.MINUTES); } - @Override - public void deconstruct() { + public void stop() { + specVerifierScheduler.shutdown(); if (!terminated.compareAndSet(false, true)) { throw new RuntimeException("Can not re-stop a node agent."); } - log.log(LogLevel.INFO, objectToString() + ": Deconstruct called"); + + // First we need to stop NodeAdminStateUpdater thread to make sure no new NodeAgents are spawned signalWorkToBeDone(); - try { - loopThread.join(10000); - if (loopThread.isAlive()) { - log.log(LogLevel.ERROR, "Could not stop tick thread"); + + do { + try { + loopThread.join(); + specVerifierScheduler.awaitTermination(Long.MAX_VALUE, TimeUnit.NANOSECONDS); + } catch (InterruptedException e1) { + log.info("Interrupted while waiting for NodeAdminStateUpdater thread and specVerfierScheduler to shutdown"); } - } catch (InterruptedException e1) { - log.log(LogLevel.ERROR, "Interrupted; Could not stop thread"); - } - nodeAdmin.shutdown(); - log.log(LogLevel.INFO, objectToString() + ": Deconstruct complete"); + } while (loopThread.isAlive() || !specVerifierScheduler.isTerminated()); + + // Finally, stop NodeAdmin and all the NodeAgents + nodeAdmin.stop(); } } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgent.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgent.java index 5d31c10fcc1..92c44969d5e 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgent.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgent.java @@ -29,7 +29,7 @@ public interface NodeAgent { * Starts the agent. After this method is called, the agent will asynchronously maintain the node, continuously * striving to make the current state equal to the wanted state. */ - void start(int intervalMillis); + void start(); /** * Signals to the agent that the node is at the end of its lifecycle and no longer needs a managing agent. diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java index 29484c08472..77348a9dc45 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java @@ -64,20 +64,20 @@ public class NodeAgentImpl implements NodeAgent { private final PrefixLogger logger; private DockerImage imageBeingDownloaded = null; - private final String hostname; private final ContainerName containerName; + private final String hostname; private final NodeRepository nodeRepository; private final Orchestrator orchestrator; private final DockerOperations dockerOperations; - private final Optional<StorageMaintainer> storageMaintainer; + private final StorageMaintainer storageMaintainer; + private final AclMaintainer aclMaintainer; private final Environment environment; private final Clock clock; - private final Optional<AclMaintainer> aclMaintainer; + private final Duration timeBetweenEachConverge; private final SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); private final LinkedList<String> debugMessages = new LinkedList<>(); - private long delaysBetweenEachConvergeMillis = 30_000; private int numberOfUnhandledException = 0; private Instant lastConverge; @@ -85,7 +85,7 @@ public class NodeAgentImpl implements NodeAgent { private final ScheduledExecutorService filebeatRestarter = Executors.newScheduledThreadPool(1, ThreadFactoryFactory.getDaemonThreadFactory("filebeatrestarter")); - private final Consumer<String> serviceRestarter; + private Consumer<String> serviceRestarter; private Future<?> currentFilebeatRestarter; private boolean resumeScriptRun = false; @@ -114,33 +114,23 @@ public class NodeAgentImpl implements NodeAgent { final NodeRepository nodeRepository, final Orchestrator orchestrator, final DockerOperations dockerOperations, - final Optional<StorageMaintainer> storageMaintainer, + final StorageMaintainer storageMaintainer, + final AclMaintainer aclMaintainer, final Environment environment, final Clock clock, - final Optional<AclMaintainer> aclMaintainer) { + final Duration timeBetweenEachConverge) { + this.containerName = ContainerName.fromHostname(hostName); + this.logger = PrefixLogger.getNodeAgentLogger(NodeAgentImpl.class, containerName); + this.hostname = hostName; this.nodeRepository = nodeRepository; this.orchestrator = orchestrator; - this.hostname = hostName; - this.containerName = ContainerName.fromHostname(hostName); this.dockerOperations = dockerOperations; this.storageMaintainer = storageMaintainer; - this.logger = PrefixLogger.getNodeAgentLogger(NodeAgentImpl.class, containerName); + this.aclMaintainer = aclMaintainer; this.environment = environment; this.clock = clock; - this.aclMaintainer = aclMaintainer; + this.timeBetweenEachConverge = timeBetweenEachConverge; this.lastConverge = clock.instant(); - this.serviceRestarter = service -> { - try { - ProcessResult processResult = dockerOperations.executeCommandInContainerAsRoot( - containerName, "service", service, "restart"); - - if (!processResult.isSuccess()) { - logger.error("Failed to restart service " + service + ": " + processResult); - } - } catch (Exception e) { - logger.error("Failed to restart service " + service, e); - } - }; } @Override @@ -183,11 +173,11 @@ public class NodeAgentImpl implements NodeAgent { } @Override - public void start(int intervalMillis) { - String message = "Starting with interval " + intervalMillis + " ms"; + public void start() { + String message = "Starting with interval " + timeBetweenEachConverge.toMillis() + " ms"; logger.info(message); addDebugMessage(message); - delaysBetweenEachConvergeMillis = intervalMillis; + if (loopThread != null) { throw new RuntimeException("Can not restart a node agent."); } @@ -197,6 +187,19 @@ public class NodeAgentImpl implements NodeAgent { }); loopThread.setName("tick-" + hostname); loopThread.start(); + + serviceRestarter = service -> { + try { + ProcessResult processResult = dockerOperations.executeCommandInContainerAsRoot( + containerName, "service", service, "restart"); + + if (!processResult.isSuccess()) { + logger.error("Failed to restart service " + service + ": " + processResult); + } + } catch (Exception e) { + logger.error("Failed to restart service " + service, e); + } + }; } @Override @@ -207,19 +210,15 @@ public class NodeAgentImpl implements NodeAgent { throw new RuntimeException("Can not re-stop a node agent."); } signalWorkToBeDone(); - try { - loopThread.join(10000); - if (loopThread.isAlive()) { - logger.error("Could not stop host thread " + hostname); + + do { + try { + loopThread.join(); + filebeatRestarter.awaitTermination(Long.MAX_VALUE, TimeUnit.NANOSECONDS); + } catch (InterruptedException e) { + logger.error("Interrupted while waiting for converge thread and filebeatRestarter scheduler to shutdown"); } - } catch (InterruptedException e1) { - logger.error("Interrupted; Could not stop host thread " + hostname); - } - try { - filebeatRestarter.awaitTermination(10, TimeUnit.SECONDS); - } catch (InterruptedException e) { - logger.error("Interrupted; Could not stop filebeatrestarter thread"); - } + } while (loopThread.isAlive() || !filebeatRestarter.isTerminated()); logger.info("Stopped"); } @@ -257,15 +256,13 @@ public class NodeAgentImpl implements NodeAgent { } private void startContainer(ContainerNodeSpec nodeSpec) { - aclMaintainer.ifPresent(AclMaintainer::run); + aclMaintainer.run(); dockerOperations.startContainer(containerName, nodeSpec); lastCpuMetric = new CpuUsageReporter(); currentFilebeatRestarter = filebeatRestarter.scheduleWithFixedDelay(() -> serviceRestarter.accept("filebeat"), 1, 1, TimeUnit.DAYS); - storageMaintainer.ifPresent(maintainer -> { - maintainer.writeMetricsConfig(containerName, nodeSpec); - maintainer.writeFilebeatConfig(containerName, nodeSpec); - }); + storageMaintainer.writeMetricsConfig(containerName, nodeSpec); + storageMaintainer.writeFilebeatConfig(containerName, nodeSpec); resumeScriptRun = false; containerState = UNKNOWN; @@ -377,7 +374,7 @@ public class NodeAgentImpl implements NodeAgent { boolean isFrozenCopy; synchronized (monitor) { while (!workToDoNow) { - long remainder = delaysBetweenEachConvergeMillis - Duration.between(lastConverge, clock.instant()).toMillis(); + long remainder = timeBetweenEachConverge.minus(Duration.between(lastConverge, clock.instant())).toMillis(); if (remainder > 0) { try { monitor.wait(remainder); @@ -439,9 +436,7 @@ public class NodeAgentImpl implements NodeAgent { // will change and we will be reporting duplicate metrics. // TODO: Should be retried if writing fails if (container.isPresent()) { - storageMaintainer.ifPresent(maintainer -> { - maintainer.writeMetricsConfig(containerName, nodeSpec); - }); + storageMaintainer.writeMetricsConfig(containerName, nodeSpec); } } @@ -454,14 +449,13 @@ public class NodeAgentImpl implements NodeAgent { updateNodeRepoWithCurrentAttributes(nodeSpec); break; case active: - storageMaintainer.ifPresent(maintainer -> { - maintainer.handleCoreDumpsForContainer(containerName, nodeSpec, false); + storageMaintainer.handleCoreDumpsForContainer(containerName, nodeSpec, false); + + storageMaintainer.getDiskUsageFor(containerName) + .map(diskUsage -> (double) diskUsage / BYTES_IN_GB / nodeSpec.minDiskAvailableGb) + .filter(diskUtil -> diskUtil >= 0.8) + .ifPresent(diskUtil -> storageMaintainer.removeOldFilesFromNode(containerName)); - maintainer.getDiskUsageFor(containerName) - .map(diskUsage -> (double) diskUsage / BYTES_IN_GB / nodeSpec.minDiskAvailableGb) - .filter(diskUtil -> diskUtil >= 0.8) - .ifPresent(diskUtil -> maintainer.removeOldFilesFromNode(containerName)); - }); scheduleDownLoadIfNeeded(nodeSpec); if (isDownloadingImage()) { addDebugMessage("Waiting for image to download " + imageBeingDownloaded.asString()); @@ -469,7 +463,7 @@ public class NodeAgentImpl implements NodeAgent { } container = removeContainerIfNeededUpdateContainerState(nodeSpec, container); if (! container.isPresent()) { - storageMaintainer.ifPresent(maintainer -> maintainer.handleCoreDumpsForContainer(containerName, nodeSpec, false)); + storageMaintainer.handleCoreDumpsForContainer(containerName, nodeSpec, false); startContainer(nodeSpec); } @@ -498,7 +492,7 @@ public class NodeAgentImpl implements NodeAgent { case dirty: removeContainerIfNeededUpdateContainerState(nodeSpec, container); logger.info("State is " + nodeSpec.nodeState + ", will delete application storage and mark node as ready"); - storageMaintainer.ifPresent(maintainer -> maintainer.cleanupNodeStorage(containerName, nodeSpec)); + storageMaintainer.cleanupNodeStorage(containerName, nodeSpec); updateNodeRepoWithCurrentAttributes(nodeSpec); nodeRepository.markNodeAvailableForNewAllocation(hostname); break; @@ -531,8 +525,7 @@ public class NodeAgentImpl implements NodeAgent { final long memoryTotalBytesUsage = ((Number) stats.getMemoryStats().get("usage")).longValue(); final long memoryTotalBytesCache = ((Number) ((Map) stats.getMemoryStats().get("stats")).get("cache")).longValue(); final long diskTotalBytes = (long) (nodeSpec.minDiskAvailableGb * BYTES_IN_GB); - final Optional<Long> diskTotalBytesUsed = storageMaintainer.flatMap(maintainer -> maintainer - .getDiskUsageFor(containerName)); + final Optional<Long> diskTotalBytesUsed = storageMaintainer.getDiskUsageFor(containerName); // CPU usage by a container as percentage of total host CPU, cpuPercentageOfHost, is given by dividing used // CPU time by the container with CPU time used by the entire system. diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/noderepository/NodeRepository.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/noderepository/NodeRepository.java index d14cd2f1330..9f4c6916b48 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/noderepository/NodeRepository.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/noderepository/NodeRepository.java @@ -13,7 +13,7 @@ import java.util.Optional; * @author stiankri */ public interface NodeRepository { - List<ContainerNodeSpec> getContainersToRun() throws IOException; + List<ContainerNodeSpec> getContainersToRun(String baseHostName) throws IOException; Optional<ContainerNodeSpec> getContainerNodeSpec(String hostName); diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/noderepository/NodeRepositoryImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/noderepository/NodeRepositoryImpl.java index 7d73d05ca36..08957a489b6 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/noderepository/NodeRepositoryImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/noderepository/NodeRepositoryImpl.java @@ -28,18 +28,17 @@ import java.util.stream.Collectors; */ public class NodeRepositoryImpl implements NodeRepository { private static final PrefixLogger NODE_ADMIN_LOGGER = PrefixLogger.getNodeAdminLogger(NodeRepositoryImpl.class); - private final String baseHostName; - private final int port; + private final ConfigServerHttpRequestExecutor requestExecutor; + private final int port; - public NodeRepositoryImpl(ConfigServerHttpRequestExecutor requestExecutor, int configPort, String baseHostName) { - this.baseHostName = baseHostName; - this.port = configPort; + public NodeRepositoryImpl(ConfigServerHttpRequestExecutor requestExecutor, int port) { this.requestExecutor = requestExecutor; + this.port = port; } @Override - public List<ContainerNodeSpec> getContainersToRun() throws IOException { + public List<ContainerNodeSpec> getContainersToRun(String baseHostName) throws IOException { try { final GetNodesResponse nodesForHost = requestExecutor.get( "/nodes/v2/node/?parentHost=" + baseHostName + "&recursive=true", diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/orchestrator/OrchestratorImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/orchestrator/OrchestratorImpl.java index 5117a1bb079..bd9df486e7b 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/orchestrator/OrchestratorImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/orchestrator/OrchestratorImpl.java @@ -1,8 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.node.admin.orchestrator; -import com.yahoo.vespa.defaults.Defaults; - import com.yahoo.vespa.hosted.node.admin.util.ConfigServerHttpRequestExecutor; import com.yahoo.vespa.orchestrator.restapi.HostApi; @@ -19,7 +17,6 @@ import java.util.Optional; * @author dybis */ public class OrchestratorImpl implements Orchestrator { - static final int WEB_SERVICE_PORT = Defaults.getDefaults().vespaWebServicePort(); // TODO: Find a way to avoid duplicating this (present in orchestrator's services.xml also). private static final String ORCHESTRATOR_PATH_PREFIX = "/orchestrator"; static final String ORCHESTRATOR_PATH_PREFIX_HOST_API @@ -28,9 +25,11 @@ public class OrchestratorImpl implements Orchestrator { = ORCHESTRATOR_PATH_PREFIX + HostSuspensionApi.PATH_PREFIX; private final ConfigServerHttpRequestExecutor requestExecutor; + private final int port; - public OrchestratorImpl(ConfigServerHttpRequestExecutor requestExecutor) { + public OrchestratorImpl(ConfigServerHttpRequestExecutor requestExecutor, int port) { this.requestExecutor = requestExecutor; + this.port = port; } @Override @@ -38,7 +37,7 @@ public class OrchestratorImpl implements Orchestrator { UpdateHostResponse response; try { response = requestExecutor.put(getSuspendPath(hostName), - WEB_SERVICE_PORT, + port, Optional.empty(), /* body */ UpdateHostResponse.class); } catch (ConfigServerHttpRequestExecutor.NotFoundException n) { @@ -58,7 +57,7 @@ public class OrchestratorImpl implements Orchestrator { try { batchOperationResult = requestExecutor.put( ORCHESTRATOR_PATH_PREFIX_HOST_SUSPENSION_API, - WEB_SERVICE_PORT, + port, Optional.of(new BatchHostSuspendRequest(parentHostName, hostNames)), BatchOperationResult.class); } catch (Exception e) { @@ -75,7 +74,7 @@ public class OrchestratorImpl implements Orchestrator { UpdateHostResponse response; try { String path = getSuspendPath(hostName); - response = requestExecutor.delete(path, WEB_SERVICE_PORT, UpdateHostResponse.class); + response = requestExecutor.delete(path, port, UpdateHostResponse.class); } catch (ConfigServerHttpRequestExecutor.NotFoundException n) { throw new OrchestratorNotFoundException("Failed to resume " + hostName + ", host not found"); } catch (Exception e) { diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/provider/ComponentsProvider.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/provider/ComponentsProvider.java deleted file mode 100644 index 3211e4feb56..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/provider/ComponentsProvider.java +++ /dev/null @@ -1,16 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.provider; - -import com.yahoo.vespa.hosted.dockerapi.metrics.MetricReceiverWrapper; -import com.yahoo.vespa.hosted.node.admin.nodeadmin.NodeAdminStateUpdater; - -/** - * Class for setting up instances of classes; enables testing. - * - * @author dybis - */ -public interface ComponentsProvider { - NodeAdminStateUpdater getNodeAdminStateUpdater(); - - MetricReceiverWrapper getMetricReceiverWrapper(); -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/provider/ComponentsProviderImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/provider/ComponentsProviderImpl.java deleted file mode 100644 index d61dd3c591b..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/provider/ComponentsProviderImpl.java +++ /dev/null @@ -1,126 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.provider; - -import com.google.inject.Inject; -import com.yahoo.net.HostName; -import static com.yahoo.vespa.defaults.Defaults.getDefaults; - -import com.yahoo.system.ProcessExecuter; -import com.yahoo.vespa.hosted.dockerapi.ContainerName; -import com.yahoo.vespa.hosted.dockerapi.Docker; -import com.yahoo.vespa.hosted.dockerapi.metrics.MetricReceiverWrapper; -import com.yahoo.vespa.hosted.node.admin.docker.DockerOperations; -import com.yahoo.vespa.hosted.node.admin.maintenance.acl.AclMaintainer; -import com.yahoo.vespa.hosted.node.admin.maintenance.StorageMaintainer; -import com.yahoo.vespa.hosted.node.admin.nodeadmin.NodeAdmin; -import com.yahoo.vespa.hosted.node.admin.nodeadmin.NodeAdminImpl; -import com.yahoo.vespa.hosted.node.admin.nodeadmin.NodeAdminStateUpdater; -import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgent; -import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentImpl; -import com.yahoo.vespa.hosted.node.admin.docker.DockerOperationsImpl; -import com.yahoo.vespa.hosted.node.admin.noderepository.NodeRepository; -import com.yahoo.vespa.hosted.node.admin.noderepository.NodeRepositoryImpl; -import com.yahoo.vespa.hosted.node.admin.orchestrator.Orchestrator; -import com.yahoo.vespa.hosted.node.admin.orchestrator.OrchestratorImpl; -import com.yahoo.vespa.hosted.node.admin.util.ConfigServerHttpRequestExecutor; -import com.yahoo.vespa.hosted.node.admin.util.Environment; -import com.yahoo.vespa.hosted.node.admin.util.SecretAgentScheduleMaker; - -import java.io.IOException; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.time.Clock; -import java.util.Optional; -import java.util.Set; -import java.util.function.Function; - -/** - * Set up node admin for production. - * - * @author dybis - */ -public class ComponentsProviderImpl implements ComponentsProvider { - private static final ContainerName NODE_ADMIN_CONTAINER_NAME = new ContainerName("node-admin"); - - private final NodeAdminStateUpdater nodeAdminStateUpdater; - private final MetricReceiverWrapper metricReceiverWrapper; - - private static final int NODE_AGENT_SCAN_INTERVAL_MILLIS = 30000; - private static final int WEB_SERVICE_PORT = getDefaults().vespaWebServicePort(); - - // Converge towards desired node admin state every 30 seconds - private static final int NODE_ADMIN_CONVERGE_STATE_INTERVAL_MILLIS = 30000; - - public ComponentsProviderImpl(Docker docker, MetricReceiverWrapper metricReceiver, Environment environment) { - String baseHostName = HostName.getLocalhost(); - Set<String> configServerHosts = environment.getConfigServerHosts(); - if (configServerHosts.isEmpty()) { - throw new IllegalStateException("Environment setting for config servers missing or empty."); - } - - Clock clock = Clock.systemUTC(); - ProcessExecuter processExecuter = new ProcessExecuter(); - ConfigServerHttpRequestExecutor requestExecutor = ConfigServerHttpRequestExecutor.create(configServerHosts); - Orchestrator orchestrator = new OrchestratorImpl(requestExecutor); - NodeRepository nodeRepository = new NodeRepositoryImpl(requestExecutor, WEB_SERVICE_PORT, baseHostName); - DockerOperations dockerOperations = new DockerOperationsImpl(docker, environment, processExecuter); - - Optional<StorageMaintainer> storageMaintainer = environment.isRunningLocally() ? - Optional.empty() : Optional.of(new StorageMaintainer(docker, processExecuter, metricReceiver, environment, clock)); - Optional<AclMaintainer> aclMaintainer = environment.isRunningLocally() ? - Optional.empty() : Optional.of(new AclMaintainer(dockerOperations, nodeRepository, baseHostName)); - - Function<String, NodeAgent> nodeAgentFactory = - (hostName) -> new NodeAgentImpl(hostName, nodeRepository, orchestrator, dockerOperations, - storageMaintainer, environment, clock, aclMaintainer); - NodeAdmin nodeAdmin = new NodeAdminImpl(dockerOperations, nodeAgentFactory, storageMaintainer, - NODE_AGENT_SCAN_INTERVAL_MILLIS, metricReceiver, aclMaintainer, clock); - nodeAdminStateUpdater = new NodeAdminStateUpdater(nodeRepository, nodeAdmin, storageMaintainer, clock, orchestrator, baseHostName); - nodeAdminStateUpdater.start(NODE_ADMIN_CONVERGE_STATE_INTERVAL_MILLIS); - - metricReceiverWrapper = metricReceiver; - - if (! environment.isRunningLocally()) { - setCorePattern(docker); - initializeNodeAgentSecretAgent(docker); - } - } - - @Inject - public ComponentsProviderImpl(final Docker docker, final MetricReceiverWrapper metricReceiver) { - this(docker, metricReceiver, new Environment()); - } - - @Override - public NodeAdminStateUpdater getNodeAdminStateUpdater() { - return nodeAdminStateUpdater; - } - - @Override - public MetricReceiverWrapper getMetricReceiverWrapper() { - return metricReceiverWrapper; - } - - - private void setCorePattern(Docker docker) { - final String[] sysctlCorePattern = {"sysctl", "-w", "kernel.core_pattern=" + - getDefaults().underVespaHome("var/crash/%e.core.%p")}; - docker.executeInContainerAsRoot(NODE_ADMIN_CONTAINER_NAME, sysctlCorePattern); - } - - private void initializeNodeAgentSecretAgent(Docker docker) { - final Path yamasAgentFolder = Paths.get("/etc/yamas-agent/"); - docker.executeInContainerAsRoot(NODE_ADMIN_CONTAINER_NAME, "chmod", "a+w", yamasAgentFolder.toString()); - - Path nodeAdminCheckPath = Paths.get("/usr/bin/curl"); - SecretAgentScheduleMaker scheduleMaker = new SecretAgentScheduleMaker("node-admin", 60, nodeAdminCheckPath, - "localhost:4080/rest/metrics"); - - try { - scheduleMaker.writeTo(yamasAgentFolder); - docker.executeInContainerAsRoot(NODE_ADMIN_CONTAINER_NAME, "service", "yamas-agent", "restart"); - } catch (IOException e) { - throw new RuntimeException("Failed to write secret-agent schedules for node-admin", e); - } - } -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/provider/NodeAdminProvider.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/provider/NodeAdminProvider.java new file mode 100644 index 00000000000..5536ee1551b --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/provider/NodeAdminProvider.java @@ -0,0 +1,105 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.provider; + +import com.google.inject.Inject; +import com.yahoo.concurrent.lock.Lock; +import com.yahoo.concurrent.lock.Locking; +import com.yahoo.container.di.componentgraph.Provider; +import com.yahoo.log.LogLevel; +import com.yahoo.net.HostName; + +import com.yahoo.system.ProcessExecuter; +import com.yahoo.vespa.hosted.dockerapi.Docker; +import com.yahoo.vespa.hosted.dockerapi.metrics.MetricReceiverWrapper; +import com.yahoo.vespa.hosted.node.admin.docker.DockerOperations; +import com.yahoo.vespa.hosted.node.admin.maintenance.acl.AclMaintainer; +import com.yahoo.vespa.hosted.node.admin.maintenance.StorageMaintainer; +import com.yahoo.vespa.hosted.node.admin.nodeadmin.NodeAdmin; +import com.yahoo.vespa.hosted.node.admin.nodeadmin.NodeAdminImpl; +import com.yahoo.vespa.hosted.node.admin.nodeadmin.NodeAdminStateUpdater; +import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgent; +import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentImpl; +import com.yahoo.vespa.hosted.node.admin.docker.DockerOperationsImpl; +import com.yahoo.vespa.hosted.node.admin.noderepository.NodeRepository; +import com.yahoo.vespa.hosted.node.admin.noderepository.NodeRepositoryImpl; +import com.yahoo.vespa.hosted.node.admin.orchestrator.Orchestrator; +import com.yahoo.vespa.hosted.node.admin.orchestrator.OrchestratorImpl; +import com.yahoo.vespa.hosted.node.admin.util.ConfigServerHttpRequestExecutor; +import com.yahoo.vespa.hosted.node.admin.util.Environment; + +import java.time.Clock; +import java.time.Duration; +import java.util.function.Function; +import java.util.logging.Logger; + +import static com.yahoo.vespa.defaults.Defaults.getDefaults; + +/** + * Set up node admin for production. + * + * @author dybis + */ +public class NodeAdminProvider implements Provider<NodeAdminStateUpdater> { + private static final int WEB_SERVICE_PORT = getDefaults().vespaWebServicePort(); + private static final Duration NODE_AGENT_SCAN_INTERVAL = Duration.ofSeconds(30); + private static final Duration NODE_ADMIN_CONVERGE_STATE_INTERVAL = Duration.ofSeconds(30); + + private final Logger log = Logger.getLogger(NodeAdminProvider.class.getName()); + private final NodeAdminStateUpdater nodeAdminStateUpdater; + private final Lock classLock; + + @Inject + public NodeAdminProvider(Docker docker, MetricReceiverWrapper metricReceiver, Locking locking) { + log.log(LogLevel.INFO, objectToString() + ": Creating object, acquiring lock..."); + classLock = locking.lock(this.getClass()); + try { + log.log(LogLevel.INFO, objectToString() + ": Lock acquired"); + + Clock clock = Clock.systemUTC(); + String dockerHostHostName = HostName.getLocalhost(); + ProcessExecuter processExecuter = new ProcessExecuter(); + Environment environment = new Environment(); + + ConfigServerHttpRequestExecutor requestExecutor = ConfigServerHttpRequestExecutor.create(environment.getConfigServerHosts()); + NodeRepository nodeRepository = new NodeRepositoryImpl(requestExecutor, WEB_SERVICE_PORT); + Orchestrator orchestrator = new OrchestratorImpl(requestExecutor, WEB_SERVICE_PORT); + DockerOperations dockerOperations = new DockerOperationsImpl(docker, environment, processExecuter); + + StorageMaintainer storageMaintainer = new StorageMaintainer(docker, processExecuter, metricReceiver, environment, clock); + AclMaintainer aclMaintainer = new AclMaintainer(dockerOperations, nodeRepository, dockerHostHostName); + + Function<String, NodeAgent> nodeAgentFactory = + (hostName) -> new NodeAgentImpl(hostName, nodeRepository, orchestrator, dockerOperations, + storageMaintainer, aclMaintainer, environment, clock, NODE_AGENT_SCAN_INTERVAL); + NodeAdmin nodeAdmin = new NodeAdminImpl(dockerOperations, nodeAgentFactory, storageMaintainer, aclMaintainer, + metricReceiver, clock); + + nodeAdminStateUpdater = new NodeAdminStateUpdater(nodeRepository, orchestrator, storageMaintainer, nodeAdmin, + dockerHostHostName, clock, NODE_ADMIN_CONVERGE_STATE_INTERVAL); + nodeAdminStateUpdater.start(); + } catch (Exception e) { + classLock.close(); + throw e; + } + } + + @Override + public NodeAdminStateUpdater get() { + return nodeAdminStateUpdater; + } + + @Override + public void deconstruct() { + log.log(LogLevel.INFO, objectToString() + ": Stop called"); + + nodeAdminStateUpdater.stop(); + log.log(LogLevel.INFO, objectToString() + ": Stop complete"); + + classLock.close(); + log.log(LogLevel.INFO, objectToString() + ": Lock released"); + } + + private String objectToString() { + return this.getClass().getSimpleName() + "@" + Integer.toString(System.identityHashCode(this)); + } +} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/restapi/RestApiHandler.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/restapi/RestApiHandler.java index ff6ac9ce1e7..adfb937b8d7 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/restapi/RestApiHandler.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/restapi/RestApiHandler.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.hosted.node.admin.restapi; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; +import com.yahoo.container.di.componentgraph.Provider; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.container.jdisc.LoggingRequestHandler; @@ -10,8 +11,8 @@ import com.yahoo.container.logging.AccessLog; import com.yahoo.vespa.hosted.dockerapi.metrics.DimensionMetrics; import com.yahoo.vespa.hosted.dockerapi.metrics.MetricReceiverWrapper; import com.yahoo.vespa.hosted.node.admin.nodeadmin.NodeAdminStateUpdater; -import com.yahoo.vespa.hosted.node.admin.provider.ComponentsProvider; +import javax.inject.Inject; import javax.ws.rs.core.MediaType; import java.io.IOException; import java.io.OutputStream; @@ -36,10 +37,13 @@ public class RestApiHandler extends LoggingRequestHandler{ private final NodeAdminStateUpdater refresher; private final MetricReceiverWrapper metricReceiverWrapper; - public RestApiHandler(Executor executor, AccessLog accessLog, ComponentsProvider componentsProvider) { + @Inject + public RestApiHandler(Executor executor, AccessLog accessLog, + Provider<NodeAdminStateUpdater> componentsProvider, + MetricReceiverWrapper metricReceiverWrapper) { super(executor, accessLog); - this.refresher = componentsProvider.getNodeAdminStateUpdater(); - this.metricReceiverWrapper = componentsProvider.getMetricReceiverWrapper(); + this.refresher = componentsProvider.get(); + this.metricReceiverWrapper = metricReceiverWrapper; } @Override diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/util/ConfigServerHttpRequestExecutor.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/util/ConfigServerHttpRequestExecutor.java index 4434213989f..9c8dc198388 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/util/ConfigServerHttpRequestExecutor.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/util/ConfigServerHttpRequestExecutor.java @@ -52,6 +52,10 @@ public class ConfigServerHttpRequestExecutor { } public static ConfigServerHttpRequestExecutor create(Set<String> configServerHosts) { + if (configServerHosts.isEmpty()) { + throw new IllegalStateException("Environment setting for config servers missing or empty."); + } + PoolingHttpClientConnectionManager cm = new PoolingHttpClientConnectionManager(); // Increase max total connections to 200, which should be enough cm.setMaxTotal(200); diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/util/Environment.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/util/Environment.java index 03433ba64a8..65a4d84b4c3 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/util/Environment.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/util/Environment.java @@ -15,8 +15,8 @@ import java.time.Instant; import java.util.Arrays; import java.util.Collections; import java.util.Date; +import java.util.HashSet; import java.util.List; -import java.util.Optional; import java.util.Set; import java.util.TimeZone; import java.util.stream.Collectors; @@ -35,7 +35,6 @@ public class Environment { private static final String ENVIRONMENT = "ENVIRONMENT"; private static final String REGION = "REGION"; private static final String LOGSTASH_NODES = "LOGSTASH_NODES"; - private static final String RUNNING_LOCALLY = "RUNNING_LOCALLY"; private static final String COREDUMP_FEED_ENDPOINT = "COREDUMP_FEED_ENDPOINT"; private final Set<String> configServerHosts; @@ -46,7 +45,6 @@ public class Environment { private final PathResolver pathResolver; private final List<String> logstashNodes; private final String feedEndpoint; - private final boolean isRunningLocally; static { filenameFormatter.setTimeZone(TimeZone.getTimeZone("UTC")); @@ -60,8 +58,7 @@ public class Environment { new InetAddressResolver(), new PathResolver(), getLogstashNodesFromEnvironment(), - getEnvironmentVariable(COREDUMP_FEED_ENDPOINT), - Optional.ofNullable(System.getenv(RUNNING_LOCALLY)).map(Boolean::valueOf).orElse(false)); + getEnvironmentVariable(COREDUMP_FEED_ENDPOINT)); } public Environment(Set<String> configServerHosts, @@ -71,8 +68,7 @@ public class Environment { InetAddressResolver inetAddressResolver, PathResolver pathResolver, List<String> logstashNodes, - String feedEndpoint, - boolean isRunningLocally) { + String feedEndpoint) { this.configServerHosts = configServerHosts; this.environment = environment; this.region = region; @@ -81,7 +77,6 @@ public class Environment { this.pathResolver = pathResolver; this.logstashNodes = logstashNodes; this.feedEndpoint = feedEndpoint; - this.isRunningLocally = isRunningLocally; } public Set<String> getConfigServerHosts() { return configServerHosts; } @@ -115,7 +110,7 @@ public class Environment { } final List<String> hostNameStrings = Arrays.asList(configServerHosts.split("[,\\s]+")); - return hostNameStrings.stream().collect(Collectors.toSet()); + return new HashSet<>(hostNameStrings); } private static List<String> getLogstashNodesFromEnvironment() { @@ -138,10 +133,6 @@ public class Environment { return feedEndpoint; } - public boolean isRunningLocally() { - return isRunningLocally; - } - /** * Absolute path in node admin to directory with processed and reported core dumps */ @@ -205,7 +196,6 @@ public class Environment { private PathResolver pathResolver; private List<String> logstashNodes = Collections.emptyList(); private String feedEndpoint; - private boolean isRunningLocally = false; public Builder configServerHosts(String... hosts) { configServerHosts = Arrays.stream(hosts).collect(Collectors.toSet()); @@ -247,14 +237,10 @@ public class Environment { return this; } - public Builder isRunningLocally(boolean isRunningLocally) { - this.isRunningLocally = isRunningLocally; - return this; - } public Environment build() { return new Environment(configServerHosts, environment, region, parentHostHostname, inetAddressResolver, - pathResolver, logstashNodes, feedEndpoint, isRunningLocally); + pathResolver, logstashNodes, feedEndpoint); } } } diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImplTest.java index c0f123d7044..43170c49a33 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImplTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImplTest.java @@ -9,7 +9,6 @@ import com.yahoo.vespa.hosted.dockerapi.Docker; import com.yahoo.vespa.hosted.dockerapi.DockerImage; import com.yahoo.vespa.hosted.dockerapi.ProcessResult; import com.yahoo.vespa.hosted.node.admin.util.Environment; -import org.hamcrest.CoreMatchers; import org.junit.Test; import org.mockito.InOrder; diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/docker/LocalZoneUtils.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/docker/LocalZoneUtils.java deleted file mode 100644 index aa1ffca2d46..00000000000 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/docker/LocalZoneUtils.java +++ /dev/null @@ -1,309 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.docker; - -import com.yahoo.net.HostName; -import static com.yahoo.vespa.defaults.Defaults.getDefaults; -import com.yahoo.vespa.hosted.dockerapi.Container; -import com.yahoo.vespa.hosted.dockerapi.ContainerName; -import com.yahoo.vespa.hosted.dockerapi.Docker; -import com.yahoo.vespa.hosted.dockerapi.DockerImage; -import com.yahoo.vespa.hosted.dockerapi.DockerImpl; -import com.yahoo.vespa.hosted.dockerapi.DockerTestUtils; -import com.yahoo.vespa.hosted.dockerapi.ProcessResult; -import com.yahoo.vespa.hosted.node.admin.ContainerNodeSpec; -import com.yahoo.vespa.hosted.node.admin.noderepository.NodeRepositoryImpl; -import com.yahoo.vespa.hosted.node.admin.noderepository.bindings.GetNodesResponse; -import com.yahoo.vespa.hosted.node.admin.util.ConfigServerHttpRequestExecutor; -import com.yahoo.vespa.hosted.node.admin.util.Environment; -import com.yahoo.vespa.hosted.provision.Node; - -import java.io.BufferedReader; -import java.io.IOException; -import java.io.InputStreamReader; -import java.net.HttpURLConnection; -import java.net.InetAddress; -import java.net.URL; -import java.net.UnknownHostException; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.time.Duration; -import java.time.Instant; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.Set; -import java.util.stream.Collectors; - -/** - * @author freva - */ -public class LocalZoneUtils { - public static final int CONFIG_SERVER_WEB_SERVICE_PORT = 4080; - public static final String CONFIG_SERVER_HOSTNAME = "config-server"; - public static final ContainerName CONFIG_SERVER_CONTAINER_NAME = new ContainerName(CONFIG_SERVER_HOSTNAME); - public static final String NODE_ADMIN_HOSTNAME = getParentHostHostname(); - public static final ContainerName NODE_ADMIN_CONTAINER_NAME = new ContainerName("node-admin"); - - private static final ConfigServerHttpRequestExecutor requestExecutor = ConfigServerHttpRequestExecutor.create( - Collections.singleton(CONFIG_SERVER_HOSTNAME)); - private static final String APP_HOSTNAME_PREFIX = "cnode-"; - private static final String TENANT_NAME = "localtenant"; - private static final String APPLICATION_NAME = "default"; - - public static void startConfigServerIfNeeded(Docker docker, Environment environment, DockerImage dockerImage, Path pathToProjectRoot) throws UnknownHostException { - Optional<Container> container = docker.getContainer(CONFIG_SERVER_CONTAINER_NAME); - if (container.isPresent()) { - if (container.get().state.isRunning()) return; - else docker.deleteContainer(CONFIG_SERVER_CONTAINER_NAME); - } - - Path pathToConfigServerApp = Paths.get(getDefaults().underVespaHome("conf/configserver-app")); - docker.createContainerCommand(dockerImage, CONFIG_SERVER_CONTAINER_NAME, CONFIG_SERVER_HOSTNAME) - .withNetworkMode(DockerImpl.DOCKER_CUSTOM_MACVLAN_NETWORK_NAME) - .withIpAddress(environment.getInetAddressForHost(CONFIG_SERVER_HOSTNAME)) - .withEnvironment("HOSTED_VESPA_ENVIRONMENT", environment.getEnvironment()) - .withEnvironment("HOSTED_VESPA_REGION", environment.getRegion()) - .withEnvironment("CONFIG_SERVER_HOSTNAME", CONFIG_SERVER_HOSTNAME) - .withUlimit("nofile", 262_144, 262_144) - .withUlimit("nproc", 32_768, 409_600) - .withUlimit("core", -1, -1) - .withEntrypoint(pathToConfigServerApp.resolve("start-config-server.sh").toString()) - .create(); - - docker.copyArchiveToContainer(pathToProjectRoot.resolve("node-admin/configserver-app").toString(), - CONFIG_SERVER_CONTAINER_NAME, getDefaults().underVespaHome("conf")); - - docker.startContainer(CONFIG_SERVER_CONTAINER_NAME); - } - - public static void startNodeAdminIfNeeded(Docker docker, Environment environment, DockerImage dockerImage, Path pathToContainerStorage) { - Optional<Docker.ContainerStats> containerStats = docker.getContainerStats(NODE_ADMIN_CONTAINER_NAME); - if (containerStats.isPresent()) - return; - else - docker.deleteContainer(NODE_ADMIN_CONTAINER_NAME); - - Docker.CreateContainerCommand createCmd = docker.createContainerCommand(dockerImage, - NODE_ADMIN_CONTAINER_NAME, NODE_ADMIN_HOSTNAME) - .withNetworkMode("host") - .withVolume("/proc", "/host/proc") - .withVolume("/var/run/docker.sock", "/host/var/run/docker.sock") - .withVolume(pathToContainerStorage.toString(), "/host" + pathToContainerStorage.toString()) - .withEnvironment("ENVIRONMENT", environment.getEnvironment()) - .withEnvironment("REGION", environment.getRegion()) - .withEnvironment("CONFIG_SERVER_ADDRESS", CONFIG_SERVER_HOSTNAME) - .withEnvironment("COREDUMP_FEED_ENDPOINT", "http://feed-endpoint.hostname.tld/feed") - .withEnvironment("RUNNING_LOCALLY", "true") - .withUlimit("nofile", 262_144, 262_144) - .withUlimit("nproc", 32_768, 409_600) - .withUlimit("core", -1, -1) - .withEntrypoint("/usr/local/bin/start-services.sh", "--run-local"); - - if (DockerTestUtils.getSystemOS() == DockerTestUtils.OS.Mac_OS_X) { - createCmd.withNetworkMode(DockerImpl.DOCKER_CUSTOM_MACVLAN_NETWORK_NAME); - } - - Arrays.asList( - "logs/daemontools_y", - "logs/jdisc_core", - "logs/langdetect/", - "logs/vespa", - "logs/yca", - "logs/yck", - "logs/yell", - "logs/ykeykey", - "logs/ykeykeyd", - "logs/yms_agent", - "logs/ysar", - "logs/ystatus", - "logs/zpe_policy_updater", - "var/cache", - "var/crash", - "var/db/jdisc", - "var/db/vespa", - "var/jdisc_container", - "var/jdisc_core", - "var/maven", - "var/run", - "var/scoreboards", - "var/service", - "var/share", - "var/spool", - "var/vespa", - "var/yca", - "var/ycore++", - "var/zookeeper") - .forEach(path -> { - String absPath = getDefaults().underVespaHome(path); - Path resolved = pathToContainerStorage.resolve("node-admin" + absPath); - createCmd.withVolume(resolved.toString(), absPath); - }); - - createCmd.create(); - docker.startContainer(NODE_ADMIN_CONTAINER_NAME); - docker.executeInContainerAsRoot(NODE_ADMIN_CONTAINER_NAME, "chown", getDefaults().vespaUser(), "/host/var/run/docker.sock"); - } - - public static Optional<ContainerNodeSpec> getContainerNodeSpec(String hostName) { - try { - GetNodesResponse.Node nodeResponse = requestExecutor.get("/nodes/v2/node/" + hostName, - CONFIG_SERVER_WEB_SERVICE_PORT, GetNodesResponse.Node.class); - if (nodeResponse == null) { - return Optional.empty(); - } - return Optional.of(NodeRepositoryImpl.createContainerNodeSpec(nodeResponse)); - } catch (ConfigServerHttpRequestExecutor.NotFoundException e) { - return Optional.empty(); - } - } - - public static void provisionHost(String hostname) { - List<Map<String, String>> nodesToAdd = new ArrayList<>(); - Map<String, String> provisionNodeRequest = new HashMap<>(); - provisionNodeRequest.put("type", "host"); - provisionNodeRequest.put("flavor", "docker"); - provisionNodeRequest.put("hostname", hostname); - provisionNodeRequest.put("openStackId", "fake-" + hostname); - nodesToAdd.add(provisionNodeRequest); - - try { - requestExecutor.post("/nodes/v2/node", CONFIG_SERVER_WEB_SERVICE_PORT, nodesToAdd, Map.class); - } catch (RuntimeException e) { - if (! e.getMessage().contains("A node with this name already exists")) throw e; - } - } - - /** - * Adds numberOfNodes to node-repo and returns a set of node hostnames. - */ - public static Set<String> provisionNodes(String parentHostname, int numberOfNodes) { - List<Map<String, Object>> nodesToAdd = new ArrayList<>(); - for (int i = 1; i <= numberOfNodes; i++) { - final String hostname = APP_HOSTNAME_PREFIX + i; - Map<String, Object> provisionNodeRequest = new HashMap<>(); - provisionNodeRequest.put("parentHostname", parentHostname); - provisionNodeRequest.put("type", "tenant"); - provisionNodeRequest.put("flavor", "docker"); - provisionNodeRequest.put("hostname", hostname); - provisionNodeRequest.put("ipAddresses", Collections.singletonList("172.18.2." + i)); - provisionNodeRequest.put("openStackId", "fake-" + hostname); - nodesToAdd.add(provisionNodeRequest); - } - - try { - requestExecutor.post("/nodes/v2/node", CONFIG_SERVER_WEB_SERVICE_PORT, nodesToAdd, Map.class); - } catch (RuntimeException e) { - if (! e.getMessage().contains("A node with this name already exists")) throw e; - } - return nodesToAdd.stream().map(i -> (String) i.get("hostname")).collect(Collectors.toSet()); - } - - public static void setState(Node.State state, String hostname) { - try { - requestExecutor.put("/nodes/v2/state/" + state + "/" + hostname, - CONFIG_SERVER_WEB_SERVICE_PORT, Optional.empty(), Map.class); - } catch (RuntimeException e) { - if (! e.getMessage().contains("Not registered as provisioned, dirty, failed or parked")) throw e; - } - } - - public static void deployApp(Docker docker, Path pathToApp) { - deployApp(docker, pathToApp, TENANT_NAME, APPLICATION_NAME); - } - - public static void deployApp(Docker docker, Path pathToApp, String tenantName, String applicationName) { - Path pathToAppOnConfigServer = Paths.get("/tmp").resolve(pathToApp.getFileName()); - docker.copyArchiveToContainer(pathToApp.toString(), CONFIG_SERVER_CONTAINER_NAME, pathToAppOnConfigServer.getParent().toString() + "/"); - - try { // Add tenant, ignore exception if tenant already exists - requestExecutor.put("/application/v2/tenant/" + tenantName, CONFIG_SERVER_WEB_SERVICE_PORT, Optional.empty(), Map.class); - } catch (RuntimeException e) { - if (! e.getMessage().contains("There already exists a tenant '" + tenantName)) { - throw e; - } - } - System.out.println("prepare " + applicationName); - final String deployPath = getDefaults().underVespaHome("bin/vespa-deploy"); - ProcessResult copyProcess = docker.executeInContainer(CONFIG_SERVER_CONTAINER_NAME, deployPath, "-e", - tenantName, "-a", applicationName, "prepare", pathToAppOnConfigServer.toString()); - if (! copyProcess.isSuccess()) { - throw new RuntimeException("Could not prepare " + pathToApp + " on " + CONFIG_SERVER_CONTAINER_NAME.asString() + - "\n" + copyProcess.getOutput() + "\n" + copyProcess.getErrors()); - } - - System.out.println("activate " + applicationName); - ProcessResult execProcess = docker.executeInContainer(CONFIG_SERVER_CONTAINER_NAME, deployPath, "-e", - tenantName, "-a", applicationName, "activate"); - if (! execProcess.isSuccess()) { - throw new RuntimeException("Could not activate application\n" + copyProcess.getOutput() + "\n" + copyProcess.getErrors()); - } - } - - public static Set<String> getContainersForApp() { - return getContainersForApp(TENANT_NAME, APPLICATION_NAME, "default"); - } - - @SuppressWarnings("unchecked") - public static Set<String> getContainersForApp(String tenant, String application, String instance) { - String app = String.join(".", tenant, application, instance); - Map<String, Object> response = requestExecutor.get("/nodes/v2/node/?recursive=true&clusterType=container&application=" + app, - CONFIG_SERVER_WEB_SERVICE_PORT, Map.class); - List<Map<String, Object>> nodes = (List<Map<String, Object>>) response.get("nodes"); - return nodes.stream().map(nodeMap -> (String) nodeMap.get("hostname")).collect(Collectors.toSet()); - } - - public static void packageApp(Path pathToApp) { - try { - Process process = Runtime.getRuntime().exec("mvn package", null, pathToApp.toFile()); - BufferedReader buff = new BufferedReader(new InputStreamReader(process.getInputStream())); - - String line; - while((line = buff.readLine()) != null) System.out.println(line); - - assert process.waitFor() == 0; - } catch (IOException | InterruptedException e) { - throw new RuntimeException("Failed to package application", e); - } - } - - public static void deleteApplication() { - deleteApplication(TENANT_NAME, APPLICATION_NAME); - } - - public static void deleteApplication(String tenantName, String appName) { - requestExecutor.delete("/application/v2/tenant/" + tenantName + "/application/" + appName, - CONFIG_SERVER_WEB_SERVICE_PORT, Map.class); - } - - public static boolean isReachableURL(URL url, Duration timeout) { - Instant start = Instant.now(); - while (Instant.now().minus(timeout).isBefore(start)) { - try { - Thread.sleep(100); - HttpURLConnection http = (HttpURLConnection) url.openConnection(); - if (http.getResponseCode() == 200) return true; - } catch (IOException | InterruptedException ignored) { } - } - - return false; - } - - private static String getParentHostHostname() { - String hostname = HostName.getLocalhost(); - // Avoid issue with hostname not being FQDN on MacOs - if (DockerTestUtils.getSystemOS() == DockerTestUtils.OS.Mac_OS_X) { - try { - InetAddress ipAddress = InetAddress.getByName(hostname); - return InetAddress.getByAddress(ipAddress.getAddress()).getHostName(); - } catch (UnknownHostException e) { - throw new RuntimeException("Could not find hostname"); - } - } - return hostname; - } -} - diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/docker/RunVespaLocal.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/docker/RunVespaLocal.java deleted file mode 100644 index e075b0f4d62..00000000000 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/docker/RunVespaLocal.java +++ /dev/null @@ -1,145 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.docker; - -import com.yahoo.net.HostName; -import com.yahoo.vespa.hosted.dockerapi.Docker; -import com.yahoo.vespa.hosted.dockerapi.DockerImage; -import com.yahoo.vespa.hosted.dockerapi.DockerTestUtils; -import com.yahoo.vespa.hosted.node.admin.util.Environment; -import com.yahoo.vespa.hosted.node.admin.util.InetAddressResolver; -import com.yahoo.vespa.hosted.provision.Node; - -import java.io.IOException; -import java.net.MalformedURLException; -import java.net.URL; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.StandardCopyOption; -import java.time.Duration; -import java.util.Set; -import java.util.logging.Logger; -import java.util.stream.Stream; - -import static org.junit.Assert.assertTrue; - -/** - * <pre> - * Requires docker daemon, see {@link com.yahoo.vespa.hosted.dockerapi.DockerTestUtils} for more details. - * - * Issues: - * 1. If you cannot make Docker Toolbox start, try starting Virtualbox and turn off the "default" machine - * 2. If the above is not enough try "sudo ifconfig vboxnet0 down && sudo ifconfig vboxnet0 up" (see https://github.com/docker/kitematic/issues/1193) - * </pre> - * - * @author freva - */ -public class RunVespaLocal { - private static final Environment environment = new Environment.Builder() - .configServerHosts(LocalZoneUtils.CONFIG_SERVER_HOSTNAME) - .environment("dev") - .region("vespa-local") - .parentHostHostname(HostName.getLocalhost()) - .inetAddressResolver(new InetAddressResolver()) - .build(); - - private final Logger logger = Logger.getLogger("RunVespaLocal"); - private final Docker docker; - private final Path pathToVespaRoot; - - public RunVespaLocal(Path pathToVespaRoot) { - this.docker = DockerTestUtils.getDocker(); - this.pathToVespaRoot = pathToVespaRoot; - } - - - /** - * Starts config server, provisions numNodesToProvision and puts them in ready state - */ - public void startLocalZoneWithNodes(DockerImage dockerImage, int numNodesToProvision) throws IOException { - logger.info("Starting config-server"); - LocalZoneUtils.startConfigServerIfNeeded(docker, environment, dockerImage, pathToVespaRoot); - - logger.info("Waiting until config-server is ready to serve"); - URL configServerUrl = new URL("http://" + LocalZoneUtils.CONFIG_SERVER_HOSTNAME + - ":" + LocalZoneUtils.CONFIG_SERVER_WEB_SERVICE_PORT + "/state/v1/health"); - assertTrue("Could not start config server", LocalZoneUtils.isReachableURL(configServerUrl, Duration.ofSeconds(120))); - - logger.info("Provisioning nodes"); - Set<String> hostnames = LocalZoneUtils.provisionNodes(LocalZoneUtils.NODE_ADMIN_HOSTNAME, numNodesToProvision); - hostnames.stream() - .map(LocalZoneUtils::getContainerNodeSpec) - .flatMap(optional -> optional.map(Stream::of).orElseGet(Stream::empty)) // Remove with JDK 9 - .forEach(nodeSpec -> { - if (nodeSpec.nodeState == Node.State.provisioned) LocalZoneUtils.setState(Node.State.dirty, nodeSpec.hostname); - if (nodeSpec.nodeState == Node.State.dirty) LocalZoneUtils.setState(Node.State.ready, nodeSpec.hostname); - }); - } - - /** - * Starts node-admin inside a container - * @param dockerImage Docker image that node-admin should be running - * @param pathToContainerStorage Path to where the container data will be stored, the path must exist and must - * be writeable by user, normally /home/docker/container-storage - */ - public void startNodeAdminAsContainer(DockerImage dockerImage, Path pathToContainerStorage) throws IOException { - logger.info("Starting node-admin"); - String parentHostHostname = LocalZoneUtils.NODE_ADMIN_HOSTNAME; - LocalZoneUtils.startNodeAdminIfNeeded(docker, environment, dockerImage, pathToContainerStorage); - - logger.info("Provisioning host at " + parentHostHostname); - LocalZoneUtils.provisionHost(parentHostHostname); - LocalZoneUtils.getContainerNodeSpec(parentHostHostname) - .ifPresent(nodeSpec -> { - if (nodeSpec.nodeState == Node.State.provisioned) { - LocalZoneUtils.setState(Node.State.dirty, nodeSpec.hostname); - LocalZoneUtils.setState(Node.State.ready, nodeSpec.hostname); - } - }); - - logger.info("Deploying node-admin app"); - Path pathToNodeAdminApp = pathToVespaRoot.resolve("node-admin/node-admin-zone-app"); - Path pathToNodeAdminAppComponents = pathToNodeAdminApp.resolve("components"); - Files.createDirectories(pathToNodeAdminAppComponents); - Path[] appComponents = {pathToVespaRoot.resolve("node-admin/target/node-admin-jar-with-dependencies.jar"), - pathToVespaRoot.resolve("docker-api/target/docker-api-jar-with-dependencies.jar")}; - - for (Path path : appComponents) { - Files.copy(path, pathToNodeAdminAppComponents.resolve(path.getFileName()), StandardCopyOption.REPLACE_EXISTING); - } - - LocalZoneUtils.deployApp(docker, pathToNodeAdminApp, "vespa", "node-admin"); - - logger.info("Waiting for node-admin to serve"); - try { - URL nodeUrl = new URL("http://localhost:" + System.getenv("VESPA_WEB_SERVICE_PORT") + "/"); - assertTrue(LocalZoneUtils.isReachableURL(nodeUrl, Duration.ofSeconds(120))); - } catch (MalformedURLException e) { - e.printStackTrace(); - } - } - - /** - * Packages, deploys an app and waits for the node to come up - * @param pathToApp Path to the directory of the application to deploy - */ - public void deployApplication(Path pathToApp) { - logger.info("Packaging application"); - LocalZoneUtils.packageApp(pathToApp); - logger.info("Deploying application"); - LocalZoneUtils.deployApp(docker, pathToApp.resolve("target/application.zip")); - - Set<String> containers = LocalZoneUtils.getContainersForApp(); - try { - URL nodeUrl = new URL("http://" + containers.iterator().next() + ":" + System.getenv("VESPA_WEB_SERVICE_PORT") + "/"); - assertTrue(LocalZoneUtils.isReachableURL(nodeUrl, Duration.ofMinutes(3))); - logger.info("Endpoint " + nodeUrl + " is now ready"); - } catch (MalformedURLException e) { - e.printStackTrace(); - } - } - - public void deleteApplication() { - logger.info("Deleting application"); - LocalZoneUtils.deleteApplication(); - } -} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerFailTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerFailTest.java index 60a8da80f2b..34506bb3143 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerFailTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerFailTest.java @@ -31,7 +31,7 @@ public class DockerFailTest { dockerTester.addContainerNodeSpec(containerNodeSpec); // Wait for node admin to be notified with node repo state and the docker container has been started - while (dockerTester.getNodeAdmin().getListOfHosts().size() == 0) { + while (dockerTester.nodeAdmin.getListOfHosts().size() == 0) { Thread.sleep(10); } diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerTester.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerTester.java index 714e8df64ec..ab752bbe4c0 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerTester.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerTester.java @@ -7,6 +7,7 @@ import com.yahoo.vespa.hosted.dockerapi.metrics.MetricReceiverWrapper; import com.yahoo.vespa.hosted.node.admin.ContainerNodeSpec; import com.yahoo.vespa.hosted.node.admin.docker.DockerOperations; import com.yahoo.vespa.hosted.node.admin.docker.DockerOperationsImpl; +import com.yahoo.vespa.hosted.node.admin.maintenance.acl.AclMaintainer; import com.yahoo.vespa.hosted.node.admin.nodeadmin.NodeAdmin; import com.yahoo.vespa.hosted.node.admin.nodeadmin.NodeAdminImpl; import com.yahoo.vespa.hosted.node.admin.nodeadmin.NodeAdminStateUpdater; @@ -20,7 +21,7 @@ import java.net.InetAddress; import java.net.UnknownHostException; import java.nio.file.Paths; import java.time.Clock; -import java.util.Optional; +import java.time.Duration; import java.util.function.Function; import static org.mockito.Matchers.any; @@ -30,17 +31,20 @@ import static org.mockito.Mockito.when; /** * @author musum */ -// Need to deconstruct updater +// Need to deconstruct nodeAdminStateUpdater public class DockerTester implements AutoCloseable { + private static final Duration NODE_AGENT_SCAN_INTERVAL = Duration.ofMillis(100); + private static final Duration NODE_ADMIN_CONVERGE_STATE_INTERVAL = Duration.ofMillis(10); + final CallOrderVerifier callOrderVerifier = new CallOrderVerifier(); final Docker dockerMock = new DockerMock(callOrderVerifier); final NodeRepoMock nodeRepositoryMock = new NodeRepoMock(callOrderVerifier); - final OrchestratorMock orchestratorMock = new OrchestratorMock(callOrderVerifier); - private final NodeAdminStateUpdater updater; - private final NodeAdmin nodeAdmin; + final NodeAdminStateUpdater nodeAdminStateUpdater; + final NodeAdmin nodeAdmin; + private final OrchestratorMock orchestratorMock = new OrchestratorMock(callOrderVerifier); - public DockerTester() { + DockerTester() { InetAddressResolver inetAddressResolver = mock(InetAddressResolver.class); try { when(inetAddressResolver.getInetAddressForHost(any(String.class))).thenReturn(InetAddress.getByName("1.1.1.1")); @@ -53,32 +57,25 @@ public class DockerTester implements AutoCloseable { .pathResolver(new PathResolver(Paths.get("/tmp"), Paths.get("/tmp"))).build(); Clock clock = Clock.systemUTC(); StorageMaintainerMock storageMaintainer = new StorageMaintainerMock(dockerMock, null, environment, callOrderVerifier, clock); + AclMaintainer aclMaintainer = mock(AclMaintainer.class); MetricReceiverWrapper mr = new MetricReceiverWrapper(MetricReceiver.nullImplementation); - final DockerOperations dockerOperations = new DockerOperationsImpl(dockerMock, environment, null); + DockerOperations dockerOperations = new DockerOperationsImpl(dockerMock, environment, null); Function<String, NodeAgent> nodeAgentFactory = (hostName) -> new NodeAgentImpl(hostName, nodeRepositoryMock, - orchestratorMock, dockerOperations, Optional.of(storageMaintainer), environment, clock, Optional.empty()); - nodeAdmin = new NodeAdminImpl(dockerOperations, nodeAgentFactory, Optional.of(storageMaintainer), 100, mr, Optional.empty(), Clock.systemUTC()); - updater = new NodeAdminStateUpdater(nodeRepositoryMock, nodeAdmin, Optional.of(storageMaintainer), - clock, orchestratorMock, "basehostname"); - updater.start(5); + orchestratorMock, dockerOperations, storageMaintainer, aclMaintainer, environment, clock, NODE_AGENT_SCAN_INTERVAL); + nodeAdmin = new NodeAdminImpl(dockerOperations, nodeAgentFactory, storageMaintainer, aclMaintainer, mr, Clock.systemUTC()); + nodeAdminStateUpdater = new NodeAdminStateUpdater(nodeRepositoryMock, orchestratorMock, storageMaintainer, + nodeAdmin, "basehostname", clock, NODE_ADMIN_CONVERGE_STATE_INTERVAL); + nodeAdminStateUpdater.start(); } - public void addContainerNodeSpec(ContainerNodeSpec containerNodeSpec) { + void addContainerNodeSpec(ContainerNodeSpec containerNodeSpec) { nodeRepositoryMock.updateContainerNodeSpec(containerNodeSpec); } - public NodeAdmin getNodeAdmin() { - return nodeAdmin; - } - - public NodeAdminStateUpdater getNodeAdminStateUpdater() { - return updater; - } - @Override public void close() { - updater.deconstruct(); + nodeAdminStateUpdater.stop(); } } diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/MultiDockerTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/MultiDockerTest.java index a6bc8a9b4e2..75a0727cb04 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/MultiDockerTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/MultiDockerTest.java @@ -82,7 +82,7 @@ public class MultiDockerTest { tester.addContainerNodeSpec(containerNodeSpec); // Wait for node admin to be notified with node repo state and the docker container has been started - while (tester.getNodeAdmin().getListOfHosts().size() != tester.nodeRepositoryMock.getNumberOfContainerSpecs()) { + while (tester.nodeAdmin.getListOfHosts().size() != tester.nodeRepositoryMock.getNumberOfContainerSpecs()) { Thread.sleep(10); } diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/ComponentsProviderWithMocks.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/NodeAdminProviderWithMocks.java index 166a2d3f70e..674ee6b7d17 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/ComponentsProviderWithMocks.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/NodeAdminProviderWithMocks.java @@ -1,9 +1,12 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.node.admin.integrationTests; +import com.yahoo.container.di.componentgraph.Provider; import com.yahoo.metrics.simple.MetricReceiver; import com.yahoo.vespa.hosted.dockerapi.metrics.MetricReceiverWrapper; import com.yahoo.vespa.hosted.node.admin.docker.DockerOperations; +import com.yahoo.vespa.hosted.node.admin.maintenance.StorageMaintainer; +import com.yahoo.vespa.hosted.node.admin.maintenance.acl.AclMaintainer; import com.yahoo.vespa.hosted.node.admin.nodeadmin.NodeAdmin; import com.yahoo.vespa.hosted.node.admin.nodeadmin.NodeAdminImpl; import com.yahoo.vespa.hosted.node.admin.nodeadmin.NodeAdminStateUpdater; @@ -11,11 +14,10 @@ import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgent; import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentImpl; import com.yahoo.vespa.hosted.node.admin.noderepository.NodeRepository; import com.yahoo.vespa.hosted.node.admin.orchestrator.Orchestrator; -import com.yahoo.vespa.hosted.node.admin.provider.ComponentsProvider; import com.yahoo.vespa.hosted.node.admin.util.Environment; import java.time.Clock; -import java.util.Optional; +import java.time.Duration; import java.util.function.Function; import static org.mockito.Mockito.mock; @@ -25,30 +27,36 @@ import static org.mockito.Mockito.mock; * * @author dybis */ -public class ComponentsProviderWithMocks implements ComponentsProvider { +public class NodeAdminProviderWithMocks implements Provider<NodeAdminStateUpdater> { + private static final Duration NODE_AGENT_SCAN_INTERVAL = Duration.ofMillis(100); + private static final Duration NODE_ADMIN_CONVERGE_STATE_INTERVAL = Duration.ofMillis(5); + static final NodeRepository nodeRepositoryMock = mock(NodeRepository.class); static final Orchestrator orchestratorMock = mock(Orchestrator.class); static final DockerOperations dockerOperationsMock = mock(DockerOperations.class); + private final StorageMaintainer storageMaintainer = mock(StorageMaintainer.class); + private final AclMaintainer aclMaintainer = mock(AclMaintainer.class); private final Environment environment = new Environment.Builder().build(); private final MetricReceiverWrapper mr = new MetricReceiverWrapper(MetricReceiver.nullImplementation); private final Function<String, NodeAgent> nodeAgentFactory = - (hostName) -> new NodeAgentImpl(hostName, nodeRepositoryMock, orchestratorMock, - dockerOperationsMock, Optional.empty(), environment, Clock.systemUTC(), Optional.empty()); - private final NodeAdmin nodeAdmin = new NodeAdminImpl(dockerOperationsMock, nodeAgentFactory, Optional.empty(), 100, mr, Optional.empty(), Clock.systemUTC()); - private final NodeAdminStateUpdater nodeAdminStateUpdater = new NodeAdminStateUpdater(nodeRepositoryMock, nodeAdmin, Optional.empty(), Clock.systemUTC(), orchestratorMock, "localhost.test.yahoo.com"); - - public ComponentsProviderWithMocks() { - nodeAdminStateUpdater.start(10); + (hostName) -> new NodeAgentImpl(hostName, nodeRepositoryMock, orchestratorMock, dockerOperationsMock, + storageMaintainer, aclMaintainer, environment, Clock.systemUTC(), NODE_AGENT_SCAN_INTERVAL); + private final NodeAdmin nodeAdmin = new NodeAdminImpl(dockerOperationsMock, nodeAgentFactory, storageMaintainer, aclMaintainer, mr, Clock.systemUTC()); + private final NodeAdminStateUpdater nodeAdminStateUpdater = new NodeAdminStateUpdater(nodeRepositoryMock, + orchestratorMock, storageMaintainer, nodeAdmin, "localhost.test.yahoo.com", Clock.systemUTC(), NODE_ADMIN_CONVERGE_STATE_INTERVAL); + + public NodeAdminProviderWithMocks() { + nodeAdminStateUpdater.start(); } @Override - public NodeAdminStateUpdater getNodeAdminStateUpdater() { + public NodeAdminStateUpdater get() { return nodeAdminStateUpdater; } @Override - public MetricReceiverWrapper getMetricReceiverWrapper() { - return null; + public void deconstruct() { + nodeAdminStateUpdater.stop(); } } diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/NodeRepoMock.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/NodeRepoMock.java index 1442226a4c9..2d2a622f8f0 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/NodeRepoMock.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/NodeRepoMock.java @@ -32,7 +32,7 @@ public class NodeRepoMock implements NodeRepository { } @Override - public List<ContainerNodeSpec> getContainersToRun() throws IOException { + public List<ContainerNodeSpec> getContainersToRun(String dockerHostHostname) throws IOException { synchronized (monitor) { return new ArrayList<>(containerNodeSpecsByHostname.values()); } diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/NodeStateTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/NodeStateTest.java index 26194ace7cf..669556f07ba 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/NodeStateTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/NodeStateTest.java @@ -32,7 +32,7 @@ public class NodeStateTest { tester.addContainerNodeSpec(initialContainerNodeSpec); // Wait for node admin to be notified with node repo state and the docker container has been started - while (tester.getNodeAdmin().getListOfHosts().size() == 0) { + while (tester.nodeAdmin.getListOfHosts().size() == 0) { Thread.sleep(10); } diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/RebootTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/RebootTest.java index b7561180264..0b5d67dd13c 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/RebootTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/RebootTest.java @@ -32,7 +32,7 @@ public class RebootTest { dockerTester.addContainerNodeSpec(createContainerNodeSpec()); // Wait for node admin to be notified with node repo state and the docker container has been started - while (dockerTester.getNodeAdmin().getListOfHosts().size() == 0) { + while (dockerTester.nodeAdmin.getListOfHosts().size() == 0) { Thread.sleep(10); } @@ -41,13 +41,13 @@ public class RebootTest { "createContainerCommand with DockerImage { imageId=dockerImage }, HostName: host1.test.yahoo.com, ContainerName { name=host1 }", "updateNodeAttributes with HostName: host1.test.yahoo.com, NodeAttributes{restartGeneration=1, rebootGeneration=null, dockerImage=dockerImage, vespaVersion='null'}"); - NodeAdminStateUpdater updater = dockerTester.getNodeAdminStateUpdater(); + NodeAdminStateUpdater updater = dockerTester.nodeAdminStateUpdater; assertThat(updater.setResumeStateAndCheckIfResumed(NodeAdminStateUpdater.State.SUSPENDED), is(Optional.of("Not all node agents are frozen."))); updater.setResumeStateAndCheckIfResumed(NodeAdminStateUpdater.State.SUSPENDED); - NodeAdmin nodeAdmin = dockerTester.getNodeAdmin(); + NodeAdmin nodeAdmin = dockerTester.nodeAdmin; // Wait for node admin to be frozen while ( ! nodeAdmin.isFrozen()) { System.out.println("Node admin not frozen yet"); diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/RestartTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/RestartTest.java index 6922aa6cfe2..24821e3b75d 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/RestartTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/RestartTest.java @@ -25,7 +25,7 @@ public class RestartTest { dockerTester.addContainerNodeSpec(createContainerNodeSpec(wantedRestartGeneration, currentRestartGeneration)); // Wait for node admin to be notified with node repo state and the docker container has been started - while (dockerTester.getNodeAdmin().getListOfHosts().size() == 0) { + while (dockerTester.nodeAdmin.getListOfHosts().size() == 0) { Thread.sleep(10); } diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/RunInContainerTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/RunInContainerTest.java index 0fbf7ff7751..63a7a3d1e4e 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/RunInContainerTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/RunInContainerTest.java @@ -42,7 +42,7 @@ import static org.mockito.Mockito.when; */ public class RunInContainerTest { private final Logger logger = Logger.getLogger("RunInContainerTest"); - private final Orchestrator orchestrator = ComponentsProviderWithMocks.orchestratorMock; + private final Orchestrator orchestrator = NodeAdminProviderWithMocks.orchestratorMock; private final String parentHostname = "localhost.test.yahoo.com"; private JDisc container; private int port; @@ -118,7 +118,7 @@ public class RunInContainerTest { @Test public void testGetContainersToRunAPi() throws IOException, InterruptedException { doThrow(new OrchestratorException("Cannot suspend because...")).when(orchestrator).suspend(parentHostname); - when(ComponentsProviderWithMocks.nodeRepositoryMock.getContainersToRun()).thenReturn(Collections.emptyList()); + when(NodeAdminProviderWithMocks.nodeRepositoryMock.getContainersToRun(eq(parentHostname))).thenReturn(Collections.emptyList()); waitForJdiscContainerToServe(); assertTrue("The initial resume command should fail because it needs to converge first", @@ -144,7 +144,7 @@ public class RunInContainerTest { assertTrue(verifyWithRetries("resume", true)); // Lets try the same, but with an active container running on this host - when(ComponentsProviderWithMocks.nodeRepositoryMock.getContainersToRun()).thenReturn( + when(NodeAdminProviderWithMocks.nodeRepositoryMock.getContainersToRun(eq(parentHostname))).thenReturn( Collections.singletonList(new ContainerNodeSpec.Builder() .hostname("host1.test.yahoo.com") .wantedDockerImage(new DockerImage("dockerImage")) @@ -167,9 +167,9 @@ public class RunInContainerTest { assertTrue(verifyWithRetries("suspend/node-admin", true)); // Allow stopping services in active nodes - doNothing().when(ComponentsProviderWithMocks.dockerOperationsMock) + doNothing().when(NodeAdminProviderWithMocks.dockerOperationsMock) .trySuspendNode(eq(new ContainerName("host1"))); - doNothing().when(ComponentsProviderWithMocks.dockerOperationsMock) + doNothing().when(NodeAdminProviderWithMocks.dockerOperationsMock) .stopServicesOnNode(eq(new ContainerName("host1"))); assertTrue(verifyWithRetries("suspend", false)); @@ -191,7 +191,7 @@ public class RunInContainerTest { " <handler id=\"com.yahoo.vespa.hosted.node.admin.restapi.RestApiHandler\" bundle=\"node-admin\">\n" + " <binding>http://*/rest/*</binding>\n" + " </handler>\n" + - " <component id=\"node-admin\" class=\"com.yahoo.vespa.hosted.node.admin.integrationTests.ComponentsProviderWithMocks\" bundle=\"node-admin\"/>\n" + + " <component id=\"node-admin\" class=\"com.yahoo.vespa.hosted.node.admin.integrationTests.NodeAdminProviderWithMocks\" bundle=\"node-admin\"/>\n" + " <http>" + " <server id=\'myServer\' port=\'" + port + "\' />" + " </http>" + diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImplTest.java index bbfd0f73fd3..582992869aa 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImplTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImplTest.java @@ -7,12 +7,13 @@ import com.yahoo.test.ManualClock; import com.yahoo.vespa.hosted.dockerapi.ContainerName; import com.yahoo.vespa.hosted.dockerapi.metrics.MetricReceiverWrapper; import com.yahoo.vespa.hosted.node.admin.docker.DockerOperations; +import com.yahoo.vespa.hosted.node.admin.maintenance.StorageMaintainer; +import com.yahoo.vespa.hosted.node.admin.maintenance.acl.AclMaintainer; import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgent; import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentImpl; import org.junit.Test; import org.mockito.InOrder; -import java.time.Clock; import java.time.Duration; import java.util.ArrayList; import java.util.Collections; @@ -30,7 +31,6 @@ import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; import static org.mockito.Matchers.any; import static org.mockito.Matchers.anyBoolean; -import static org.mockito.Matchers.anyInt; import static org.mockito.Matchers.eq; import static org.mockito.Mockito.inOrder; import static org.mockito.Mockito.mock; @@ -45,15 +45,17 @@ import static org.mockito.Mockito.when; public class NodeAdminImplTest { // Trick to allow mocking of typed interface without casts/warnings. private interface NodeAgentFactory extends Function<String, NodeAgent> {} + private final DockerOperations dockerOperations = mock(DockerOperations.class); + private final Function<String, NodeAgent> nodeAgentFactory = mock(NodeAgentFactory.class); + private final StorageMaintainer storageMaintainer = mock(StorageMaintainer.class); + private final AclMaintainer aclMaintainer = mock(AclMaintainer.class); + private final ManualClock clock = new ManualClock(); + + private final NodeAdminImpl nodeAdmin = new NodeAdminImpl(dockerOperations, nodeAgentFactory, storageMaintainer, aclMaintainer, + new MetricReceiverWrapper(MetricReceiver.nullImplementation), clock); @Test public void nodeAgentsAreProperlyLifeCycleManaged() throws Exception { - final DockerOperations dockerOperations = mock(DockerOperations.class); - final Function<String, NodeAgent> nodeAgentFactory = mock(NodeAgentFactory.class); - - final NodeAdminImpl nodeAdmin = new NodeAdminImpl(dockerOperations, nodeAgentFactory, Optional.empty(), 100, - new MetricReceiverWrapper(MetricReceiver.nullImplementation), Optional.empty(), Clock.systemUTC()); - final String hostName1 = "host1.test.yahoo.com"; final String hostName2 = "host2.test.yahoo.com"; final ContainerName containerName1 = ContainerName.fromHostname(hostName1); @@ -69,12 +71,12 @@ public class NodeAdminImplTest { nodeAdmin.synchronizeNodeSpecsToNodeAgents(Collections.singletonList(hostName1), Collections.singletonList(containerName1)); inOrder.verify(nodeAgentFactory).apply(hostName1); - inOrder.verify(nodeAgent1).start(100); + inOrder.verify(nodeAgent1).start(); inOrder.verify(nodeAgent1, never()).stop(); nodeAdmin.synchronizeNodeSpecsToNodeAgents(Collections.singletonList(hostName1), Collections.singletonList(containerName1)); inOrder.verify(nodeAgentFactory, never()).apply(any(String.class)); - inOrder.verify(nodeAgent1, never()).start(anyInt()); + inOrder.verify(nodeAgent1, never()).start(); inOrder.verify(nodeAgent1, never()).stop(); nodeAdmin.synchronizeNodeSpecsToNodeAgents(Collections.emptyList(), Collections.singletonList(containerName1)); @@ -83,13 +85,13 @@ public class NodeAdminImplTest { nodeAdmin.synchronizeNodeSpecsToNodeAgents(Collections.singletonList(hostName2), Collections.singletonList(containerName1)); inOrder.verify(nodeAgentFactory).apply(hostName2); - inOrder.verify(nodeAgent2).start(100); + inOrder.verify(nodeAgent2).start(); inOrder.verify(nodeAgent2, never()).stop(); verify(nodeAgent1).stop(); nodeAdmin.synchronizeNodeSpecsToNodeAgents(Collections.emptyList(), Collections.emptyList()); inOrder.verify(nodeAgentFactory, never()).apply(any(String.class)); - inOrder.verify(nodeAgent2, never()).start(anyInt()); + inOrder.verify(nodeAgent2, never()).start(); inOrder.verify(nodeAgent2).stop(); verifyNoMoreInteractions(nodeAgent1); @@ -98,12 +100,6 @@ public class NodeAdminImplTest { @Test public void testSetFrozen() throws Exception { - final DockerOperations dockerOperations = mock(DockerOperations.class); - final Function<String, NodeAgent> nodeAgentFactory = mock(NodeAgentFactory.class); - - final NodeAdminImpl nodeAdmin = new NodeAdminImpl(dockerOperations, nodeAgentFactory, Optional.empty(), 100, - new MetricReceiverWrapper(MetricReceiver.nullImplementation), Optional.empty(), Clock.systemUTC()); - List<NodeAgent> nodeAgents = new ArrayList<>(); List<String> existingContainerHostnames = new ArrayList<>(); for (int i = 0; i < 3; i++) { @@ -153,18 +149,6 @@ public class NodeAdminImplTest { @Test public void testSubsystemFreezeDuration() { - final DockerOperations dockerOperations = mock(DockerOperations.class); - final Function<String, NodeAgent> nodeAgentFactory = mock(NodeAgentFactory.class); - final ManualClock clock = new ManualClock(); - final NodeAdminImpl nodeAdmin = new NodeAdminImpl( - dockerOperations, - nodeAgentFactory, - Optional.empty(), - 100, - new MetricReceiverWrapper(MetricReceiver.nullImplementation), - Optional.empty(), - clock); - // Initially everything is frozen to force convergence assertTrue(nodeAdmin.isFrozen()); assertTrue(nodeAdmin.subsystemFreezeDuration().isZero()); diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdaterTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdaterTest.java index 712000a8cb7..e1501cf59fe 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdaterTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdaterTest.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.hosted.node.admin.nodeadmin; import com.yahoo.test.ManualClock; import com.yahoo.vespa.hosted.node.admin.ContainerNodeSpec; +import com.yahoo.vespa.hosted.node.admin.maintenance.StorageMaintainer; import com.yahoo.vespa.hosted.node.admin.noderepository.NodeRepository; import com.yahoo.vespa.hosted.node.admin.orchestrator.Orchestrator; import com.yahoo.vespa.hosted.node.admin.orchestrator.OrchestratorException; @@ -14,7 +15,6 @@ import java.time.Duration; import java.util.ArrayList; import java.util.Arrays; import java.util.List; -import java.util.Optional; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; @@ -32,17 +32,20 @@ import static org.mockito.Mockito.when; /** * Basic test of NodeAdminStateUpdater + * * @author freva */ public class NodeAdminStateUpdaterTest { - private final String parentHostname = "basehost1.test.yahoo.com"; - - private final ManualClock clock = new ManualClock(); private final NodeRepository nodeRepository = mock(NodeRepository.class); - private final NodeAdmin nodeAdmin = mock(NodeAdmin.class); private final Orchestrator orchestrator = mock(Orchestrator.class); + private final StorageMaintainer storageMaintainer = mock(StorageMaintainer.class); + private final NodeAdmin nodeAdmin = mock(NodeAdmin.class); + private final String parentHostname = "basehost1.test.yahoo.com"; + private final ManualClock clock = new ManualClock(); + private final Duration convergeStateInterval = Duration.ofSeconds(30); + private final NodeAdminStateUpdater refresher = spy(new NodeAdminStateUpdater( - nodeRepository, nodeAdmin, Optional.empty(), clock, orchestrator, parentHostname)); + nodeRepository, orchestrator, storageMaintainer, nodeAdmin, parentHostname, clock, convergeStateInterval)); @Test @@ -65,7 +68,7 @@ public class NodeAdminStateUpdaterTest { List<String> suspendHostnames = new ArrayList<>(activeHostnames); suspendHostnames.add(parentHostname); - when(nodeRepository.getContainersToRun()).thenReturn(containersToRun); + when(nodeRepository.getContainersToRun(eq(parentHostname))).thenReturn(containersToRun); // Initially everything is frozen to force convergence assertFalse(refresher.setResumeStateAndCheckIfResumed(NodeAdminStateUpdater.State.RESUMED)); diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java index 09e897e1fc2..ceaa1d58f92 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java @@ -57,6 +57,7 @@ import static org.mockito.Mockito.when; * @author bakksjo */ public class NodeAgentImplTest { + private static final Duration NODE_AGENT_SCAN_INTERVAL = Duration.ofSeconds(30); private static final double MIN_CPU_CORES = 2; private static final double MIN_MAIN_MEMORY_AVAILABLE_GB = 16; private static final double MIN_DISK_AVAILABLE_GB = 250; @@ -621,6 +622,6 @@ public class NodeAgentImplTest { doNothing().when(storageMaintainer).writeMetricsConfig(any(), any()); return new NodeAgentImpl(hostName, nodeRepository, orchestrator, dockerOperations, - Optional.of(storageMaintainer), environment, clock, Optional.of(aclMaintainer)); + storageMaintainer, aclMaintainer, environment, clock, NODE_AGENT_SCAN_INTERVAL); } } diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/noderepository/NodeRepositoryImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/noderepository/NodeRepositoryImplTest.java index bfabf0a4e4e..7456b07d2f4 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/noderepository/NodeRepositoryImplTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/noderepository/NodeRepositoryImplTest.java @@ -63,10 +63,10 @@ public class NodeRepositoryImplTest { private void waitForJdiscContainerToServe() throws InterruptedException { Instant start = Instant.now(); - NodeRepository nodeRepositoryApi = new NodeRepositoryImpl(requestExecutor, port, "foobar"); + NodeRepository nodeRepositoryApi = new NodeRepositoryImpl(requestExecutor, port); while (Instant.now().minusSeconds(120).isBefore(start)) { try { - nodeRepositoryApi.getContainersToRun(); + nodeRepositoryApi.getContainersToRun("foobar"); return; } catch (Exception e) { Thread.sleep(100); @@ -85,8 +85,10 @@ public class NodeRepositoryImplTest { @Test public void testGetContainersToRunApi() throws IOException, InterruptedException { waitForJdiscContainerToServe(); - NodeRepository nodeRepositoryApi = new NodeRepositoryImpl(requestExecutor, port, "dockerhost1.yahoo.com"); - final List<ContainerNodeSpec> containersToRun = nodeRepositoryApi.getContainersToRun(); + NodeRepository nodeRepositoryApi = new NodeRepositoryImpl(requestExecutor, port); + String dockerHostHostname = "dockerhost1.yahoo.com"; + + final List<ContainerNodeSpec> containersToRun = nodeRepositoryApi.getContainersToRun(dockerHostHostname); assertThat(containersToRun.size(), is(1)); final ContainerNodeSpec nodeSpec = containersToRun.get(0); assertThat(nodeSpec.hostname, is("host4.yahoo.com")); @@ -102,7 +104,7 @@ public class NodeRepositoryImplTest { @Test public void testGetContainer() throws InterruptedException, IOException { waitForJdiscContainerToServe(); - NodeRepository nodeRepositoryApi = new NodeRepositoryImpl(requestExecutor, port, "dockerhost1.yahoo.com"); + NodeRepository nodeRepositoryApi = new NodeRepositoryImpl(requestExecutor, port); String hostname = "host4.yahoo.com"; Optional<ContainerNodeSpec> nodeSpec = nodeRepositoryApi.getContainerNodeSpec(hostname); assertThat(nodeSpec.isPresent(), is(true)); @@ -112,7 +114,7 @@ public class NodeRepositoryImplTest { @Test public void testGetContainerForNonExistingNode() throws InterruptedException, IOException { waitForJdiscContainerToServe(); - NodeRepository nodeRepositoryApi = new NodeRepositoryImpl(requestExecutor, port, "dockerhost1.yahoo.com"); + NodeRepository nodeRepositoryApi = new NodeRepositoryImpl(requestExecutor, port); String hostname = "host-that-does-not-exist"; Optional<ContainerNodeSpec> nodeSpec = nodeRepositoryApi.getContainerNodeSpec(hostname); assertFalse(nodeSpec.isPresent()); @@ -121,7 +123,7 @@ public class NodeRepositoryImplTest { @Test public void testUpdateNodeAttributes() throws InterruptedException, IOException { waitForJdiscContainerToServe(); - NodeRepository nodeRepositoryApi = new NodeRepositoryImpl(requestExecutor, port, "dockerhost1.yahoo.com"); + NodeRepository nodeRepositoryApi = new NodeRepositoryImpl(requestExecutor, port); String hostname = "host4.yahoo.com"; nodeRepositoryApi.updateNodeAttributes( hostname, @@ -134,7 +136,7 @@ public class NodeRepositoryImplTest { @Test(expected = RuntimeException.class) public void testUpdateNodeAttributesWithBadValue() throws InterruptedException, IOException { waitForJdiscContainerToServe(); - NodeRepository nodeRepositoryApi = new NodeRepositoryImpl(requestExecutor, port, "dockerhost1.yahoo.com"); + NodeRepository nodeRepositoryApi = new NodeRepositoryImpl(requestExecutor, port); String hostname = "host4.yahoo.com"; nodeRepositoryApi.updateNodeAttributes( hostname, @@ -146,7 +148,7 @@ public class NodeRepositoryImplTest { @Test public void testMarkAsReady() throws InterruptedException, IOException { - NodeRepository nodeRepositoryApi = new NodeRepositoryImpl(requestExecutor, port, "dockerhost1.yahoo.com"); + NodeRepository nodeRepositoryApi = new NodeRepositoryImpl(requestExecutor, port); waitForJdiscContainerToServe(); nodeRepositoryApi.markAsDirty("host5.yahoo.com"); diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/orchestrator/OrchestratorImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/orchestrator/OrchestratorImplTest.java index 6f08016dff5..cef43a058c0 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/orchestrator/OrchestratorImplTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/orchestrator/OrchestratorImplTest.java @@ -6,7 +6,6 @@ import com.yahoo.vespa.orchestrator.restapi.wire.BatchHostSuspendRequest; import com.yahoo.vespa.orchestrator.restapi.wire.BatchOperationResult; import com.yahoo.vespa.orchestrator.restapi.wire.HostStateChangeDenialReason; import com.yahoo.vespa.orchestrator.restapi.wire.UpdateHostResponse; -import org.junit.Before; import org.junit.Test; import java.util.Arrays; @@ -20,20 +19,17 @@ import static org.mockito.Mockito.*; */ public class OrchestratorImplTest { private static final String hostName = "host123.yahoo.com"; - private ConfigServerHttpRequestExecutor requestExecutor; - private OrchestratorImpl orchestrator; - @Before - public void setup() { - requestExecutor = mock(ConfigServerHttpRequestExecutor.class); - orchestrator = new OrchestratorImpl(requestExecutor); - } + private final ConfigServerHttpRequestExecutor requestExecutor = mock(ConfigServerHttpRequestExecutor.class); + private final int port = 1234; + private final OrchestratorImpl orchestrator = new OrchestratorImpl(requestExecutor, port); + @Test public void testSuspendCall() { when(requestExecutor.put( OrchestratorImpl.ORCHESTRATOR_PATH_PREFIX_HOST_API + "/" + hostName+ "/suspended", - OrchestratorImpl.WEB_SERVICE_PORT, + port, Optional.empty(), UpdateHostResponse.class )).thenReturn(new UpdateHostResponse(hostName, null)); @@ -45,7 +41,7 @@ public class OrchestratorImplTest { public void testSuspendCallWithFailureReason() { when(requestExecutor.put( OrchestratorImpl.ORCHESTRATOR_PATH_PREFIX_HOST_API + "/" + hostName+ "/suspended", - OrchestratorImpl.WEB_SERVICE_PORT, + port, Optional.empty(), UpdateHostResponse.class )).thenReturn(new UpdateHostResponse(hostName, new HostStateChangeDenialReason("hostname", "fail"))); @@ -82,7 +78,7 @@ public class OrchestratorImplTest { public void testResumeCall() { when(requestExecutor.delete( OrchestratorImpl.ORCHESTRATOR_PATH_PREFIX_HOST_API + "/" + hostName+ "/suspended", - OrchestratorImpl.WEB_SERVICE_PORT, + port, UpdateHostResponse.class )).thenReturn(new UpdateHostResponse(hostName, null)); @@ -93,7 +89,7 @@ public class OrchestratorImplTest { public void testResumeCallWithFailureReason() { when(requestExecutor.delete( OrchestratorImpl.ORCHESTRATOR_PATH_PREFIX_HOST_API + "/" + hostName+ "/suspended", - OrchestratorImpl.WEB_SERVICE_PORT, + port, UpdateHostResponse.class )).thenReturn(new UpdateHostResponse(hostName, new HostStateChangeDenialReason("hostname", "fail"))); @@ -131,7 +127,7 @@ public class OrchestratorImplTest { when(requestExecutor.put( OrchestratorImpl.ORCHESTRATOR_PATH_PREFIX_HOST_SUSPENSION_API, - OrchestratorImpl.WEB_SERVICE_PORT, + port, Optional.of(new BatchHostSuspendRequest(parentHostName, hostNames)), BatchOperationResult.class )).thenReturn(BatchOperationResult.successResult()); @@ -147,7 +143,7 @@ public class OrchestratorImplTest { when(requestExecutor.put( OrchestratorImpl.ORCHESTRATOR_PATH_PREFIX_HOST_SUSPENSION_API, - OrchestratorImpl.WEB_SERVICE_PORT, + port, Optional.of(new BatchHostSuspendRequest(parentHostName, hostNames)), BatchOperationResult.class )).thenReturn(new BatchOperationResult(failureReason)); @@ -163,7 +159,7 @@ public class OrchestratorImplTest { when(requestExecutor.put( OrchestratorImpl.ORCHESTRATOR_PATH_PREFIX_HOST_SUSPENSION_API, - OrchestratorImpl.WEB_SERVICE_PORT, + port, Optional.of(new BatchHostSuspendRequest(parentHostName, hostNames)), BatchOperationResult.class )).thenThrow(new RuntimeException(exceptionMessage)); diff --git a/orchestrator/pom.xml b/orchestrator/pom.xml index 55bf086dfc7..c2091e74435 100644 --- a/orchestrator/pom.xml +++ b/orchestrator/pom.xml @@ -72,6 +72,12 @@ <scope>provided</scope> </dependency> <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>configdefinitions</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> + <dependency> <groupId>org.glassfish.jersey.ext</groupId> <artifactId>jersey-proxy-client</artifactId> <exclusions> diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorImpl.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorImpl.java index ea45d955861..e88e8a7dd4a 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorImpl.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorImpl.java @@ -9,6 +9,7 @@ import com.yahoo.vespa.applicationmodel.ApplicationInstanceReference; import com.yahoo.vespa.applicationmodel.ClusterId; import com.yahoo.vespa.applicationmodel.HostName; import com.yahoo.vespa.applicationmodel.ServiceCluster; +import com.yahoo.vespa.orchestrator.config.OrchestratorConfig; import com.yahoo.vespa.orchestrator.controller.ClusterControllerClient; import com.yahoo.vespa.orchestrator.controller.ClusterControllerClientFactory; import com.yahoo.vespa.orchestrator.controller.ClusterControllerNodeState; diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/ZookeeperStatusService.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/ZookeeperStatusService.java index a9392711d7d..c84473a5199 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/ZookeeperStatusService.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/ZookeeperStatusService.java @@ -8,20 +8,14 @@ import com.yahoo.vespa.applicationmodel.HostName; import com.yahoo.vespa.curator.Curator; import com.yahoo.vespa.curator.Lock; import com.yahoo.vespa.orchestrator.OrchestratorUtil; -import org.apache.curator.SessionFailRetryLoop; -import org.apache.curator.SessionFailRetryLoop.Mode; -import org.apache.curator.SessionFailRetryLoop.SessionFailedException; import org.apache.curator.framework.recipes.locks.InterProcessSemaphoreMutex; import org.apache.zookeeper.KeeperException.NoNodeException; import org.apache.zookeeper.KeeperException.NodeExistsException; import org.apache.zookeeper.data.Stat; -import javax.annotation.concurrent.GuardedBy; import javax.inject.Inject; import java.time.Duration; -import java.util.HashMap; import java.util.HashSet; -import java.util.Map; import java.util.Set; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; @@ -37,12 +31,6 @@ public class ZookeeperStatusService implements StatusService { private static final Logger log = Logger.getLogger(ZookeeperStatusService.class.getName()); - //For debug purposes only: Used to check that operations depending on a lock is done from a single thread, - //and that a threads doing operations actually owns a corresponding lock, - //and that a single thread only owns a single lock (across all ZookeeperStatusServices) - @GuardedBy("threadsHoldingLock") - private static final Map<Thread, ApplicationInstanceReference> threadsHoldingLock = new HashMap<>(); - final static String HOST_STATUS_BASE_PATH = "/vespa/host-status-service"; final static String APPLICATION_STATUS_BASE_PATH = "/vespa/application-status-service"; @@ -108,55 +96,16 @@ public class ZookeeperStatusService implements StatusService { MutableStatusRegistry lockApplicationInstance_forCurrentThreadOnly( ApplicationInstanceReference applicationInstanceReference, long timeoutSeconds) { - Thread currentThread = Thread.currentThread(); - - //Due to limitations in SessionFailRetryLoop. - assertThreadDoesNotHoldLock(currentThread,"Can't lock " + applicationInstanceReference); + String lockPath = applicationInstanceLock2Path(applicationInstanceReference); + Lock lock = new Lock(lockPath, curator); + lock.acquire(Duration.ofSeconds(timeoutSeconds)); try { - SessionFailRetryLoop sessionFailRetryLoop = - curator.framework().getZookeeperClient().newSessionFailRetryLoop(Mode.FAIL); - sessionFailRetryLoop.start(); - try { - String lockPath = applicationInstanceLockPath(applicationInstanceReference); - InterProcessSemaphoreMutex mutex = acquireMutexOrThrow(timeoutSeconds, TimeUnit.SECONDS, lockPath); - - // TODO: Once rolled out, make this the only lock mechanism - Lock lock2; - try { - String lock2Path = applicationInstanceLock2Path(applicationInstanceReference); - lock2 = new Lock(lock2Path, curator); - lock2.acquire(Duration.ofSeconds(timeoutSeconds)); - } catch (Throwable t) { - mutex.release(); - throw t; - } - - synchronized (threadsHoldingLock) { - threadsHoldingLock.put(currentThread, applicationInstanceReference); - } - - return new ZkMutableStatusRegistry( - lock2, - mutex, - sessionFailRetryLoop, - applicationInstanceReference, - currentThread); - } catch (Throwable t) { - sessionFailRetryLoop.close(); - throw t; - } - } catch (Exception e) { - //TODO: IOException with explanation - throw new RuntimeException(e); - } - } - - private void assertThreadDoesNotHoldLock(Thread currentThread, String message) { - synchronized (threadsHoldingLock) { - if (threadsHoldingLock.containsKey(currentThread)) { - throw new AssertionError(message + ", already have a lock on " + threadsHoldingLock.get(currentThread)); - } + return new ZkMutableStatusRegistry(lock, applicationInstanceReference); + } catch (Throwable t) { + // In case the constructor throws an exception. + lock.close(); + throw t; } } @@ -176,8 +125,6 @@ public class ZookeeperStatusService implements StatusService { private void setHostStatus(ApplicationInstanceReference applicationInstanceReference, HostName hostName, HostStatus status) { - assertThreadHoldsLock(applicationInstanceReference); - String path = hostAllowedDownPath(applicationInstanceReference, hostName); try { @@ -195,24 +142,6 @@ public class ZookeeperStatusService implements StatusService { } } - private static void assertThreadHoldsLock(ApplicationInstanceReference applicationInstanceReference) { - synchronized (threadsHoldingLock) { - ApplicationInstanceReference lockedApplicationInstanceReference = - threadsHoldingLock.get(Thread.currentThread()); - - if (lockedApplicationInstanceReference == null) { - throw new AssertionError("The current thread does not own any status service locks. " + - "Application Instance = " + applicationInstanceReference); - } - - if (!lockedApplicationInstanceReference.equals(applicationInstanceReference)) { - throw new AssertionError("The current thread does not have a lock on " + - "application instance " + applicationInstanceReference + - ", but instead have a lock on " + lockedApplicationInstanceReference); - } - } - } - private void deleteNode_ignoreNoNodeException(String path, String debugLogMessageIfNotExists) throws Exception { try { curator.framework().delete().forPath(path); @@ -259,7 +188,6 @@ public class ZookeeperStatusService implements StatusService { private HostStatus getHostStatusWithLock( final ApplicationInstanceReference applicationInstanceReference, final HostName hostName) { - assertThreadHoldsLock(applicationInstanceReference); return getInternalHostStatus(applicationInstanceReference, hostName); } @@ -290,23 +218,13 @@ public class ZookeeperStatusService implements StatusService { private class ZkMutableStatusRegistry implements MutableStatusRegistry { private final Lock lock; - private final InterProcessSemaphoreMutex mutex; - private final SessionFailRetryLoop sessionFailRetryLoop; private final ApplicationInstanceReference applicationInstanceReference; - private final Thread lockingThread; public ZkMutableStatusRegistry( Lock lock, - InterProcessSemaphoreMutex mutex, - SessionFailRetryLoop sessionFailRetryLoop, - ApplicationInstanceReference applicationInstanceReference, - Thread lockingThread) { - - this.mutex = mutex; + ApplicationInstanceReference applicationInstanceReference) { this.lock = lock; - this.sessionFailRetryLoop = sessionFailRetryLoop; this.applicationInstanceReference = applicationInstanceReference; - this.lockingThread = lockingThread; } @Override @@ -316,8 +234,6 @@ public class ZookeeperStatusService implements StatusService { @Override public void setApplicationInstanceStatus(ApplicationInstanceStatus applicationInstanceStatus) { - assertThreadHoldsLock(applicationInstanceReference); - String path = applicationInstanceSuspendedPath(applicationInstanceReference); try { @@ -349,10 +265,6 @@ public class ZookeeperStatusService implements StatusService { @Override @NoThrow public void close() { - synchronized (threadsHoldingLock) { - threadsHoldingLock.remove(lockingThread, applicationInstanceReference); - } - try { lock.close(); } catch (RuntimeException e) { @@ -360,33 +272,6 @@ public class ZookeeperStatusService implements StatusService { log.log(LogLevel.WARNING, "Failed to close application lock for " + ZookeeperStatusService.class.getSimpleName() + ", will ignore and continue", e); } - - try { - mutex.release(); - } catch (Exception e) { - if (e.getCause() instanceof SessionFailedException) { - log.log(LogLevel.DEBUG, "Session expired, mutex should be freed automatically", e); - } else { - //Failing to unlock the mutex should not fail the request, - //since the status database has already been updated at this point. - log.log(LogLevel.WARNING, "Failed unlocking application instance " + applicationInstanceReference, e); - } - } - - //Similar precondition checked in sessionFailRetryLoop.close, - //but this has more useful debug output. - if (lockingThread != Thread.currentThread()) { - throw new AssertionError("LockHandle should only be used from a single thread. " - + "Application instance = " + applicationInstanceReference - + " Locking thread = " + lockingThread - + " Current thread = " + Thread.currentThread()); - } - - try { - sessionFailRetryLoop.close(); - } catch (Exception e) { - log.log(LogLevel.ERROR, "Failed closing SessionRetryLoop", e); - } } } } diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorImplTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorImplTest.java index 11518bb4545..92d6216583f 100644 --- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorImplTest.java +++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorImplTest.java @@ -5,6 +5,7 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.vespa.applicationmodel.ApplicationInstance; import com.yahoo.vespa.applicationmodel.ApplicationInstanceReference; import com.yahoo.vespa.applicationmodel.HostName; +import com.yahoo.vespa.orchestrator.config.OrchestratorConfig; import com.yahoo.vespa.orchestrator.controller.ClusterControllerClientFactoryMock; import com.yahoo.vespa.orchestrator.policy.BatchHostStateChangeDeniedException; import com.yahoo.vespa.orchestrator.policy.HostStateChangeDeniedException; diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/status/ZookeeperStatusServiceTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/status/ZookeeperStatusServiceTest.java index e997743fcf0..1e742b5940a 100644 --- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/status/ZookeeperStatusServiceTest.java +++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/status/ZookeeperStatusServiceTest.java @@ -128,26 +128,6 @@ public class ZookeeperStatusServiceTest { } @Test - public void session_expiry_when_holding_lock_causes_operations_to_fail() throws Exception { - try (MutableStatusRegistry statusRegistry = zookeeperStatusService.lockApplicationInstance_forCurrentThreadOnly( - TestIds.APPLICATION_INSTANCE_REFERENCE)) { - - KillSession.kill(curator.framework().getZookeeperClient().getZooKeeper(), testingServer.getConnectString()); - - assertSessionFailed(() -> - statusRegistry.setHostState( - TestIds.HOST_NAME1, - HostStatus.ALLOWED_TO_BE_DOWN)); - - - assertSessionFailed(() -> - statusRegistry.getHostStatus( - TestIds.HOST_NAME1)); - - } - } - - @Test public void failing_to_get_lock_closes_SessionFailRetryLoop() throws Exception { try (Curator curator = createConnectedCuratorFramework(testingServer)) { ZookeeperStatusService zookeeperStatusService2 = new ZookeeperStatusService(curator); @@ -221,23 +201,6 @@ public class ZookeeperStatusServiceTest { } } - /** - * This requirement is due to limitations in SessionFailRetryLoop - */ - @Test(expected = AssertionError.class) - public void multiple_locks_in_a_single_thread_gives_error() throws InterruptedException { - try (Curator curator = createConnectedCuratorFramework(testingServer)) { - ZookeeperStatusService zookeeperStatusService2 = new ZookeeperStatusService(curator); - - try (MutableStatusRegistry statusRegistry1 = zookeeperStatusService - .lockApplicationInstance_forCurrentThreadOnly(TestIds.APPLICATION_INSTANCE_REFERENCE); - MutableStatusRegistry statusRegistry2 = zookeeperStatusService2 - .lockApplicationInstance_forCurrentThreadOnly(TestIds.APPLICATION_INSTANCE_REFERENCE2)) - { - } - } - } - @Test public void suspend_and_resume_application_works_and_is_symmetric() { diff --git a/predicate-search-core/src/main/java/com/yahoo/search/predicate/SubqueryBitmap.java b/predicate-search-core/src/main/java/com/yahoo/search/predicate/SubqueryBitmap.java index 6582e5f08ca..d6ca4971329 100644 --- a/predicate-search-core/src/main/java/com/yahoo/search/predicate/SubqueryBitmap.java +++ b/predicate-search-core/src/main/java/com/yahoo/search/predicate/SubqueryBitmap.java @@ -2,7 +2,8 @@ package com.yahoo.search.predicate; /** - * Constants related to the subquery bitmap in predicate/boolean search. + * Constants related to the subquery bitmap in predicate search. + * * @author bjorncs */ public interface SubqueryBitmap { @@ -17,5 +18,4 @@ public interface SubqueryBitmap { */ long DEFAULT_VALUE = ALL_SUBQUERIES; - } diff --git a/predicate-search/README.md b/predicate-search/README.md index 8de3bf9003a..2e21b01b406 100644 --- a/predicate-search/README.md +++ b/predicate-search/README.md @@ -1,3 +1,3 @@ -# Boolean search library +# Predicate search library -Java library for indexing boolean expressions and querying them using boolean constraints. +Java library for indexing predicates and querying them using boolean constraints. diff --git a/predicate-search/src/main/java/com/yahoo/search/predicate/index/PredicateSearch.java b/predicate-search/src/main/java/com/yahoo/search/predicate/index/PredicateSearch.java index 2e6d6a0c7fd..a46b50916b0 100644 --- a/predicate-search/src/main/java/com/yahoo/search/predicate/index/PredicateSearch.java +++ b/predicate-search/src/main/java/com/yahoo/search/predicate/index/PredicateSearch.java @@ -15,12 +15,13 @@ import java.util.stream.Stream; import java.util.stream.StreamSupport; /** - * Implementation of the "Interval" boolean search algorithm. + * Implementation of the "Interval" predicate search algorithm. * - * @author <a href="mailto:magnarn@yahoo-inc.com">Magnar Nedland</a> + * @author Magnar Nedland * @author bjorncs */ public class PredicateSearch { + private final PostingList[] postingLists; private final byte[] nPostingListsForDocument; private final byte[] minFeatureIndex; @@ -37,6 +38,7 @@ public class PredicateSearch { /** * Creates a search for a set of posting lists. + * * @param postingLists Posting lists for the boolean variables that evaluate to true * @param nPostingListsForDocument The number of posting list for each docId * @param minFeatureIndex Index from docId to min-feature value. @@ -278,4 +280,5 @@ public class PredicateSearch { } } + } diff --git a/searchcommon/src/vespa/searchcommon/common/range.h b/searchcommon/src/vespa/searchcommon/common/range.h index e34bf85e622..f00afa0c0aa 100644 --- a/searchcommon/src/vespa/searchcommon/common/range.h +++ b/searchcommon/src/vespa/searchcommon/common/range.h @@ -1,6 +1,4 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// Copyright (C) 1998-2003 Fast Search & Transfer ASA -// Copyright (C) 2003 Overture Services Norway AS #pragma once diff --git a/searchcore/src/tests/proton/documentdb/feedview/feedview_test.cpp b/searchcore/src/tests/proton/documentdb/feedview/feedview_test.cpp index c02aa033bb9..eeec8122703 100644 --- a/searchcore/src/tests/proton/documentdb/feedview/feedview_test.cpp +++ b/searchcore/src/tests/proton/documentdb/feedview/feedview_test.cpp @@ -198,7 +198,7 @@ public: { } - virtual void notifyPut(document::GlobalId gid, uint32_t lid, SerialNum) override { + virtual void notifyPutDone(document::GlobalId gid, uint32_t lid, SerialNum) override { _changeGid = gid; _changeLid = lid; _gidToLid[gid] = lid; diff --git a/searchcore/src/tests/proton/matching/matching_test.cpp b/searchcore/src/tests/proton/matching/matching_test.cpp index fefc6f361a9..261827a1e06 100644 --- a/searchcore/src/tests/proton/matching/matching_test.cpp +++ b/searchcore/src/tests/proton/matching/matching_test.cpp @@ -292,8 +292,8 @@ struct MyWorld { MatchToolsFactory::UP match_tools_factory = matcher->create_match_tools_factory( *request, searchContext, attributeContext, metaStore, overrides); MatchTools::UP match_tools = match_tools_factory->createMatchTools(); - RankProgram::UP rank_program = match_tools->first_phase_program(); - return rank_program->match_data().get_termwise_limit(); + match_tools->setup_first_phase(); + return match_tools->match_data().get_termwise_limit(); } SearchReply::UP performSearch(SearchRequest::SP req, size_t threads) { diff --git a/searchcore/src/tests/proton/reference/gid_to_lid_change_handler/gid_to_lid_change_handler_test.cpp b/searchcore/src/tests/proton/reference/gid_to_lid_change_handler/gid_to_lid_change_handler_test.cpp index 1c5287f15e5..625e9c1f6a9 100644 --- a/searchcore/src/tests/proton/reference/gid_to_lid_change_handler/gid_to_lid_change_handler_test.cpp +++ b/searchcore/src/tests/proton/reference/gid_to_lid_change_handler/gid_to_lid_change_handler_test.cpp @@ -53,7 +53,7 @@ public: EXPECT_EQUAL(_createdListeners, _destroyedListeners); } - void notifyPut() { + void notifyPutDone() { lock_guard guard(_lock); ++_putChanges; } @@ -101,7 +101,7 @@ public: _stats.markCreatedListener(); } virtual ~MyListener() { _stats.markDestroyedListener(); } - virtual void notifyPut(GlobalId, uint32_t) override { _stats.notifyPut(); } + virtual void notifyPutDone(GlobalId, uint32_t) override { _stats.notifyPutDone(); } virtual void notifyRemove(GlobalId) override { _stats.notifyRemove(); } virtual void notifyRegistered() override { _stats.markRegisteredListener(); } virtual const vespalib::string &getName() const override { return _name; } @@ -138,8 +138,8 @@ struct Fixture _handler->addListener(std::move(listener)); } - void notifyPut(GlobalId gid, uint32_t lid, SerialNum serialNum) { - _handler->notifyPut(gid, lid, serialNum); + void notifyPutDone(GlobalId gid, uint32_t lid, SerialNum serialNum) { + _handler->notifyPutDone(gid, lid, serialNum); } void notifyRemove(GlobalId gid, SerialNum serialNum) { @@ -164,7 +164,7 @@ TEST_F("Test that we can register a listener", Fixture) TEST_DO(stats.assertListeners(1, 0, 0)); f.addListener(std::move(listener)); TEST_DO(stats.assertListeners(1, 1, 0)); - f.notifyPut(toGid(doc1), 10, 10); + f.notifyPutDone(toGid(doc1), 10, 10); TEST_DO(stats.assertChanges(1, 0)); f.removeListeners("testdoc", {}); TEST_DO(stats.assertListeners(1, 1, 1)); @@ -187,7 +187,7 @@ TEST_F("Test that we can register multiple listeners", Fixture) TEST_DO(stats1.assertListeners(1, 1, 0)); TEST_DO(stats2.assertListeners(1, 1, 0)); TEST_DO(stats3.assertListeners(1, 1, 0)); - f.notifyPut(toGid(doc1), 10, 10); + f.notifyPutDone(toGid(doc1), 10, 10); TEST_DO(stats1.assertChanges(1, 0)); TEST_DO(stats2.assertChanges(1, 0)); TEST_DO(stats3.assertChanges(1, 0)); @@ -222,42 +222,85 @@ TEST_F("Test that we keep old listener when registering duplicate", Fixture) TEST_DO(stats.assertListeners(2, 1, 1)); } -TEST_F("Test that put is ignored if we have a pending remove", Fixture) +class StatsFixture : public Fixture +{ + ListenerStats &_stats; + +public: + StatsFixture() + : Fixture(), + _stats(addStats()) + { + addListener(std::make_unique<MyListener>(_stats, "test", "testdoc")); + } + + ~StatsFixture() + { + removeListeners("testdoc", {}); + } + + void assertChanges(uint32_t expPutChanges, uint32_t expRemoveChanges) + { + TEST_DO(_stats.assertChanges(expPutChanges, expRemoveChanges)); + } +}; + +TEST_F("Test that put is ignored if we have a pending remove", StatsFixture) { - auto &stats = f.addStats(); - auto listener = std::make_unique<MyListener>(stats, "test", "testdoc"); - f.addListener(std::move(listener)); f.notifyRemove(toGid(doc1), 20); - TEST_DO(stats.assertChanges(0, 1)); - f.notifyPut(toGid(doc1), 10, 10); - TEST_DO(stats.assertChanges(0, 1)); + TEST_DO(f.assertChanges(0, 1)); + f.notifyPutDone(toGid(doc1), 10, 10); + TEST_DO(f.assertChanges(0, 1)); f.notifyRemoveDone(toGid(doc1), 20); - TEST_DO(stats.assertChanges(0, 1)); - f.notifyPut(toGid(doc1), 11, 30); - TEST_DO(stats.assertChanges(1, 1)); - f.removeListeners("testdoc", {}); + TEST_DO(f.assertChanges(0, 1)); + f.notifyPutDone(toGid(doc1), 11, 30); + TEST_DO(f.assertChanges(1, 1)); } -TEST_F("Test that pending removes are merged", Fixture) +TEST_F("Test that pending removes are merged", StatsFixture) { - auto &stats = f.addStats(); - auto listener = std::make_unique<MyListener>(stats, "test", "testdoc"); - f.addListener(std::move(listener)); f.notifyRemove(toGid(doc1), 20); - TEST_DO(stats.assertChanges(0, 1)); + TEST_DO(f.assertChanges(0, 1)); f.notifyRemove(toGid(doc1), 40); - TEST_DO(stats.assertChanges(0, 1)); - f.notifyPut(toGid(doc1), 10, 10); - TEST_DO(stats.assertChanges(0, 1)); + TEST_DO(f.assertChanges(0, 1)); + f.notifyPutDone(toGid(doc1), 10, 10); + TEST_DO(f.assertChanges(0, 1)); f.notifyRemoveDone(toGid(doc1), 20); - TEST_DO(stats.assertChanges(0, 1)); - f.notifyPut(toGid(doc1), 11, 30); - TEST_DO(stats.assertChanges(0, 1)); + TEST_DO(f.assertChanges(0, 1)); + f.notifyPutDone(toGid(doc1), 11, 30); + TEST_DO(f.assertChanges(0, 1)); f.notifyRemoveDone(toGid(doc1), 40); - TEST_DO(stats.assertChanges(0, 1)); - f.notifyPut(toGid(doc1), 12, 50); - TEST_DO(stats.assertChanges(1, 1)); - f.removeListeners("testdoc", {}); + TEST_DO(f.assertChanges(0, 1)); + f.notifyPutDone(toGid(doc1), 12, 50); + TEST_DO(f.assertChanges(1, 1)); +} + +TEST_F("Test that out of order notifyRemoveDone is handled", StatsFixture) +{ + f.notifyRemove(toGid(doc1), 20); + TEST_DO(f.assertChanges(0, 1)); + f.notifyRemove(toGid(doc1), 40); + TEST_DO(f.assertChanges(0, 1)); + f.notifyRemoveDone(toGid(doc1), 40); + TEST_DO(f.assertChanges(0, 1)); + f.notifyRemoveDone(toGid(doc1), 20); + TEST_DO(f.assertChanges(0, 1)); + f.notifyPutDone(toGid(doc1), 12, 50); + TEST_DO(f.assertChanges(1, 1)); +} + +TEST_F("Test that out of order notifyPutDone is partially handled", StatsFixture) +{ + f.notifyRemove(toGid(doc1), 20); + TEST_DO(f.assertChanges(0, 1)); + f.notifyPutDone(toGid(doc1), 12, 50); + TEST_DO(f.assertChanges(1, 1)); + f.notifyPutDone(toGid(doc1), 11, 40); + TEST_DO(f.assertChanges(1, 1)); + f.notifyPutDone(toGid(doc1), 13, 55); + TEST_DO(f.assertChanges(2, 1)); + f.notifyRemoveDone(toGid(doc1), 20); + TEST_DO(f.assertChanges(2, 1)); } } diff --git a/searchcore/src/tests/proton/reference/gid_to_lid_change_listener/gid_to_lid_change_listener_test.cpp b/searchcore/src/tests/proton/reference/gid_to_lid_change_listener/gid_to_lid_change_listener_test.cpp index 4c6f9017000..1cea981e9ff 100644 --- a/searchcore/src/tests/proton/reference/gid_to_lid_change_listener/gid_to_lid_change_listener_test.cpp +++ b/searchcore/src/tests/proton/reference/gid_to_lid_change_listener/gid_to_lid_change_listener_test.cpp @@ -94,8 +94,8 @@ struct Fixture _listener = std::make_unique<GidToLidChangeListener>(_writer, _attr, _refCount, "test", "testdoc"); } - void notifyPut(const GlobalId &gid, uint32_t referencedDoc) { - _listener->notifyPut(gid, referencedDoc); + void notifyPutDone(const GlobalId &gid, uint32_t referencedDoc) { + _listener->notifyPutDone(gid, referencedDoc); } void notifyListenerRegistered() { @@ -114,9 +114,9 @@ TEST_F("Test that we can use gid to lid change listener", Fixture) TEST_DO(f.assertRefLid(0, 2)); TEST_DO(f.assertRefLid(0, 3)); f.allocListener(); - f.notifyPut(toGid(doc1), 10); - f.notifyPut(toGid(doc2), 20); - f.notifyPut(toGid(doc3), 30); + f.notifyPutDone(toGid(doc1), 10); + f.notifyPutDone(toGid(doc2), 20); + f.notifyPutDone(toGid(doc3), 30); TEST_DO(f.assertRefLid(10, 1)); TEST_DO(f.assertRefLid(20, 2)); TEST_DO(f.assertRefLid(10, 3)); diff --git a/searchcore/src/tests/proton/reference/gid_to_lid_change_registrator/gid_to_lid_change_registrator_test.cpp b/searchcore/src/tests/proton/reference/gid_to_lid_change_registrator/gid_to_lid_change_registrator_test.cpp index a5231647158..d240c437ef2 100644 --- a/searchcore/src/tests/proton/reference/gid_to_lid_change_registrator/gid_to_lid_change_registrator_test.cpp +++ b/searchcore/src/tests/proton/reference/gid_to_lid_change_registrator/gid_to_lid_change_registrator_test.cpp @@ -24,7 +24,7 @@ public: { } virtual ~MyListener() { } - virtual void notifyPut(document::GlobalId, uint32_t) override { } + virtual void notifyPutDone(document::GlobalId, uint32_t) override { } virtual void notifyRemove(document::GlobalId) override { } virtual void notifyRegistered() override { } virtual const vespalib::string &getName() const override { return _name; } diff --git a/searchcore/src/vespa/searchcore/config/CMakeLists.txt b/searchcore/src/vespa/searchcore/config/CMakeLists.txt index 416019a8344..c7669efc876 100644 --- a/searchcore/src/vespa/searchcore/config/CMakeLists.txt +++ b/searchcore/src/vespa/searchcore/config/CMakeLists.txt @@ -4,11 +4,11 @@ vespa_add_library(searchcore_fconfig STATIC DEPENDS ) vespa_generate_config(searchcore_fconfig partitions.def) -install(FILES partitions.def DESTINATION var/db/vespa/config_server/serverdb/classes) +install(FILES partitions.def RENAME vespa.config.search.core.partitions.def DESTINATION var/db/vespa/config_server/serverdb/classes) vespa_generate_config(searchcore_fconfig fdispatchrc.def) -install(FILES fdispatchrc.def DESTINATION var/db/vespa/config_server/serverdb/classes) +install(FILES fdispatchrc.def RENAME vespa.config.search.core.fdispatchrc.def DESTINATION var/db/vespa/config_server/serverdb/classes) vespa_generate_config(searchcore_fconfig proton.def) -install(FILES proton.def DESTINATION var/db/vespa/config_server/serverdb/classes) +install(FILES proton.def RENAME vespa.config.search.core.proton.def DESTINATION var/db/vespa/config_server/serverdb/classes) vespa_generate_config(searchcore_fconfig ranking-constants.def) -install(FILES ranking-constants.def DESTINATION var/db/vespa/config_server/serverdb/classes) +install(FILES ranking-constants.def RENAME vespa.config.search.core.ranking-constants.def DESTINATION var/db/vespa/config_server/serverdb/classes) vespa_generate_config(searchcore_fconfig hwinfo.def) diff --git a/searchcore/src/vespa/searchcore/config/proton.def b/searchcore/src/vespa/searchcore/config/proton.def index 428ec0cfcb1..5311e9bde07 100644 --- a/searchcore/src/vespa/searchcore/config/proton.def +++ b/searchcore/src/vespa/searchcore/config/proton.def @@ -20,12 +20,6 @@ rtcspec string default="tcp/localhost:8004" restart ## Port to use for the web server httpport int default=0 restart -## Port to use for the persistence provider rpc server -persistenceprovider.port int default=8006 restart - -## Number of threads used to drive the internal persistence SPI -persistenceprovider.threads int default=16 restart - ## Cluster name clustername string default="" restart diff --git a/searchcore/src/vespa/searchcore/fdispatch/common/appcontext.h b/searchcore/src/vespa/searchcore/fdispatch/common/appcontext.h index 16c8c81bf05..7a5652577a0 100644 --- a/searchcore/src/vespa/searchcore/fdispatch/common/appcontext.h +++ b/searchcore/src/vespa/searchcore/fdispatch/common/appcontext.h @@ -1,6 +1,4 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// Copyright (C) 1998-2003 Fast Search & Transfer ASA -// Copyright (C) 2003 Overture Services Norway AS #pragma once diff --git a/searchcore/src/vespa/searchcore/fdispatch/common/perftask.cpp b/searchcore/src/vespa/searchcore/fdispatch/common/perftask.cpp index d9ebca1dd2c..17113285fab 100644 --- a/searchcore/src/vespa/searchcore/fdispatch/common/perftask.cpp +++ b/searchcore/src/vespa/searchcore/fdispatch/common/perftask.cpp @@ -1,5 +1,4 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// Copyright (C) 2005 Overture Services Norway AS #include "perftask.h" #include "appcontext.h" diff --git a/searchcore/src/vespa/searchcore/fdispatch/common/perftask.h b/searchcore/src/vespa/searchcore/fdispatch/common/perftask.h index 86498149e46..09c3b9840ab 100644 --- a/searchcore/src/vespa/searchcore/fdispatch/common/perftask.h +++ b/searchcore/src/vespa/searchcore/fdispatch/common/perftask.h @@ -1,5 +1,4 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// Copyright (C) 2005 Overture Services Norway AS #pragma once diff --git a/searchcore/src/vespa/searchcore/fdispatch/common/properties.h b/searchcore/src/vespa/searchcore/fdispatch/common/properties.h index b55bcdbdf64..783022ac96e 100644 --- a/searchcore/src/vespa/searchcore/fdispatch/common/properties.h +++ b/searchcore/src/vespa/searchcore/fdispatch/common/properties.h @@ -1,6 +1,4 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// Copyright (C) 1999-2003 Fast Search & Transfer ASA -// Copyright (C) 2003 Overture Services Norway AS #pragma once diff --git a/searchcore/src/vespa/searchcore/fdispatch/program/fdispatch.h b/searchcore/src/vespa/searchcore/fdispatch/program/fdispatch.h index e240989ddd6..a0294e22655 100644 --- a/searchcore/src/vespa/searchcore/fdispatch/program/fdispatch.h +++ b/searchcore/src/vespa/searchcore/fdispatch/program/fdispatch.h @@ -1,6 +1,4 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// Copyright (C) 1998-2003 Fast Search & Transfer ASA -// Copyright (C) 2003 Overture Services Norway AS #pragma once diff --git a/searchcore/src/vespa/searchcore/fdispatch/search/dataset_base.h b/searchcore/src/vespa/searchcore/fdispatch/search/dataset_base.h index 447de11147a..75c47fffc27 100644 --- a/searchcore/src/vespa/searchcore/fdispatch/search/dataset_base.h +++ b/searchcore/src/vespa/searchcore/fdispatch/search/dataset_base.h @@ -1,6 +1,4 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// Copyright (C) 1998-2003 Fast Search & Transfer ASA -// Copyright (C) 2003 Overture Services Norway AS #pragma once diff --git a/searchcore/src/vespa/searchcore/fdispatch/search/datasetcollection.h b/searchcore/src/vespa/searchcore/fdispatch/search/datasetcollection.h index be60d754439..aed33803c02 100644 --- a/searchcore/src/vespa/searchcore/fdispatch/search/datasetcollection.h +++ b/searchcore/src/vespa/searchcore/fdispatch/search/datasetcollection.h @@ -1,6 +1,4 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// Copyright (C) 1998-2003 Fast Search & Transfer ASA -// Copyright (C) 2003 Overture Services Norway AS #pragma once diff --git a/searchcore/src/vespa/searchcore/fdispatch/search/engine_base.h b/searchcore/src/vespa/searchcore/fdispatch/search/engine_base.h index 51c8eb5fbc4..5dbbddb3bb7 100644 --- a/searchcore/src/vespa/searchcore/fdispatch/search/engine_base.h +++ b/searchcore/src/vespa/searchcore/fdispatch/search/engine_base.h @@ -1,6 +1,4 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// Copyright (C) 1998-2003 Fast Search & Transfer ASA -// Copyright (C) 2003 Overture Services Norway AS #pragma once diff --git a/searchcore/src/vespa/searchcore/fdispatch/search/fnet_dataset.h b/searchcore/src/vespa/searchcore/fdispatch/search/fnet_dataset.h index ed9c4ddde08..62e9686f132 100644 --- a/searchcore/src/vespa/searchcore/fdispatch/search/fnet_dataset.h +++ b/searchcore/src/vespa/searchcore/fdispatch/search/fnet_dataset.h @@ -1,6 +1,4 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// Copyright (C) 1998-2003 Fast Search & Transfer ASA -// Copyright (C) 2003 Overture Services Norway AS #pragma once diff --git a/searchcore/src/vespa/searchcore/fdispatch/search/fnet_search.h b/searchcore/src/vespa/searchcore/fdispatch/search/fnet_search.h index 5a2c3c6a0c9..3f9fc3db077 100644 --- a/searchcore/src/vespa/searchcore/fdispatch/search/fnet_search.h +++ b/searchcore/src/vespa/searchcore/fdispatch/search/fnet_search.h @@ -1,6 +1,4 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// Copyright (C) 1998-2003 Fast Search & Transfer ASA -// Copyright (C) 2003 Overture Services Norway AS #pragma once diff --git a/searchcore/src/vespa/searchcore/fdispatch/search/plain_dataset.h b/searchcore/src/vespa/searchcore/fdispatch/search/plain_dataset.h index 9afd44046a4..827abc13ce7 100644 --- a/searchcore/src/vespa/searchcore/fdispatch/search/plain_dataset.h +++ b/searchcore/src/vespa/searchcore/fdispatch/search/plain_dataset.h @@ -1,6 +1,4 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// Copyright (C) 1998-2003 Fast Search & Transfer ASA -// Copyright (C) 2003 Overture Services Norway AS #pragma once diff --git a/searchcore/src/vespa/searchcore/fdispatch/search/query.h b/searchcore/src/vespa/searchcore/fdispatch/search/query.h index 6aede1b3c13..171a68de9ab 100644 --- a/searchcore/src/vespa/searchcore/fdispatch/search/query.h +++ b/searchcore/src/vespa/searchcore/fdispatch/search/query.h @@ -1,6 +1,4 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// Copyright (C) 1999-2003 Fast Search & Transfer ASA -// Copyright (C) 2003 Overture Services Norway AS #pragma once diff --git a/searchcore/src/vespa/searchcore/fdispatch/search/querycacheutil.h b/searchcore/src/vespa/searchcore/fdispatch/search/querycacheutil.h index c9871475e71..6fdca46530c 100644 --- a/searchcore/src/vespa/searchcore/fdispatch/search/querycacheutil.h +++ b/searchcore/src/vespa/searchcore/fdispatch/search/querycacheutil.h @@ -1,6 +1,4 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// Copyright (C) 1998-2003 Fast Search & Transfer ASA -// Copyright (C) 2003 Overture Services Norway AS #pragma once diff --git a/searchcore/src/vespa/searchcore/fdispatch/search/rowstate.cpp b/searchcore/src/vespa/searchcore/fdispatch/search/rowstate.cpp index f90b14ed0f0..c4b0319e6cb 100644 --- a/searchcore/src/vespa/searchcore/fdispatch/search/rowstate.cpp +++ b/searchcore/src/vespa/searchcore/fdispatch/search/rowstate.cpp @@ -1,6 +1,4 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// Copyright (C) 1998-2003 Fast Search & Transfer ASA -// Copyright (C) 2003 Overture Services Norway AS #include <vespa/searchcore/fdispatch/search/rowstate.h> diff --git a/searchcore/src/vespa/searchcore/fdispatch/search/rowstate.h b/searchcore/src/vespa/searchcore/fdispatch/search/rowstate.h index 46dee3518d1..07bc769cfdd 100644 --- a/searchcore/src/vespa/searchcore/fdispatch/search/rowstate.h +++ b/searchcore/src/vespa/searchcore/fdispatch/search/rowstate.h @@ -1,6 +1,4 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// Copyright (C) 1998-2003 Fast Search & Transfer ASA -// Copyright (C) 2003 Overture Services Norway AS #pragma once diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_vector_explorer.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attribute_vector_explorer.cpp index a90b60fea4e..25ab7e27387 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_vector_explorer.cpp +++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_vector_explorer.cpp @@ -4,6 +4,7 @@ #include <vespa/searchlib/attribute/enumstorebase.h> #include <vespa/searchlib/attribute/multi_value_mapping.h> #include <vespa/searchlib/attribute/attributevector.h> +#include <vespa/searchlib/attribute/ipostinglistattributebase.h> #include <vespa/vespalib/data/slime/cursor.h> using search::attribute::Status; @@ -13,6 +14,7 @@ using search::AttributeVector; using search::EnumStoreBase; using search::MemoryUsage; using search::attribute::MultiValueMappingBase; +using search::attribute::IPostingListAttributeBase; using namespace vespalib::slime; namespace proton { @@ -87,7 +89,6 @@ convertMultiValueToSlime(const MultiValueMappingBase &multiValue, Cursor &object convertMemoryUsageToSlime(multiValue.getMemoryUsage(), object.setObject("memoryUsage")); } - void convertChangeVectorToSlime(const AttributeVector &v, Cursor &object) { @@ -95,6 +96,12 @@ convertChangeVectorToSlime(const AttributeVector &v, Cursor &object) convertMemoryUsageToSlime(usage, object); } +void +convertPostingBaseToSlime(const IPostingListAttributeBase &postingBase, Cursor &object) +{ + convertMemoryUsageToSlime(postingBase.getMemoryUsage(), object.setObject("memoryUsage")); +} + } AttributeVectorExplorer::AttributeVectorExplorer(ExclusiveAttributeReadAccessor::UP attribute) @@ -121,6 +128,10 @@ AttributeVectorExplorer::get_state(const vespalib::slime::Inserter &inserter, bo if (multiValue) { convertMultiValueToSlime(*multiValue, object.setObject("multiValue")); } + const IPostingListAttributeBase *postingBase = attr.getIPostingListAttributeBase(); + if (postingBase) { + convertPostingBaseToSlime(*postingBase, object.setObject("postingList")); + } convertChangeVectorToSlime(attr, object.setObject("changeVector")); object.setLong("committedDocIdLimit", attr.getCommittedDocIdLimit()); object.setLong("createSerialNum", attr.getCreateSerialNum()); diff --git a/searchcore/src/vespa/searchcore/proton/matching/match_master.cpp b/searchcore/src/vespa/searchcore/proton/matching/match_master.cpp index 0b5287fb094..370c4b930e1 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/match_master.cpp +++ b/searchcore/src/vespa/searchcore/proton/matching/match_master.cpp @@ -106,11 +106,15 @@ MatchMaster::getFeatureSet(const MatchToolsFactory &matchToolsFactory, const std::vector<uint32_t> &docs, bool summaryFeatures) { MatchTools::UP matchTools = matchToolsFactory.createMatchTools(); - RankProgram::UP rankProgram = summaryFeatures ? matchTools->summary_program() : - matchTools->dump_program(); + if (summaryFeatures) { + matchTools->setup_summary(); + } else { + matchTools->setup_dump(); + } + RankProgram &rankProgram = matchTools->rank_program(); std::vector<vespalib::string> featureNames; - FeatureResolver resolver(rankProgram->get_seeds()); + FeatureResolver resolver(rankProgram.get_seeds()); featureNames.reserve(resolver.num_features()); for (size_t i = 0; i < resolver.num_features(); ++i) { featureNames.emplace_back(resolver.name_of(i)); @@ -121,12 +125,12 @@ MatchMaster::getFeatureSet(const MatchToolsFactory &matchToolsFactory, } FeatureSet &fs = *retval.get(); - SearchIterator::UP search = matchTools->createSearch(rankProgram->match_data()); - search->initRange(docs.front(), docs.back()+1); + SearchIterator &search = matchTools->search(); + search.initRange(docs.front(), docs.back()+1); for (uint32_t i = 0; i < docs.size(); ++i) { - if (search->seek(docs[i])) { - uint32_t docId = search->getDocId(); - search->unpack(docId); + if (search.seek(docs[i])) { + uint32_t docId = search.getDocId(); + search.unpack(docId); search::feature_t * f = fs.getFeaturesByIndex( fs.addDocId(docId)); for (uint32_t j = 0; j < featureNames.size(); ++j) { diff --git a/searchcore/src/vespa/searchcore/proton/matching/match_thread.cpp b/searchcore/src/vespa/searchcore/proton/matching/match_thread.cpp index 3a9f59680f9..bd6a06ac4f1 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/match_thread.cpp +++ b/searchcore/src/vespa/searchcore/proton/matching/match_thread.cpp @@ -40,39 +40,22 @@ struct WaitTimer { } }; -class FastSeekWrapper -{ -private: - typedef search::queryeval::SearchIterator SearchIterator; -public: - FastSeekWrapper(SearchIterator::UP iterator) - { - reset(iterator.release()); - } - void initRange(uint32_t begin_id, uint32_t end_id) { - _search->initRange(begin_id, end_id); - } - uint32_t seekFirst(uint32_t docId) { - return _search->seekFirst(docId); - } - uint32_t seekNext(uint32_t docId) { - return _search->seekFast(docId); +// seek_next maps to SearchIterator::seekNext +struct SimpleStrategy { + static uint32_t seek_next(SearchIterator &search, uint32_t docid) { + return search.seekNext(docid); } - vespalib::string asString() const { - return _search->asString(); - } - void unpack(uint32_t docId) { - _search->unpack(docId); - } - void reset(SearchIterator * search) { - _search.reset(&dynamic_cast<OptimizedAndNotForBlackListing &>(*search)); +}; + +// seek_next maps to OptimizedAndNotForBlackListing::seekFast +struct FastBlackListingStrategy { + static bool can_use(bool do_rank, bool do_limit, SearchIterator &search) { + return (!do_rank && !do_limit && + (dynamic_cast<OptimizedAndNotForBlackListing *>(&search) != nullptr)); } - OptimizedAndNotForBlackListing * release() { - return _search.release(); + static uint32_t seek_next(SearchIterator &search, uint32_t docid) { + return static_cast<OptimizedAndNotForBlackListing &>(search).seekFast(docid); } - FastSeekWrapper * operator ->() { return this; } -private: - std::unique_ptr<OptimizedAndNotForBlackListing> _search; }; LazyValue get_score_feature(const RankProgram &rankProgram) { @@ -85,17 +68,17 @@ LazyValue get_score_feature(const RankProgram &rankProgram) { //----------------------------------------------------------------------------- -MatchThread::Context::Context(double rankDropLimit, MatchTools & matchTools, RankProgram & ranking, HitCollector & hits, - uint32_t num_threads) : - matches(0), - _matches_limit(matchTools.match_limiter().sample_hits_per_thread(num_threads)), - _score_feature(get_score_feature(ranking)), - _ranking(ranking), - _rankDropLimit(rankDropLimit), - _hits(hits), - _softDoom(matchTools.getSoftDoom()), - _limiter(matchTools.match_limiter()) -{ } +MatchThread::Context::Context(double rankDropLimit, MatchTools &tools, HitCollector &hits, + uint32_t num_threads) + : matches(0), + _matches_limit(tools.match_limiter().sample_hits_per_thread(num_threads)), + _score_feature(get_score_feature(tools.rank_program())), + _ranking(tools.rank_program()), + _rankDropLimit(rankDropLimit), + _hits(hits), + _softDoom(tools.getSoftDoom()) +{ +} void MatchThread::Context::rankHit(uint32_t docId) { @@ -122,27 +105,27 @@ MatchThread::estimate_match_frequency(uint32_t matches, uint32_t searchedSoFar) return match_freq; } -template <typename IteratorT> -void -MatchThread::maybe_limit(MaybeMatchPhaseLimiter & limiter, IteratorT & search, uint32_t matches, uint32_t docId, uint32_t endId) +SearchIterator * +MatchThread::maybe_limit(MatchTools &tools, uint32_t matches, uint32_t docId, uint32_t endId) { const uint32_t local_todo = (endId - docId - 1); const size_t searchedSoFar = (scheduler.total_size(thread_id) - local_todo); double match_freq = estimate_match_frequency(matches, searchedSoFar); const size_t global_todo = scheduler.unassigned_size(); - search = limiter.maybe_limit(std::move(search), match_freq, matchParams.numDocs); + { + auto search = tools.borrow_search(); + search = tools.match_limiter().maybe_limit(std::move(search), match_freq, matchParams.numDocs); + tools.give_back_search(std::move(search)); + if (tools.match_limiter().was_limited()) { + tools.tag_search_as_changed(); + } + } size_t left = local_todo + (global_todo / num_threads); - limiter.updateDocIdSpaceEstimate(searchedSoFar, left); + tools.match_limiter().updateDocIdSpaceEstimate(searchedSoFar, left); LOG(debug, "Limit=%d has been reached at docid=%d which is after %zu docs.", matches, docId, (scheduler.total_size(thread_id) - local_todo)); - LOG(debug, "SearchIterator after limiter: %s", search->asString().c_str()); -} - -template <> -void -MatchThread::maybe_limit(MaybeMatchPhaseLimiter &, FastSeekWrapper &, uint32_t, uint32_t, uint32_t) -{ - abort(); // We cannot replace the iterator if we inline the loop. + LOG(debug, "SearchIterator after limiter: %s", tools.search().asString().c_str()); + return &tools.search(); } bool @@ -156,10 +139,11 @@ MatchThread::try_share(DocidRange &docid_range, uint32_t next_docid) { return false; } -template <typename IteratorT, bool do_rank, bool do_limit, bool do_share_work> +template <typename Strategy, bool do_rank, bool do_limit, bool do_share_work> bool -MatchThread::inner_match_loop(Context & context, IteratorT & search, DocidRange docid_range) +MatchThread::inner_match_loop(Context &context, MatchTools &tools, DocidRange docid_range) { + SearchIterator *search = &tools.search(); search->initRange(docid_range.begin, docid_range.end); uint32_t docId = search->seekFirst(docid_range.begin); while ((docId < docid_range.end) && !context.atSoftDoom()) { @@ -171,30 +155,29 @@ MatchThread::inner_match_loop(Context & context, IteratorT & search, DocidRange } context.matches++; if (do_limit && context.isAtLimit()) { - maybe_limit(context.limiter(), search, context.matches, docId, docid_range.end); + search = maybe_limit(tools, context.matches, docId, docid_range.end); docId = search->seekFirst(docId + 1); } else if (do_share_work && any_idle() && try_share(docid_range, docId + 1)) { search->initRange(docid_range.begin, docid_range.end); docId = search->seekFirst(docid_range.begin); } else { - docId = search->seekNext(docId + 1); + docId = Strategy::seek_next(*search, docId + 1); } } return (docId < docid_range.end); } -template <typename IteratorT, bool do_rank, bool do_limit, bool do_share_work> +template <typename Strategy, bool do_rank, bool do_limit, bool do_share_work> void -MatchThread::match_loop(MatchTools &matchTools, IteratorT search, - RankProgram &ranking, HitCollector &hits) +MatchThread::match_loop(MatchTools &tools, HitCollector &hits) { bool softDoomed = false; - Context context(matchParams.rankDropLimit, matchTools, ranking, hits, num_threads); + Context context(matchParams.rankDropLimit, tools, hits, num_threads); for (DocidRange docid_range = scheduler.first_range(thread_id); !docid_range.empty() && ! softDoomed; docid_range = scheduler.next_range(thread_id)) { - softDoomed = inner_match_loop<IteratorT, do_rank, do_limit, do_share_work>(context, search, docid_range); + softDoomed = inner_match_loop<Strategy, do_rank, do_limit, do_share_work>(context, tools, docid_range); } uint32_t matches = context.matches; if (do_limit && context.isBelowLimit()) { @@ -202,7 +185,7 @@ MatchThread::match_loop(MatchTools &matchTools, IteratorT search, LOG(debug, "Limit not reached (had %d) at docid=%d which is after %zu docs.", matches, scheduler.total_span(thread_id).end, searchedSoFar); estimate_match_frequency(matches, searchedSoFar); - context.limiter().updateDocIdSpaceEstimate(searchedSoFar, 0); + tools.match_limiter().updateDocIdSpaceEstimate(searchedSoFar, 0); } thread_stats.docsMatched(matches); thread_stats.softDoomed(softDoomed); @@ -211,68 +194,77 @@ MatchThread::match_loop(MatchTools &matchTools, IteratorT search, } } -template <typename IteratorT, bool do_rank, bool do_limit> +//----------------------------------------------------------------------------- + +template <bool do_rank, bool do_limit, bool do_share> +void +MatchThread::match_loop_helper_rank_limit_share(MatchTools &tools, HitCollector &hits) +{ + if (FastBlackListingStrategy::can_use(do_rank, do_limit, tools.search())) { + match_loop<FastBlackListingStrategy, do_rank, do_limit, do_share>(tools, hits); + } else { + match_loop<SimpleStrategy, do_rank, do_limit, do_share>(tools, hits); + } +} + +template <bool do_rank, bool do_limit> void -MatchThread::match_loop_helper_2(MatchTools &matchTools, IteratorT search, - RankProgram &ranking, HitCollector &hits) +MatchThread::match_loop_helper_rank_limit(MatchTools &tools, HitCollector &hits) { if (idle_observer.is_always_zero()) { - match_loop<IteratorT, do_rank, do_limit, false>(matchTools, std::move(search), ranking, hits); + match_loop_helper_rank_limit_share<do_rank, do_limit, false>(tools, hits); + } else { + match_loop_helper_rank_limit_share<do_rank, do_limit, true>(tools, hits); + } +} + +template <bool do_rank> +void +MatchThread::match_loop_helper_rank(MatchTools &tools, HitCollector &hits) +{ + if (tools.match_limiter().is_enabled()) { + match_loop_helper_rank_limit<do_rank, true>(tools, hits); } else { - match_loop<IteratorT, do_rank, do_limit, true>(matchTools, std::move(search), ranking, hits); + match_loop_helper_rank_limit<do_rank, false>(tools, hits); } } -template <typename IteratorT, bool do_rank> void -MatchThread::match_loop_helper(MatchTools &matchTools, IteratorT search, - RankProgram &ranking, HitCollector &hits) +MatchThread::match_loop_helper(MatchTools &tools, HitCollector &hits) { - if (matchTools.match_limiter().is_enabled()) { - match_loop_helper_2<IteratorT, do_rank, true>(matchTools, std::move(search), ranking, hits); + if (match_with_ranking) { + match_loop_helper_rank<true>(tools, hits); } else { - match_loop_helper_2<IteratorT, do_rank, false>(matchTools, std::move(search), ranking, hits); + match_loop_helper_rank<false>(tools, hits); } } //----------------------------------------------------------------------------- search::ResultSet::UP -MatchThread::findMatches(MatchTools &matchTools) +MatchThread::findMatches(MatchTools &tools) { - RankProgram::UP ranking = matchTools.first_phase_program(); - SearchIterator::UP search = matchTools.createSearch(ranking->match_data()); + tools.setup_first_phase(); if (isFirstThread()) { - LOG(spam, "SearchIterator: %s", search->asString().c_str()); + LOG(spam, "SearchIterator: %s", tools.search().asString().c_str()); } - search = search::queryeval::MultiBitVectorIteratorBase::optimize(std::move(search)); + tools.give_back_search(search::queryeval::MultiBitVectorIteratorBase::optimize(tools.borrow_search())); if (isFirstThread()) { - LOG(debug, "SearchIterator after MultiBitVectorIteratorBase::optimize(): %s", search->asString().c_str()); + LOG(debug, "SearchIterator after MultiBitVectorIteratorBase::optimize(): %s", tools.search().asString().c_str()); } HitCollector hits(matchParams.numDocs, matchParams.arraySize, matchParams.heapSize); - if (match_with_ranking) { - match_loop_helper<SearchIterator::UP, true>(matchTools, std::move(search), *ranking, hits); - } else { - if ((dynamic_cast<const OptimizedAndNotForBlackListing *>(search.get()) != 0) && - ! matchTools.match_limiter().is_enabled()) // We cannot replace the iterator if we inline the loop. - { - match_loop_helper_2<FastSeekWrapper, false, false>(matchTools, FastSeekWrapper(std::move(search)), *ranking, hits); - } else { - match_loop_helper<SearchIterator::UP, false>(matchTools, std::move(search), *ranking, hits); - } - } - if (matchTools.has_second_phase_rank()) { + match_loop_helper(tools, hits); + if (tools.has_second_phase_rank()) { { // 2nd phase ranking - ranking = matchTools.second_phase_program(); - search = matchTools.createSearch(ranking->match_data()); + tools.setup_second_phase(); DocidRange docid_range = scheduler.total_span(thread_id); - search->initRange(docid_range.begin, docid_range.end); + tools.search().initRange(docid_range.begin, docid_range.end); auto sorted_scores = hits.getSortedHeapScores(); WaitTimer select_best_timer(wait_time_s); size_t useHits = communicator.selectBest(sorted_scores); select_best_timer.done(); - DocumentScorer scorer(*ranking, *search); - uint32_t reRanked = hits.reRank(scorer, matchTools.getHardDoom().doom() ? 0 : useHits); + DocumentScorer scorer(tools.rank_program(), tools.search()); + uint32_t reRanked = hits.reRank(scorer, tools.getHardDoom().doom() ? 0 : useHits); thread_stats.docsReRanked(reRanked); } { // rank scaling diff --git a/searchcore/src/vespa/searchcore/proton/matching/match_thread.h b/searchcore/src/vespa/searchcore/proton/matching/match_thread.h index a279e83f032..b08323fa099 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/match_thread.h +++ b/searchcore/src/vespa/searchcore/proton/matching/match_thread.h @@ -51,46 +51,41 @@ private: class Context { public: - Context(double rankDropLimit, MatchTools &matchTools, RankProgram & ranking, HitCollector & hits, + Context(double rankDropLimit, MatchTools &tools, HitCollector &hits, uint32_t num_threads) __attribute__((noinline)); void rankHit(uint32_t docId); void addHit(uint32_t docId) { _hits.addHit(docId, search::zero_rank_value); } bool isBelowLimit() const { return matches < _matches_limit; } bool isAtLimit() const { return matches == _matches_limit; } bool atSoftDoom() const { return _softDoom.doom(); } - MaybeMatchPhaseLimiter & limiter() { return _limiter; } - uint32_t matches; + uint32_t matches; private: - uint32_t _matches_limit; - LazyValue _score_feature; - RankProgram & _ranking; - double _rankDropLimit; - HitCollector & _hits; - const Doom & _softDoom; - MaybeMatchPhaseLimiter & _limiter; + uint32_t _matches_limit; + LazyValue _score_feature; + RankProgram &_ranking; + double _rankDropLimit; + HitCollector &_hits; + const Doom &_softDoom; }; double estimate_match_frequency(uint32_t matches, uint32_t searchedSoFar) __attribute__((noinline)); - - template <typename IteratorT> - void maybe_limit(MaybeMatchPhaseLimiter & limiter, IteratorT & search, uint32_t matches, uint32_t docId, uint32_t endId) __attribute__((noinline)); + SearchIterator *maybe_limit(MatchTools &tools, uint32_t matches, uint32_t docId, uint32_t endId) __attribute__((noinline)); bool any_idle() const { return (idle_observer.get() > 0); } bool try_share(DocidRange &docid_range, uint32_t next_docid) __attribute__((noinline)); - template <typename IteratorT, bool do_rank, bool do_limit, bool do_share_work> - bool inner_match_loop(Context & params, IteratorT & search, DocidRange docid_range) __attribute__((noinline)); - - template <typename IteratorT, bool do_rank, bool do_limit, bool do_share_work> - void match_loop(MatchTools &matchTools, IteratorT search, RankProgram &ranking, HitCollector &hits) __attribute__((noinline)); + template <typename Strategy, bool do_rank, bool do_limit, bool do_share_work> + bool inner_match_loop(Context &context, MatchTools &tools, DocidRange docid_range) __attribute__((noinline)); - template <typename IteratorT, bool do_rank, bool do_limit> - void match_loop_helper_2(MatchTools &matchTools, IteratorT search, RankProgram &ranking, HitCollector &hits); + template <typename Strategy, bool do_rank, bool do_limit, bool do_share_work> + void match_loop(MatchTools &tools, HitCollector &hits) __attribute__((noinline)); - template <typename IteratorT, bool do_rank> - void match_loop_helper(MatchTools &matchTools, IteratorT search, RankProgram &ranking, HitCollector &hits); + template <bool do_rank, bool do_limit, bool do_share> void match_loop_helper_rank_limit_share(MatchTools &tools, HitCollector &hits); + template <bool do_rank, bool do_limit> void match_loop_helper_rank_limit(MatchTools &tools, HitCollector &hits); + template <bool do_rank> void match_loop_helper_rank(MatchTools &tools, HitCollector &hits); + void match_loop_helper(MatchTools &tools, HitCollector &hits); - search::ResultSet::UP findMatches(MatchTools &matchTools); + search::ResultSet::UP findMatches(MatchTools &tools); void processResult(const Doom & hardDoom, search::ResultSet::UP result, ResultProcessor::Context &context); diff --git a/searchcore/src/vespa/searchcore/proton/matching/match_tools.cpp b/searchcore/src/vespa/searchcore/proton/matching/match_tools.cpp index 1e429616e7e..9044cb79220 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/match_tools.cpp +++ b/searchcore/src/vespa/searchcore/proton/matching/match_tools.cpp @@ -19,33 +19,48 @@ namespace proton::matching { namespace { -size_t -tagMatchData(const HandleRecorder::HandleSet & handles, MatchData & md) +bool contains_all(const HandleRecorder::HandleSet &old_set, + const HandleRecorder::HandleSet &new_set) { - size_t ignored(0); - for (TermFieldHandle handle(0); handle < md.getNumTermFields(); handle++) { + for (TermFieldHandle handle: new_set) { + if (old_set.find(handle) == old_set.end()) { + return false; + } + } + return true; +} + +void tag_match_data(const HandleRecorder::HandleSet &handles, MatchData &match_data) { + for (TermFieldHandle handle = 0; handle < match_data.getNumTermFields(); ++handle) { if (handles.find(handle) == handles.end()) { - md.resolveTermField(handle)->tagAsNotNeeded(); - ignored++; + match_data.resolveTermField(handle)->tagAsNotNeeded(); } } - return ignored; } -search::fef::RankProgram::UP setup_program(search::fef::RankProgram::UP program, - const MatchDataLayout &mdl, - const QueryEnvironment &queryEnv, - const Properties &featureOverrides) +} // namespace proton::matching::<unnamed> + +void +MatchTools::setup(search::fef::RankProgram::UP rank_program, double termwise_limit) { + if (_search) { + _match_data->soft_reset(); + } + _rank_program = std::move(rank_program); HandleRecorder recorder; { HandleRecorder::Binder bind(recorder); - program->setup(mdl, queryEnv, featureOverrides); + _rank_program->setup(*_match_data, _queryEnv, _featureOverrides); + } + bool can_reuse_search = (_search && !_search_has_changed && + contains_all(_used_handles, recorder.getHandles())); + if (!can_reuse_search) { + tag_match_data(recorder.getHandles(), *_match_data); + _match_data->set_termwise_limit(termwise_limit); + _search = _query.createSearch(*_match_data); + _used_handles = recorder.getHandles(); + _search_has_changed = false; } - tagMatchData(recorder.getHandles(), program->match_data()); - return program; -} - } MatchTools::MatchTools(QueryLimiter & queryLimiter, @@ -65,39 +80,42 @@ MatchTools::MatchTools(QueryLimiter & queryLimiter, _queryEnv(queryEnv), _rankSetup(rankSetup), _featureOverrides(featureOverrides), - _mdl(mdl), - _handleRecorder() + _match_data(mdl.createMatchData()), + _rank_program(), + _search(), + _used_handles(), + _search_has_changed(false) { - HandleRecorder::Binder bind(_handleRecorder); } -MatchTools::~MatchTools() {} +MatchTools::~MatchTools() +{ +} -search::fef::RankProgram::UP -MatchTools::first_phase_program() const { - auto program = setup_program(_rankSetup.create_first_phase_program(), - _mdl, _queryEnv, _featureOverrides); - program->match_data().set_termwise_limit(TermwiseLimit::lookup(_queryEnv.getProperties(), - _rankSetup.get_termwise_limit())); - return program; +void +MatchTools::setup_first_phase() +{ + setup(_rankSetup.create_first_phase_program(), + TermwiseLimit::lookup(_queryEnv.getProperties(), + _rankSetup.get_termwise_limit())); } -search::fef::RankProgram::UP -MatchTools::second_phase_program() const { - return setup_program(_rankSetup.create_second_phase_program(), - _mdl, _queryEnv, _featureOverrides); +void +MatchTools::setup_second_phase() +{ + setup(_rankSetup.create_second_phase_program()); } -search::fef::RankProgram::UP -MatchTools::summary_program() const { - return setup_program(_rankSetup.create_summary_program(), - _mdl, _queryEnv, _featureOverrides); +void +MatchTools::setup_summary() +{ + setup(_rankSetup.create_summary_program()); } -search::fef::RankProgram::UP -MatchTools::dump_program() const { - return setup_program(_rankSetup.create_dump_program(), - _mdl, _queryEnv, _featureOverrides); +void +MatchTools::setup_dump() +{ + setup(_rankSetup.create_dump_program()); } //----------------------------------------------------------------------------- diff --git a/searchcore/src/vespa/searchcore/proton/matching/match_tools.h b/searchcore/src/vespa/searchcore/proton/matching/match_tools.h index b4f1f997d32..f47eda16cc1 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/match_tools.h +++ b/searchcore/src/vespa/searchcore/proton/matching/match_tools.h @@ -23,16 +23,20 @@ class MatchTools { private: using IRequestContext = search::queryeval::IRequestContext; - QueryLimiter & _queryLimiter; - const vespalib::Doom & _softDoom; - const vespalib::Doom & _hardDoom; - const Query & _query; - MaybeMatchPhaseLimiter & _match_limiter; - const QueryEnvironment & _queryEnv; - const search::fef::RankSetup & _rankSetup; - const search::fef::Properties & _featureOverrides; - search::fef::MatchDataLayout _mdl; - HandleRecorder _handleRecorder; + QueryLimiter &_queryLimiter; + const vespalib::Doom &_softDoom; + const vespalib::Doom &_hardDoom; + const Query &_query; + MaybeMatchPhaseLimiter &_match_limiter; + const QueryEnvironment &_queryEnv; + const search::fef::RankSetup &_rankSetup; + const search::fef::Properties &_featureOverrides; + search::fef::MatchData::UP _match_data; + search::fef::RankProgram::UP _rank_program; + search::queryeval::SearchIterator::UP _search; + HandleRecorder::HandleSet _used_handles; + bool _search_has_changed; + void setup(search::fef::RankProgram::UP, double termwise_limit = 1.0); public: typedef std::unique_ptr<MatchTools> UP; MatchTools(const MatchTools &) = delete; @@ -51,15 +55,17 @@ public: const vespalib::Doom &getHardDoom() const { return _hardDoom; } QueryLimiter & getQueryLimiter() { return _queryLimiter; } MaybeMatchPhaseLimiter &match_limiter() { return _match_limiter; } - search::queryeval::SearchIterator::UP - createSearch(search::fef::MatchData &matchData) const { - return _query.createSearch(matchData); - } bool has_second_phase_rank() const { return !_rankSetup.getSecondPhaseRank().empty(); } - search::fef::RankProgram::UP first_phase_program() const; - search::fef::RankProgram::UP second_phase_program() const; - search::fef::RankProgram::UP summary_program() const; - search::fef::RankProgram::UP dump_program() const; + const search::fef::MatchData &match_data() const { return *_match_data; } + search::fef::RankProgram &rank_program() { return *_rank_program; } + search::queryeval::SearchIterator &search() { return *_search; } + search::queryeval::SearchIterator::UP borrow_search() { return std::move(_search); } + void give_back_search(search::queryeval::SearchIterator::UP search_in) { _search = std::move(search_in); } + void tag_search_as_changed() { _search_has_changed = true; } + void setup_first_phase(); + void setup_second_phase(); + void setup_summary(); + void setup_dump(); }; class MatchToolsFactory : public vespalib::noncopyable diff --git a/searchcore/src/vespa/searchcore/proton/metrics/documentdb_tagged_metrics.cpp b/searchcore/src/vespa/searchcore/proton/metrics/documentdb_tagged_metrics.cpp index 73bd675a363..522951412b7 100644 --- a/searchcore/src/vespa/searchcore/proton/metrics/documentdb_tagged_metrics.cpp +++ b/searchcore/src/vespa/searchcore/proton/metrics/documentdb_tagged_metrics.cpp @@ -79,6 +79,7 @@ DocumentDBTaggedMetrics::AttributeMetrics::ResourceUsageMetrics::~ResourceUsageM DocumentDBTaggedMetrics::IndexMetrics::IndexMetrics(MetricSet *parent) : MetricSet("index", "", "Index metrics (memory and disk) for this document db", parent), + diskUsage("disk_usage", "", "Disk space usage in bytes", this), memoryUsage(this) { } diff --git a/searchcore/src/vespa/searchcore/proton/metrics/documentdb_tagged_metrics.h b/searchcore/src/vespa/searchcore/proton/metrics/documentdb_tagged_metrics.h index cbfbd58bc09..2b31de03b63 100644 --- a/searchcore/src/vespa/searchcore/proton/metrics/documentdb_tagged_metrics.h +++ b/searchcore/src/vespa/searchcore/proton/metrics/documentdb_tagged_metrics.h @@ -84,6 +84,7 @@ struct DocumentDBTaggedMetrics : metrics::MetricSet struct IndexMetrics : metrics::MetricSet { + metrics::LongValueMetric diskUsage; MemoryUsageMetrics memoryUsage; IndexMetrics(metrics::MetricSet *parent); diff --git a/searchcore/src/vespa/searchcore/proton/metrics/legacy_proton_metrics.cpp b/searchcore/src/vespa/searchcore/proton/metrics/legacy_proton_metrics.cpp index a1bbf0758ac..9b7015a9f61 100644 --- a/searchcore/src/vespa/searchcore/proton/metrics/legacy_proton_metrics.cpp +++ b/searchcore/src/vespa/searchcore/proton/metrics/legacy_proton_metrics.cpp @@ -19,7 +19,7 @@ LegacyProtonMetrics::LegacyProtonMetrics() matchExecutor("matchexecutor", this), summaryExecutor("summaryexecutor", this), memoryUsage("memoryusage", "logdefault", "Total tracked memory usage", this), - diskUsage("diskusage", "logdefault", "Total tracked disk usage", this), + diskUsage("diskusage", "logdefault", "Total tracked disk usage for disk indexes", this), docsInMemory("docsinmemory", "logdefault", "Total Number of documents in memory", this), numDocs("numdocs", "logdefault", "Total number of ready/indexed documents among all document dbs (equal as numindexeddocs)", this), numActiveDocs("numactivedocs", "logdefault", diff --git a/searchcore/src/vespa/searchcore/proton/reference/CMakeLists.txt b/searchcore/src/vespa/searchcore/proton/reference/CMakeLists.txt index fe2ca7a7a88..a98b095cc21 100644 --- a/searchcore/src/vespa/searchcore/proton/reference/CMakeLists.txt +++ b/searchcore/src/vespa/searchcore/proton/reference/CMakeLists.txt @@ -10,6 +10,7 @@ vespa_add_library(searchcore_reference STATIC gid_to_lid_change_registrator.cpp gid_to_lid_mapper.cpp gid_to_lid_mapper_factory.cpp + pending_notify_remove_done.cpp DEPENDS searchcore_attribute searchcore_documentmetastore diff --git a/searchcore/src/vespa/searchcore/proton/reference/dummy_gid_to_lid_change_handler.cpp b/searchcore/src/vespa/searchcore/proton/reference/dummy_gid_to_lid_change_handler.cpp index c3c516a51dc..8f70d697b9a 100644 --- a/searchcore/src/vespa/searchcore/proton/reference/dummy_gid_to_lid_change_handler.cpp +++ b/searchcore/src/vespa/searchcore/proton/reference/dummy_gid_to_lid_change_handler.cpp @@ -15,7 +15,7 @@ DummyGidToLidChangeHandler::~DummyGidToLidChangeHandler() } void -DummyGidToLidChangeHandler::notifyPut(GlobalId, uint32_t, SerialNum) +DummyGidToLidChangeHandler::notifyPutDone(GlobalId, uint32_t, SerialNum) { } diff --git a/searchcore/src/vespa/searchcore/proton/reference/dummy_gid_to_lid_change_handler.h b/searchcore/src/vespa/searchcore/proton/reference/dummy_gid_to_lid_change_handler.h index 24773924aaa..28eb281ec95 100644 --- a/searchcore/src/vespa/searchcore/proton/reference/dummy_gid_to_lid_change_handler.h +++ b/searchcore/src/vespa/searchcore/proton/reference/dummy_gid_to_lid_change_handler.h @@ -24,7 +24,7 @@ public: DummyGidToLidChangeHandler(); virtual ~DummyGidToLidChangeHandler(); - virtual void notifyPut(GlobalId gid, uint32_t lid, SerialNum serialNum) override; + virtual void notifyPutDone(GlobalId gid, uint32_t lid, SerialNum serialNum) override; virtual void notifyRemove(GlobalId gid, SerialNum serialNum) override; virtual void notifyRemoveDone(GlobalId gid, SerialNum serialNum) override; virtual void addListener(std::unique_ptr<IGidToLidChangeListener> listener) override; diff --git a/searchcore/src/vespa/searchcore/proton/reference/gid_to_lid_change_handler.cpp b/searchcore/src/vespa/searchcore/proton/reference/gid_to_lid_change_handler.cpp index d7b2c16867a..bd3574327bc 100644 --- a/searchcore/src/vespa/searchcore/proton/reference/gid_to_lid_change_handler.cpp +++ b/searchcore/src/vespa/searchcore/proton/reference/gid_to_lid_change_handler.cpp @@ -31,10 +31,10 @@ GidToLidChangeHandler::~GidToLidChangeHandler() } void -GidToLidChangeHandler::notifyPut(GlobalId gid, uint32_t lid) +GidToLidChangeHandler::notifyPutDone(GlobalId gid, uint32_t lid) { for (const auto &listener : _listeners) { - listener->notifyPut(gid, lid); + listener->notifyPutDone(gid, lid); } } @@ -47,25 +47,39 @@ GidToLidChangeHandler::notifyRemove(GlobalId gid) } void -GidToLidChangeHandler::notifyPut(GlobalId gid, uint32_t lid, SerialNum serialNum) +GidToLidChangeHandler::notifyPutDone(GlobalId gid, uint32_t lid, SerialNum serialNum) { lock_guard guard(_lock); auto itr = _pendingRemove.find(gid); if (itr != _pendingRemove.end()) { - assert(itr->second > serialNum); - return; // Document has already been removed later on + auto &entry = itr->second; + assert(entry.removeSerialNum != serialNum); + if (entry.removeSerialNum > serialNum) { + return; // Document has already been removed later on + } + assert(entry.putSerialNum != serialNum); + if (entry.putSerialNum > serialNum) { + return; // Document has already been put later on + } + entry.putSerialNum = serialNum; } - notifyPut(gid, lid); + notifyPutDone(gid, lid); } void GidToLidChangeHandler::notifyRemove(GlobalId gid, SerialNum serialNum) { lock_guard guard(_lock); - auto insRes = _pendingRemove.insert(std::make_pair(gid, serialNum)); + auto insRes = _pendingRemove.insert(std::make_pair(gid, PendingRemoveEntry(serialNum))); if (!insRes.second) { - assert(insRes.first->second < serialNum); - insRes.first->second = serialNum; + auto &entry = insRes.first->second; + assert(entry.removeSerialNum < serialNum); + assert(entry.putSerialNum < serialNum); + if (entry.removeSerialNum < entry.putSerialNum) { + notifyRemove(gid); + } + entry.removeSerialNum = serialNum; + ++entry.refCount; } else { notifyRemove(gid); } @@ -76,9 +90,13 @@ GidToLidChangeHandler::notifyRemoveDone(GlobalId gid, SerialNum serialNum) { lock_guard guard(_lock); auto itr = _pendingRemove.find(gid); - assert(itr != _pendingRemove.end() && itr->second >= serialNum); - if (itr->second == serialNum) { + assert(itr != _pendingRemove.end()); + auto &entry = itr->second; + assert(entry.removeSerialNum >= serialNum); + if (entry.refCount == 1) { _pendingRemove.erase(itr); + } else { + --entry.refCount; } } diff --git a/searchcore/src/vespa/searchcore/proton/reference/gid_to_lid_change_handler.h b/searchcore/src/vespa/searchcore/proton/reference/gid_to_lid_change_handler.h index 840cb61cc2a..264ece76eaa 100644 --- a/searchcore/src/vespa/searchcore/proton/reference/gid_to_lid_change_handler.h +++ b/searchcore/src/vespa/searchcore/proton/reference/gid_to_lid_change_handler.h @@ -22,18 +22,36 @@ class GidToLidChangeHandler : public std::enable_shared_from_this<GidToLidChange { using lock_guard = std::lock_guard<std::mutex>; using Listeners = std::vector<std::unique_ptr<IGidToLidChangeListener>>; + struct PendingRemoveEntry { + SerialNum removeSerialNum; + SerialNum putSerialNum; + uint32_t refCount; + + PendingRemoveEntry(SerialNum removeSerialNum_) + : removeSerialNum(removeSerialNum_), + putSerialNum(0), + refCount(1) + { + } + + PendingRemoveEntry() + : PendingRemoveEntry(0) + { + } + }; + std::mutex _lock; Listeners _listeners; bool _closed; - vespalib::hash_map<GlobalId, SerialNum, GlobalId::hash> _pendingRemove; + vespalib::hash_map<GlobalId, PendingRemoveEntry, GlobalId::hash> _pendingRemove; - void notifyPut(GlobalId gid, uint32_t lid); + void notifyPutDone(GlobalId gid, uint32_t lid); void notifyRemove(GlobalId gid); public: GidToLidChangeHandler(); virtual ~GidToLidChangeHandler(); - virtual void notifyPut(GlobalId gid, uint32_t lid, SerialNum serialNum) override; + virtual void notifyPutDone(GlobalId gid, uint32_t lid, SerialNum serialNum) override; virtual void notifyRemove(GlobalId gid, SerialNum serialNum) override; virtual void notifyRemoveDone(GlobalId gid, SerialNum serialNum) override; diff --git a/searchcore/src/vespa/searchcore/proton/reference/gid_to_lid_change_listener.cpp b/searchcore/src/vespa/searchcore/proton/reference/gid_to_lid_change_listener.cpp index 6a368997770..cb5c5de0f5a 100644 --- a/searchcore/src/vespa/searchcore/proton/reference/gid_to_lid_change_listener.cpp +++ b/searchcore/src/vespa/searchcore/proton/reference/gid_to_lid_change_listener.cpp @@ -26,7 +26,7 @@ GidToLidChangeListener::~GidToLidChangeListener() } void -GidToLidChangeListener::notifyPut(document::GlobalId gid, uint32_t lid) +GidToLidChangeListener::notifyPutDone(document::GlobalId gid, uint32_t lid) { std::promise<bool> promise; std::future<bool> future = promise.get_future(); diff --git a/searchcore/src/vespa/searchcore/proton/reference/gid_to_lid_change_listener.h b/searchcore/src/vespa/searchcore/proton/reference/gid_to_lid_change_listener.h index 35ff913d7af..0e9298e3e51 100644 --- a/searchcore/src/vespa/searchcore/proton/reference/gid_to_lid_change_listener.h +++ b/searchcore/src/vespa/searchcore/proton/reference/gid_to_lid_change_listener.h @@ -30,7 +30,7 @@ public: const vespalib::string &name, const vespalib::string &docTypeName); virtual ~GidToLidChangeListener(); - virtual void notifyPut(document::GlobalId gid, uint32_t lid) override; + virtual void notifyPutDone(document::GlobalId gid, uint32_t lid) override; virtual void notifyRemove(document::GlobalId gid) override; virtual void notifyRegistered() override; virtual const vespalib::string &getName() const override; diff --git a/searchcore/src/vespa/searchcore/proton/reference/i_gid_to_lid_change_handler.h b/searchcore/src/vespa/searchcore/proton/reference/i_gid_to_lid_change_handler.h index 53ade83ff32..7de837236e8 100644 --- a/searchcore/src/vespa/searchcore/proton/reference/i_gid_to_lid_change_handler.h +++ b/searchcore/src/vespa/searchcore/proton/reference/i_gid_to_lid_change_handler.h @@ -40,7 +40,7 @@ public: /** * Notify gid to lid mapping change. */ - virtual void notifyPut(GlobalId gid, uint32_t lid, SerialNum serialNum) = 0; + virtual void notifyPutDone(GlobalId gid, uint32_t lid, SerialNum serialNum) = 0; virtual void notifyRemove(GlobalId gid, SerialNum serialNum) = 0; virtual void notifyRemoveDone(GlobalId gid, SerialNum serialNum) = 0; }; diff --git a/searchcore/src/vespa/searchcore/proton/reference/i_gid_to_lid_change_listener.h b/searchcore/src/vespa/searchcore/proton/reference/i_gid_to_lid_change_listener.h index d02979e168f..ebae8d40611 100644 --- a/searchcore/src/vespa/searchcore/proton/reference/i_gid_to_lid_change_listener.h +++ b/searchcore/src/vespa/searchcore/proton/reference/i_gid_to_lid_change_listener.h @@ -17,7 +17,7 @@ class IGidToLidChangeListener { public: virtual ~IGidToLidChangeListener() { } - virtual void notifyPut(document::GlobalId gid, uint32_t lid) = 0; + virtual void notifyPutDone(document::GlobalId gid, uint32_t lid) = 0; virtual void notifyRemove(document::GlobalId gid) = 0; virtual void notifyRegistered() = 0; virtual const vespalib::string &getName() const = 0; diff --git a/searchcore/src/vespa/searchcore/proton/reference/pending_notify_remove_done.cpp b/searchcore/src/vespa/searchcore/proton/reference/pending_notify_remove_done.cpp new file mode 100644 index 00000000000..e806628bc02 --- /dev/null +++ b/searchcore/src/vespa/searchcore/proton/reference/pending_notify_remove_done.cpp @@ -0,0 +1,50 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "pending_notify_remove_done.h" +#include <vespa/searchcore/proton/reference/i_gid_to_lid_change_handler.h> +#include <cassert> + +namespace proton +{ + +PendingNotifyRemoveDone::PendingNotifyRemoveDone() + : _gidToLidChangeHandler(nullptr), + _gid(), + _serialNum(0), + _pending(false) +{ +} + +PendingNotifyRemoveDone::PendingNotifyRemoveDone(PendingNotifyRemoveDone &&rhs) + : _gidToLidChangeHandler(rhs._gidToLidChangeHandler), + _gid(rhs._gid), + _serialNum(rhs._serialNum), + _pending(rhs._pending) +{ + rhs._pending = false; +} + +PendingNotifyRemoveDone::~PendingNotifyRemoveDone() +{ + assert(!_pending); // Fail if notifyRemoveDone is still pending +} + +void +PendingNotifyRemoveDone::setup(IGidToLidChangeHandler &gidToLidChangeHandler, document::GlobalId gid, search::SerialNum serialNum) +{ + _gidToLidChangeHandler = &gidToLidChangeHandler; + _gid = gid; + _serialNum = serialNum; + _pending = true; +} + +void +PendingNotifyRemoveDone::invoke() +{ + if (_pending) { + _gidToLidChangeHandler->notifyRemoveDone(_gid, _serialNum); + _pending = false; + } +} + +} // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/reference/pending_notify_remove_done.h b/searchcore/src/vespa/searchcore/proton/reference/pending_notify_remove_done.h new file mode 100644 index 00000000000..95aad182b10 --- /dev/null +++ b/searchcore/src/vespa/searchcore/proton/reference/pending_notify_remove_done.h @@ -0,0 +1,35 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <vespa/document/base/globalid.h> +#include <vespa/searchlib/common/serialnum.h> + +namespace proton +{ + +class IGidToLidChangeHandler; + +/* + * Class used to keep track of a pending notifyRemoveDone() call to + * a gid to lid change handler. + */ +class PendingNotifyRemoveDone +{ + IGidToLidChangeHandler *_gidToLidChangeHandler; + document::GlobalId _gid; + search::SerialNum _serialNum; + bool _pending; + +public: + PendingNotifyRemoveDone(); + PendingNotifyRemoveDone(PendingNotifyRemoveDone &&rhs); + PendingNotifyRemoveDone(const PendingNotifyRemoveDone &rhs) = delete; + PendingNotifyRemoveDone &operator=(const PendingNotifyRemoveDone &rhs) = delete; + PendingNotifyRemoveDone &operator=(PendingNotifyRemoveDone &&rhs) = delete; + ~PendingNotifyRemoveDone(); + void setup(IGidToLidChangeHandler &gidToLidChangeHandler, document::GlobalId gid, search::SerialNum serialNum); + void invoke(); +}; + +} // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp b/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp index 16d4a47a0c5..8c89f2a36af 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp @@ -1060,6 +1060,7 @@ updateIndexMetrics(DocumentDBMetricsCollection &metrics, const search::SearchableStats &stats) { DocumentDBTaggedMetrics::IndexMetrics &indexMetrics = metrics.getTaggedMetrics().index; + indexMetrics.diskUsage.set(stats.sizeOnDisk()); indexMetrics.memoryUsage.update(stats.memoryUsage()); LegacyDocumentDBMetrics::IndexMetrics &legacyIndexMetrics = metrics.getLegacyMetrics().index; diff --git a/searchcore/src/vespa/searchcore/proton/server/putdonecontext.cpp b/searchcore/src/vespa/searchcore/proton/server/putdonecontext.cpp index f05d8bc0823..efb5a58dd2e 100644 --- a/searchcore/src/vespa/searchcore/proton/server/putdonecontext.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/putdonecontext.cpp @@ -31,7 +31,7 @@ PutDoneContext::~PutDoneContext() _docIdLimit->bumpUpLimit(_lid + 1); } if (_enableNotifyPut) { - _gidToLidChangeHandler.notifyPut(_gid, _lid, _serialNum); + _gidToLidChangeHandler.notifyPutDone(_gid, _lid, _serialNum); } } diff --git a/searchcore/src/vespa/searchcore/proton/server/removedonecontext.cpp b/searchcore/src/vespa/searchcore/proton/server/removedonecontext.cpp index 522b0aed617..627e8d9f627 100644 --- a/searchcore/src/vespa/searchcore/proton/server/removedonecontext.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/removedonecontext.cpp @@ -12,18 +12,12 @@ RemoveDoneContext::RemoveDoneContext(std::unique_ptr<FeedToken> token, PerDocTypeFeedMetrics &metrics, vespalib::Executor &executor, IDocumentMetaStore &documentMetaStore, - IGidToLidChangeHandler &gidToLidChangeHandler, - const document::GlobalId &gid, - uint32_t lid, - search::SerialNum serialNum, - bool enableNotifyRemoveDone) + PendingNotifyRemoveDone &&pendingNotifyRemoveDone, + uint32_t lid) : OperationDoneContext(std::move(token), opType, metrics), _executor(executor), _task(), - _gidToLidChangeHandler(gidToLidChangeHandler), - _gid(gid), - _serialNum(serialNum), - _enableNotifyRemoveDone(enableNotifyRemoveDone) + _pendingNotifyRemoveDone(std::move(pendingNotifyRemoveDone)) { if (lid != 0) { _task = std::make_unique<RemoveDoneTask>(documentMetaStore, lid); @@ -32,9 +26,7 @@ RemoveDoneContext::RemoveDoneContext(std::unique_ptr<FeedToken> token, RemoveDoneContext::~RemoveDoneContext() { - if (_enableNotifyRemoveDone) { - _gidToLidChangeHandler.notifyRemoveDone(_gid, _serialNum); - } + _pendingNotifyRemoveDone.invoke(); ack(); if (_task) { vespalib::Executor::Task::UP res = _executor.execute(std::move(_task)); diff --git a/searchcore/src/vespa/searchcore/proton/server/removedonecontext.h b/searchcore/src/vespa/searchcore/proton/server/removedonecontext.h index 9311a6d2b6e..c4fafb4e886 100644 --- a/searchcore/src/vespa/searchcore/proton/server/removedonecontext.h +++ b/searchcore/src/vespa/searchcore/proton/server/removedonecontext.h @@ -6,12 +6,12 @@ #include <vespa/vespalib/util/executor.h> #include <vespa/document/base/globalid.h> #include <vespa/searchlib/common/serialnum.h> +#include <vespa/searchcore/proton/reference/pending_notify_remove_done.h> namespace proton { class IDocumentMetaStore; -class IGidToLidChangeHandler; /** @@ -26,10 +26,7 @@ class RemoveDoneContext : public OperationDoneContext { vespalib::Executor &_executor; std::unique_ptr<vespalib::Executor::Task> _task; - IGidToLidChangeHandler &_gidToLidChangeHandler; - document::GlobalId _gid; - search::SerialNum _serialNum; - bool _enableNotifyRemoveDone; + PendingNotifyRemoveDone _pendingNotifyRemoveDone; public: RemoveDoneContext(std::unique_ptr<FeedToken> token, @@ -37,11 +34,8 @@ public: PerDocTypeFeedMetrics &metrics, vespalib::Executor &executor, IDocumentMetaStore &documentMetaStore, - IGidToLidChangeHandler &gidToLidChangeHandler, - const document::GlobalId &gid, - uint32_t lid, - search::SerialNum serialNum, - bool enableNotifyRemoveDone); + PendingNotifyRemoveDone &&pendingNotifyRemoveDone, + uint32_t lid); virtual ~RemoveDoneContext(); }; diff --git a/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.cpp b/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.cpp index 5f0fc3b8aa6..c35f942ca35 100644 --- a/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.cpp @@ -117,14 +117,11 @@ public: RemoveDoneContextForMove(std::unique_ptr<FeedToken> token, const FeedOperation::Type opType, PerDocTypeFeedMetrics &metrics, vespalib::Executor &executor, IDocumentMetaStore &documentMetaStore, - IGidToLidChangeHandler &gidToLidChangeHandler, - const document::GlobalId &gid, + PendingNotifyRemoveDone &&pendingNotifyRemoveDone, uint32_t lid, - SerialNum serialNum, - bool enableNotifyRemoveDone, IDestructorCallback::SP moveDoneCtx) - : RemoveDoneContext(std::move(token), opType, metrics, executor, documentMetaStore, gidToLidChangeHandler, gid, lid, serialNum, enableNotifyRemoveDone), - _moveDoneCtx(std::move(moveDoneCtx)) + : RemoveDoneContext(std::move(token), opType, metrics, executor, documentMetaStore, std::move(pendingNotifyRemoveDone) ,lid), + _moveDoneCtx(std::move(moveDoneCtx)) {} virtual ~RemoveDoneContextForMove() {} }; @@ -133,19 +130,16 @@ std::shared_ptr<RemoveDoneContext> createRemoveDoneContext(std::unique_ptr<FeedToken> token, const FeedOperation::Type opType, PerDocTypeFeedMetrics &metrics, vespalib::Executor &executor, IDocumentMetaStore &documentMetaStore, - IGidToLidChangeHandler &gidToLidChangeHandler, - const document::GlobalId &gid, + PendingNotifyRemoveDone &&pendingNotifyRemoveDone, uint32_t lid, - SerialNum serialNum, - bool enableNotifyRemoveDone, IDestructorCallback::SP moveDoneCtx) { if (moveDoneCtx) { return std::make_shared<RemoveDoneContextForMove> - (std::move(token), opType, metrics, executor, documentMetaStore, gidToLidChangeHandler, gid, lid, serialNum, enableNotifyRemoveDone, std::move(moveDoneCtx)); + (std::move(token), opType, metrics, executor, documentMetaStore, std::move(pendingNotifyRemoveDone), lid, std::move(moveDoneCtx)); } else { return std::make_shared<RemoveDoneContext> - (std::move(token), opType, metrics, executor, documentMetaStore, gidToLidChangeHandler, gid, lid, serialNum, enableNotifyRemoveDone); + (std::move(token), opType, metrics, executor, documentMetaStore, std::move(pendingNotifyRemoveDone), lid); } } @@ -298,7 +292,7 @@ StoreOnlyFeedView::internalPut(FeedToken::UP token, const PutOperation &putOp) putOp.getSubDbId(), putOp.getLid(), putOp.getPrevSubDbId(), putOp.getPrevLid(), _params._subDbId, doc->toString(true).size(), doc->toString(true).c_str()); - adjustMetaStore(putOp, docId); + PendingNotifyRemoveDone pendingNotifyRemoveDone = adjustMetaStore(putOp, docId); considerEarlyAck(token, putOp.getType()); bool docAlreadyExists = putOp.getValidPrevDbdId(_params._subDbId); @@ -315,8 +309,7 @@ StoreOnlyFeedView::internalPut(FeedToken::UP token, const PutOperation &putOp) } if (docAlreadyExists && putOp.changedDbdId()) { assert(!putOp.getValidDbdId(_params._subDbId)); - const document::GlobalId &gid = docId.getGlobalId(); - internalRemove(std::move(token), serialNum, gid, putOp.getPrevLid(), putOp.getType(), useDocumentMetaStore(serialNum), IDestructorCallback::SP()); + internalRemove(std::move(token), serialNum, std::move(pendingNotifyRemoveDone), putOp.getPrevLid(), putOp.getType(), IDestructorCallback::SP()); } if (token.get() != NULL) { token->ack(putOp.getType(), _params._metrics); @@ -399,10 +392,11 @@ void StoreOnlyFeedView::putSummary(SerialNum serialNum, Lid lid, Document::SP do })); #pragma GCC diagnostic pop } -void StoreOnlyFeedView::removeSummary(SerialNum serialNum, Lid lid) { +void StoreOnlyFeedView::removeSummary(SerialNum serialNum, Lid lid, OnWriteDoneType onDone) { _pendingLidTracker.produce(lid); summaryExecutor().execute( - makeLambdaTask([serialNum, lid, this] { + makeLambdaTask([serialNum, lid, onDone, this] { + (void) onDone; _summaryAdapter->remove(serialNum, lid); _pendingLidTracker.consume(lid); })); @@ -507,9 +501,7 @@ StoreOnlyFeedView::makeUpdatedDocument(SerialNum serialNum, Lid lid, DocumentUpd FeedToken *token = onWriteDone->getToken(); token->trace(1, "The update looks like : " + upd.toString(token->shouldTrace(2))); } - vespalib::nbostream os; - prevDoc->serialize(os); - newDoc = std::make_unique<Document>(*_repo, os); + newDoc = std::move(prevDoc); if (useDocumentStore(serialNum)) { LOG(spam, "Original document :\n%s", newDoc->toXml(" ").c_str()); LOG(spam, "Update\n%s", upd.toXml().c_str()); @@ -581,7 +573,7 @@ StoreOnlyFeedView::internalRemove(FeedToken::UP token, const RemoveOperation &rm _params._docTypeName.toString().c_str(), serialNum, docId.toString().c_str(), rmOp.getSubDbId(), rmOp.getLid(), rmOp.getPrevSubDbId(), rmOp.getPrevLid(), _params._subDbId); - adjustMetaStore(rmOp, docId); + PendingNotifyRemoveDone pendingNotifyRemoveDone = adjustMetaStore(rmOp, docId); considerEarlyAck(token, rmOp.getType()); if (rmOp.getValidDbdId(_params._subDbId)) { @@ -593,8 +585,7 @@ StoreOnlyFeedView::internalRemove(FeedToken::UP token, const RemoveOperation &rm if (rmOp.getValidPrevDbdId(_params._subDbId)) { if (rmOp.changedDbdId()) { assert(!rmOp.getValidDbdId(_params._subDbId)); - const document::GlobalId &gid = docId.getGlobalId(); - internalRemove(std::move(token), serialNum, gid, rmOp.getPrevLid(), rmOp.getType(), useDocumentMetaStore(serialNum), IDestructorCallback::SP()); + internalRemove(std::move(token), serialNum, std::move(pendingNotifyRemoveDone), rmOp.getPrevLid(), rmOp.getType(), IDestructorCallback::SP()); } } if (token.get() != NULL) { @@ -603,22 +594,23 @@ StoreOnlyFeedView::internalRemove(FeedToken::UP token, const RemoveOperation &rm } void -StoreOnlyFeedView::internalRemove(FeedToken::UP token, SerialNum serialNum, const document::GlobalId &gid, Lid lid, - FeedOperation::Type opType, bool enableNotifyRemoveDone, IDestructorCallback::SP moveDoneCtx) +StoreOnlyFeedView::internalRemove(FeedToken::UP token, SerialNum serialNum, PendingNotifyRemoveDone &&pendingNotifyRemoveDone, Lid lid, + FeedOperation::Type opType, IDestructorCallback::SP moveDoneCtx) { - removeSummary(serialNum, lid); bool explicitReuseLid = _lidReuseDelayer.delayReuse(lid); std::shared_ptr<RemoveDoneContext> onWriteDone; onWriteDone = createRemoveDoneContext(std::move(token), opType, _params._metrics, _writeService.master(), - _metaStore, _gidToLidChangeHandler, gid, (explicitReuseLid ? lid : 0u), serialNum, enableNotifyRemoveDone, moveDoneCtx); + _metaStore, std::move(pendingNotifyRemoveDone), (explicitReuseLid ? lid : 0u), moveDoneCtx); + removeSummary(serialNum, lid, onWriteDone); bool immediateCommit = _commitTimeTracker.needCommit(); removeAttributes(serialNum, lid, immediateCommit, onWriteDone); removeIndexedFields(serialNum, lid, immediateCommit, onWriteDone); } -void +PendingNotifyRemoveDone StoreOnlyFeedView::adjustMetaStore(const DocumentOperation &op, const DocumentId &docId) { + PendingNotifyRemoveDone pendingNotifyRemoveDone; const SerialNum serialNum = op.getSerialNum(); if (useDocumentMetaStore(serialNum)) { if (op.getValidDbdId(_params._subDbId)) { @@ -632,10 +624,12 @@ StoreOnlyFeedView::adjustMetaStore(const DocumentOperation &op, const DocumentId } } else if (op.getValidPrevDbdId(_params._subDbId)) { _gidToLidChangeHandler.notifyRemove(docId.getGlobalId(), serialNum); + pendingNotifyRemoveDone.setup(_gidToLidChangeHandler, docId.getGlobalId(), serialNum); removeMetaData(_metaStore, docId, op, _params._subDbType == SubDbType::REMOVED); } _metaStore.commit(serialNum, serialNum); } + return pendingNotifyRemoveDone; } void @@ -684,7 +678,7 @@ StoreOnlyFeedView::removeDocuments(const RemoveDocumentsOperation &op, bool remo } if (useDocumentStore(serialNum + 1)) { for (const auto &lid : lidsToRemove) { - removeSummary(serialNum, lid); + removeSummary(serialNum, lid, onWriteDone); } } return lidsToRemove.size(); @@ -750,7 +744,7 @@ StoreOnlyFeedView::handleMove(const MoveOperation &moveOp, IDestructorCallback:: moveOp.getSubDbId(), moveOp.getLid(), moveOp.getPrevSubDbId(), moveOp.getPrevLid(), _params._subDbId, doc->toString(true).size(), doc->toString(true).c_str()); - adjustMetaStore(moveOp, docId); + PendingNotifyRemoveDone pendingNotifyRemoveDone = adjustMetaStore(moveOp, docId); bool docAlreadyExists = moveOp.getValidPrevDbdId(_params._subDbId); if (moveOp.getValidDbdId(_params._subDbId)) { bool immediateCommit = _commitTimeTracker.needCommit(); @@ -765,9 +759,7 @@ StoreOnlyFeedView::handleMove(const MoveOperation &moveOp, IDestructorCallback:: putIndexedFields(serialNum, moveOp.getLid(), doc, immediateCommit, onWriteDone); } if (docAlreadyExists && moveOp.changedDbdId()) { - const document::GlobalId &gid = docId.getGlobalId(); - bool enableNotifyRemoveDone = useDocumentMetaStore(serialNum) && !moveOp.getValidDbdId(_params._subDbId); - internalRemove(FeedToken::UP(), serialNum, gid, moveOp.getPrevLid(), moveOp.getType(), enableNotifyRemoveDone, doneCtx); + internalRemove(FeedToken::UP(), serialNum, std::move(pendingNotifyRemoveDone), moveOp.getPrevLid(), moveOp.getType(), doneCtx); } } diff --git a/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.h b/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.h index 01a8122ed1e..fbc8888ac79 100644 --- a/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.h +++ b/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.h @@ -15,6 +15,7 @@ #include <vespa/searchcore/proton/documentmetastore/documentmetastorecontext.h> #include <vespa/searchcore/proton/feedoperation/feedoperation.h> #include <vespa/searchcore/proton/persistenceengine/resulthandler.h> +#include <vespa/searchcore/proton/reference/pending_notify_remove_done.h> #include <vespa/searchcorespi/index/ithreadingservice.h> #include <vespa/searchlib/query/base.h> #include <vespa/vespalib/util/threadstackexecutorbase.h> @@ -157,7 +158,7 @@ private: } void putSummary(SerialNum serialNum, Lid lid, FutureStream doc, OnOperationDoneType onDone); void putSummary(SerialNum serialNum, Lid lid, Document::SP doc, OnOperationDoneType onDone); - void removeSummary(SerialNum serialNum, Lid lid); + void removeSummary(SerialNum serialNum, Lid lid, OnWriteDoneType onDone); void heartBeatSummary(SerialNum serialNum); @@ -168,7 +169,7 @@ private: return replaySerialNum > _params._flushedDocumentMetaStoreSerialNum; } - void adjustMetaStore(const DocumentOperation &op, const document::DocumentId &docId); + PendingNotifyRemoveDone adjustMetaStore(const DocumentOperation &op, const document::DocumentId &docId); void internalPut(FeedTokenUP token, const PutOperation &putOp); void internalUpdate(FeedTokenUP token, const UpdateOperation &updOp); @@ -180,8 +181,8 @@ private: size_t removeDocuments(const RemoveDocumentsOperation &op, bool remove_index_and_attribute_fields, bool immediateCommit); - void internalRemove(FeedTokenUP token, SerialNum serialNum, const document::GlobalId &gid, Lid lid, - FeedOperation::Type opType, bool enableNotifyRemoveDone, std::shared_ptr<search::IDestructorCallback> moveDoneCtx); + void internalRemove(FeedTokenUP token, SerialNum serialNum, PendingNotifyRemoveDone &&pendingNotifyRemoveDone, Lid lid, + FeedOperation::Type opType, std::shared_ptr<search::IDestructorCallback> moveDoneCtx); // Ack token early if visibility delay is nonzero void considerEarlyAck(FeedTokenUP &token, FeedOperation::Type opType); diff --git a/searchcore/src/vespa/searchcore/proton/test/mock_gid_to_lid_change_handler.h b/searchcore/src/vespa/searchcore/proton/test/mock_gid_to_lid_change_handler.h index 41efb55e61c..c1c2e2909d9 100644 --- a/searchcore/src/vespa/searchcore/proton/test/mock_gid_to_lid_change_handler.h +++ b/searchcore/src/vespa/searchcore/proton/test/mock_gid_to_lid_change_handler.h @@ -44,7 +44,7 @@ public: _removes.emplace_back(docTypeName, keepNames); } - virtual void notifyPut(document::GlobalId, uint32_t, SerialNum) override { } + virtual void notifyPutDone(document::GlobalId, uint32_t, SerialNum) override { } virtual void notifyRemove(document::GlobalId, SerialNum) override { } virtual void notifyRemoveDone(document::GlobalId, SerialNum) override { } diff --git a/searchcore/src/vespa/searchcore/util/autoptr.h b/searchcore/src/vespa/searchcore/util/autoptr.h index 07cdff94a4e..ba294e097e4 100644 --- a/searchcore/src/vespa/searchcore/util/autoptr.h +++ b/searchcore/src/vespa/searchcore/util/autoptr.h @@ -1,6 +1,4 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// Copyright (C) 2003 Fast Search & Transfer ASA -// Copyright (C) 2003 Overture Services Norway AS #pragma once diff --git a/searchcore/src/vespa/searchcore/util/log.h b/searchcore/src/vespa/searchcore/util/log.h index 54544c5a5a4..72aa525a421 100644 --- a/searchcore/src/vespa/searchcore/util/log.h +++ b/searchcore/src/vespa/searchcore/util/log.h @@ -1,6 +1,4 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// Copyright (C) 2000-2003 Fast Search & Transfer ASA -// Copyright (C) 2003 Overture Services Norway AS #pragma once diff --git a/searchcore/src/vespa/searchcore/util/stlishheap.h b/searchcore/src/vespa/searchcore/util/stlishheap.h index 8628814e7e4..3a04f8f2d39 100644 --- a/searchcore/src/vespa/searchcore/util/stlishheap.h +++ b/searchcore/src/vespa/searchcore/util/stlishheap.h @@ -1,6 +1,4 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// Copyright (C) 1998-2003 Fast Search & Transfer ASA -// Copyright (C) 2003 Overture Services Norway AS #pragma once diff --git a/searchlib/src/tests/attribute/attribute_test.cpp b/searchlib/src/tests/attribute/attribute_test.cpp index e13c419e68e..b406afa7e7b 100644 --- a/searchlib/src/tests/attribute/attribute_test.cpp +++ b/searchlib/src/tests/attribute/attribute_test.cpp @@ -459,6 +459,7 @@ void AttributeTest::testReload(const AttributePtr & a, const AttributePtr & b, c (*(static_cast<VectorType *>(a.get())), *(static_cast<VectorType *>(b.get()))); a->setCreateSerialNum(43u); EXPECT_TRUE( a->saveAs(b->getBaseFileName()) ); + a->commit(true); if (preciseEstimatedSize(*a)) { EXPECT_EQUAL(statSize(*b), a->getEstimatedSaveByteSize()); } else { diff --git a/searchlib/src/tests/features/item_raw_score/item_raw_score_test.cpp b/searchlib/src/tests/features/item_raw_score/item_raw_score_test.cpp index f407aea811b..a954710f153 100644 --- a/searchlib/src/tests/features/item_raw_score/item_raw_score_test.cpp +++ b/searchlib/src/tests/features/item_raw_score/item_raw_score_test.cpp @@ -67,13 +67,14 @@ struct SingleLabel : public Labels { struct RankFixture : BlueprintFactoryFixture, IndexFixture { QueryEnvironment queryEnv; RankSetup rankSetup; - RankProgram::UP rankProgram; MatchDataLayout mdl; + MatchData::UP match_data; + RankProgram::UP rankProgram; std::vector<TermFieldHandle> fooHandles; std::vector<TermFieldHandle> barHandles; RankFixture(size_t fooCnt, size_t barCnt, const Labels &labels) : queryEnv(&indexEnv), rankSetup(factory, indexEnv), - rankProgram(), mdl(), fooHandles(), barHandles() + mdl(), match_data(), rankProgram(), fooHandles(), barHandles() { for (size_t i = 0; i < fooCnt; ++i) { uint32_t fieldId = indexEnv.getFieldByName("foo")->id(); @@ -95,14 +96,15 @@ struct RankFixture : BlueprintFactoryFixture, IndexFixture { rankSetup.setFirstPhaseRank(featureName); rankSetup.setIgnoreDefaultRankFeatures(true); ASSERT_TRUE(rankSetup.compile()); + match_data = mdl.createMatchData(); rankProgram = rankSetup.create_first_phase_program(); - rankProgram->setup(mdl, queryEnv); + rankProgram->setup(*match_data, queryEnv); } feature_t getScore(uint32_t docId) { return Utils::getScoreFeature(*rankProgram, docId); } void setScore(TermFieldHandle handle, uint32_t docId, feature_t score) { - rankProgram->match_data().resolveTermField(handle)->setRawScore(docId, score); + match_data->resolveTermField(handle)->setRawScore(docId, score); } void setFooScore(uint32_t i, uint32_t docId, feature_t score) { ASSERT_LESS(i, fooHandles.size()); diff --git a/searchlib/src/tests/features/native_dot_product/native_dot_product_test.cpp b/searchlib/src/tests/features/native_dot_product/native_dot_product_test.cpp index 5ba7504f7d2..1da912ccb3a 100644 --- a/searchlib/src/tests/features/native_dot_product/native_dot_product_test.cpp +++ b/searchlib/src/tests/features/native_dot_product/native_dot_product_test.cpp @@ -72,14 +72,15 @@ std::vector<uint32_t> vec(uint32_t w1, uint32_t w2, uint32_t w3) { struct RankFixture : BlueprintFactoryFixture, IndexFixture { QueryEnvironment queryEnv; RankSetup rankSetup; - RankProgram::UP rankProgram; MatchDataLayout mdl; + MatchData::UP match_data; + RankProgram::UP rankProgram; std::vector<TermFieldHandle> fooHandles; std::vector<TermFieldHandle> barHandles; RankFixture(const std::vector<uint32_t> &fooWeights, const std::vector<uint32_t> &barWeights) : queryEnv(&indexEnv), rankSetup(factory, indexEnv), - rankProgram(), mdl(), fooHandles(), barHandles() + mdl(), match_data(), rankProgram(), fooHandles(), barHandles() { for (size_t i = 0; i < fooWeights.size(); ++i) { uint32_t fieldId = indexEnv.getFieldByName("foo")->id(); @@ -100,8 +101,9 @@ struct RankFixture : BlueprintFactoryFixture, IndexFixture { rankSetup.setFirstPhaseRank(featureName); rankSetup.setIgnoreDefaultRankFeatures(true); ASSERT_TRUE(rankSetup.compile()); + match_data = mdl.createMatchData(); rankProgram = rankSetup.create_first_phase_program(); - rankProgram->setup(mdl, queryEnv); + rankProgram->setup(*match_data, queryEnv); } feature_t getScore(uint32_t docId) { return Utils::getScoreFeature(*rankProgram, docId); @@ -110,15 +112,15 @@ struct RankFixture : BlueprintFactoryFixture, IndexFixture { ASSERT_LESS(i, fooHandles.size()); TermFieldMatchDataPosition pos; pos.setElementWeight(index_weight); - rankProgram->match_data().resolveTermField(fooHandles[i])->reset(docId); - rankProgram->match_data().resolveTermField(fooHandles[i])->appendPosition(pos); + match_data->resolveTermField(fooHandles[i])->reset(docId); + match_data->resolveTermField(fooHandles[i])->appendPosition(pos); } void setBarWeight(uint32_t i, uint32_t docId, int32_t index_weight) { ASSERT_LESS(i, barHandles.size()); TermFieldMatchDataPosition pos; pos.setElementWeight(index_weight); - rankProgram->match_data().resolveTermField(barHandles[i])->reset(docId); - rankProgram->match_data().resolveTermField(barHandles[i])->appendPosition(pos); + match_data->resolveTermField(barHandles[i])->reset(docId); + match_data->resolveTermField(barHandles[i])->appendPosition(pos); } }; diff --git a/searchlib/src/tests/features/raw_score/raw_score_test.cpp b/searchlib/src/tests/features/raw_score/raw_score_test.cpp index 52587c0d8f9..4aad6c6896e 100644 --- a/searchlib/src/tests/features/raw_score/raw_score_test.cpp +++ b/searchlib/src/tests/features/raw_score/raw_score_test.cpp @@ -45,13 +45,14 @@ struct FeatureDumpFixture : public IDumpFeatureVisitor { struct RankFixture : BlueprintFactoryFixture, IndexFixture { QueryEnvironment queryEnv; RankSetup rankSetup; - RankProgram::UP rankProgram; MatchDataLayout mdl; + MatchData::UP match_data; + RankProgram::UP rankProgram; std::vector<TermFieldHandle> fooHandles; std::vector<TermFieldHandle> barHandles; RankFixture(size_t fooCnt, size_t barCnt) : queryEnv(&indexEnv), rankSetup(factory, indexEnv), - rankProgram(), mdl(), fooHandles(), barHandles() + mdl(), match_data(), rankProgram(), fooHandles(), barHandles() { for (size_t i = 0; i < fooCnt; ++i) { uint32_t fieldId = indexEnv.getFieldByName("foo")->id(); @@ -70,14 +71,15 @@ struct RankFixture : BlueprintFactoryFixture, IndexFixture { rankSetup.setFirstPhaseRank(featureName); rankSetup.setIgnoreDefaultRankFeatures(true); ASSERT_TRUE(rankSetup.compile()); + match_data = mdl.createMatchData(); rankProgram = rankSetup.create_first_phase_program(); - rankProgram->setup(mdl, queryEnv); + rankProgram->setup(*match_data, queryEnv); } feature_t getScore(uint32_t docId) { return Utils::getScoreFeature(*rankProgram, docId); } void setScore(TermFieldHandle handle, uint32_t docId, feature_t score) { - rankProgram->match_data().resolveTermField(handle)->setRawScore(docId, score); + match_data->resolveTermField(handle)->setRawScore(docId, score); } void setFooScore(uint32_t i, uint32_t docId, feature_t score) { ASSERT_LESS(i, fooHandles.size()); diff --git a/searchlib/src/tests/features/subqueries/subqueries_test.cpp b/searchlib/src/tests/features/subqueries/subqueries_test.cpp index 37e43b4d2b0..d275cf134c6 100644 --- a/searchlib/src/tests/features/subqueries/subqueries_test.cpp +++ b/searchlib/src/tests/features/subqueries/subqueries_test.cpp @@ -43,22 +43,24 @@ struct FeatureDumpFixture : public IDumpFeatureVisitor { struct RankFixture : BlueprintFactoryFixture, IndexFixture { QueryEnvironment queryEnv; RankSetup rankSetup; - RankProgram::UP rankProgram; MatchDataLayout mdl; + MatchData::UP match_data; + RankProgram::UP rankProgram; std::vector<TermFieldHandle> fooHandles; std::vector<TermFieldHandle> barHandles; RankFixture(size_t fooCnt, size_t barCnt, std::string featureName = "subqueries(foo)") : queryEnv(&indexEnv), rankSetup(factory, indexEnv), - rankProgram(), mdl(), fooHandles(), barHandles() + mdl(), match_data(), rankProgram(), fooHandles(), barHandles() { fooHandles = addFields(fooCnt, indexEnv.getFieldByName("foo")->id()); barHandles = addFields(barCnt, indexEnv.getFieldByName("bar")->id()); rankSetup.setFirstPhaseRank(featureName); rankSetup.setIgnoreDefaultRankFeatures(true); ASSERT_TRUE(rankSetup.compile()); + match_data = mdl.createMatchData(); rankProgram = rankSetup.create_first_phase_program(); - rankProgram->setup(mdl, queryEnv); + rankProgram->setup(*match_data, queryEnv); } std::vector<TermFieldHandle> addFields(size_t count, uint32_t fieldId) { std::vector<TermFieldHandle> handles; @@ -75,7 +77,7 @@ struct RankFixture : BlueprintFactoryFixture, IndexFixture { } void setSubqueries(TermFieldHandle handle, uint32_t docId, uint64_t subqueries) { - rankProgram->match_data().resolveTermField(handle)->setSubqueries(docId, subqueries); + match_data->resolveTermField(handle)->setSubqueries(docId, subqueries); } void setFooSubqueries(uint32_t i, uint32_t docId, uint64_t subqueries) { ASSERT_LESS(i, fooHandles.size()); diff --git a/searchlib/src/tests/fef/featureoverride/featureoverride.cpp b/searchlib/src/tests/fef/featureoverride/featureoverride.cpp index 0ff7e0899e9..e55bd7081a1 100644 --- a/searchlib/src/tests/fef/featureoverride/featureoverride.cpp +++ b/searchlib/src/tests/fef/featureoverride/featureoverride.cpp @@ -141,7 +141,8 @@ TEST("test overrides") overrides.add("value(1,2,3).2", "6.0"); overrides.add("bogus(feature)", "10.0"); - rankProgram->setup(mdl, queryEnv, overrides); + MatchData::UP match_data = mdl.createMatchData(); + rankProgram->setup(*match_data, queryEnv, overrides); std::map<vespalib::string, feature_t> res = Utils::getAllFeatures(*rankProgram, 2); diff --git a/searchlib/src/tests/fef/object_passing/object_passing_test.cpp b/searchlib/src/tests/fef/object_passing/object_passing_test.cpp index 5c0f09a1203..3ae5932889a 100644 --- a/searchlib/src/tests/fef/object_passing/object_passing_test.cpp +++ b/searchlib/src/tests/fef/object_passing/object_passing_test.cpp @@ -7,6 +7,7 @@ #include <vespa/searchlib/fef/test/indexenvironment.h> #include <vespa/searchlib/fef/test/queryenvironment.h> #include <vespa/searchlib/fef/test/plugin/unbox.h> +#include <vespa/searchlib/fef/matchdatalayout.h> #include <vespa/searchlib/fef/rank_program.h> #include <vespa/searchlib/fef/verify_feature.h> #include <vespa/eval/eval/value_type.h> @@ -87,10 +88,11 @@ struct Fixture { return vespalib::eval::error_value; } MatchDataLayout mdl; + MatchData::UP md = mdl.createMatchData(); QueryEnvironment queryEnv(&indexEnv); Properties overrides; RankProgram program(resolver); - program.setup(mdl, queryEnv, overrides); + program.setup(*md, queryEnv, overrides); auto result = program.get_seeds(); EXPECT_EQUAL(1u, result.num_features()); EXPECT_TRUE(!result.is_object(0)); // verifies auto-unboxing diff --git a/searchlib/src/tests/fef/rank_program/rank_program_test.cpp b/searchlib/src/tests/fef/rank_program/rank_program_test.cpp index 8e23731e961..7e28178e5f7 100644 --- a/searchlib/src/tests/fef/rank_program/rank_program_test.cpp +++ b/searchlib/src/tests/fef/rank_program/rank_program_test.cpp @@ -6,6 +6,7 @@ #include <vespa/searchlib/features/rankingexpressionfeature.h> #include <vespa/searchlib/fef/blueprintfactory.h> #include <vespa/searchlib/fef/indexproperties.h> +#include <vespa/searchlib/fef/matchdatalayout.h> #include <vespa/searchlib/fef/test/indexenvironment.h> #include <vespa/searchlib/fef/test/queryenvironment.h> #include <vespa/searchlib/fef/test/plugin/sum.h> @@ -69,10 +70,11 @@ struct Fixture { IndexEnvironment indexEnv; BlueprintResolver::SP resolver; Properties overrides; + MatchData::UP match_data; RankProgram program; size_t track_cnt; Fixture() : factory(), indexEnv(), resolver(new BlueprintResolver(factory, indexEnv)), - overrides(), program(resolver), track_cnt(0) + overrides(), match_data(), program(resolver), track_cnt(0) { factory.addPrototype(Blueprint::SP(new BoxingBlueprint())); factory.addPrototype(Blueprint::SP(new DocidBlueprint())); @@ -107,7 +109,8 @@ struct Fixture { ASSERT_TRUE(resolver->compile()); MatchDataLayout mdl; QueryEnvironment queryEnv(&indexEnv); - program.setup(mdl, queryEnv, overrides); + match_data = mdl.createMatchData(); + program.setup(*match_data, queryEnv, overrides); return *this; } double get(uint32_t docid = default_docid) { diff --git a/searchlib/src/tests/fef/termfieldmodel/termfieldmodel_test.cpp b/searchlib/src/tests/fef/termfieldmodel/termfieldmodel_test.cpp index 57fb19b7b23..01c72497246 100644 --- a/searchlib/src/tests/fef/termfieldmodel/termfieldmodel_test.cpp +++ b/searchlib/src/tests/fef/termfieldmodel/termfieldmodel_test.cpp @@ -207,4 +207,39 @@ TEST("Access subqueries") { EXPECT_EQUAL(0ULL, state.f3->getSubqueries()); } +TEST("require that TermFieldMatchData can be tagged as needed or not") { + TermFieldMatchData tfmd; + tfmd.setFieldId(123); + EXPECT_EQUAL(tfmd.getFieldId(),123u); + EXPECT_TRUE(!tfmd.isNotNeeded()); + tfmd.tagAsNotNeeded(); + EXPECT_EQUAL(tfmd.getFieldId(),123u); + EXPECT_TRUE(tfmd.isNotNeeded()); + tfmd.tagAsNeeded(); + EXPECT_EQUAL(tfmd.getFieldId(),123u); + EXPECT_TRUE(!tfmd.isNotNeeded()); +} + +TEST("require that MatchData soft_reset retains appropriate state") { + auto md = MatchData::makeTestInstance(10, 10); + md->set_termwise_limit(0.5); + auto *old_term = md->resolveTermField(7); + old_term->tagAsNotNeeded(); + old_term->populate_fixed()->setElementWeight(21); + old_term->resetOnlyDocId(42); + EXPECT_EQUAL(md->get_termwise_limit(), 0.5); + EXPECT_TRUE(old_term->isNotNeeded()); + EXPECT_EQUAL(old_term->getFieldId(), 7u); + EXPECT_EQUAL(old_term->getWeight(), 21); + EXPECT_EQUAL(old_term->getDocId(), 42u); + md->soft_reset(); + auto *new_term = md->resolveTermField(7); + EXPECT_EQUAL(new_term, old_term); + EXPECT_EQUAL(md->get_termwise_limit(), 1.0); + EXPECT_TRUE(!new_term->isNotNeeded()); + EXPECT_EQUAL(new_term->getFieldId(), 7u); + EXPECT_EQUAL(new_term->getWeight(), 21); + EXPECT_EQUAL(new_term->getDocId(), TermFieldMatchData::invalidId()); +} + TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchlib/src/tests/index/docbuilder/docbuilder_test.cpp b/searchlib/src/tests/index/docbuilder/docbuilder_test.cpp index e483276b050..3e70a8fc40f 100644 --- a/searchlib/src/tests/index/docbuilder/docbuilder_test.cpp +++ b/searchlib/src/tests/index/docbuilder/docbuilder_test.cpp @@ -1,13 +1,4 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -/* -*- mode: C++; coding: utf-8; -*- */ - -/* $Id$ - * - * Copyright (C) 2011 Yahoo! Technologies Norway AS - * - * All Rights Reserved - * - */ #include <vespa/log/log.h> LOG_SETUP("docbuilder_test"); diff --git a/searchlib/src/tests/postinglistbm/postinglistbm.cpp b/searchlib/src/tests/postinglistbm/postinglistbm.cpp index 3a20af3e00a..41d94bf7186 100644 --- a/searchlib/src/tests/postinglistbm/postinglistbm.cpp +++ b/searchlib/src/tests/postinglistbm/postinglistbm.cpp @@ -1,6 +1,4 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// Copyright (C) 2002-2003 Fast Search & Transfer ASA -// Copyright (C) 2003 Overture Services Norway AS #include <vespa/searchlib/common/bitvector.h> #include <vespa/searchlib/common/resultset.h> diff --git a/searchlib/src/tests/ranksetup/ranksetup_test.cpp b/searchlib/src/tests/ranksetup/ranksetup_test.cpp index c2ef8f3a46b..7a26180eed2 100644 --- a/searchlib/src/tests/ranksetup/ranksetup_test.cpp +++ b/searchlib/src/tests/ranksetup/ranksetup_test.cpp @@ -87,6 +87,7 @@ private: const RankEnvironment & _rankEnv; MatchDataLayout _layout; std::unique_ptr<RankSetup> _rs; + MatchData::UP _match_data; RankProgram::UP _firstPhaseProgram; RankProgram::UP _secondPhaseProgram; @@ -100,7 +101,7 @@ public: RankExecutor::RankExecutor(const vespalib::string &initRank, const vespalib::string &finalRank, const RankEnvironment &rankEnv) : _initRank(initRank), _finalRank(finalRank), _rankEnv(rankEnv), _layout(), - _rs(), _firstPhaseProgram(), _secondPhaseProgram() + _rs(), _match_data(), _firstPhaseProgram(), _secondPhaseProgram() {} RankExecutor::~RankExecutor() {} @@ -121,12 +122,13 @@ RankExecutor::setup() if (!_rs->compile()) { return false; } + _match_data = _layout.createMatchData(); _firstPhaseProgram = _rs->create_first_phase_program(); - _firstPhaseProgram->setup(_layout, _rankEnv.queryEnvironment()); + _firstPhaseProgram->setup(*_match_data, _rankEnv.queryEnvironment()); if (!_finalRank.empty()) { _secondPhaseProgram = _rs->create_second_phase_program(); - _secondPhaseProgram->setup(_layout, _rankEnv.queryEnvironment()); + _secondPhaseProgram->setup(*_match_data, _rankEnv.queryEnvironment()); } return true; } @@ -154,6 +156,7 @@ private: const RankEnvironment & _rankEnv; RankSetup _setup; MatchDataLayout _layout; + MatchData::UP _match_data; RankProgram::UP _rankProgram; public: @@ -169,6 +172,7 @@ FeatureDumper::FeatureDumper(const RankEnvironment & rankEnv) : _rankEnv(rankEnv), _setup(_rankEnv.factory(), _rankEnv.indexEnvironment()), _layout(), + _match_data(), _rankProgram() {} FeatureDumper::~FeatureDumper() {} @@ -191,8 +195,9 @@ FeatureDumper::setup() return false; } + _match_data = _layout.createMatchData(); _rankProgram = _setup.create_dump_program(); - _rankProgram->setup(_layout, _rankEnv.queryEnvironment()); + _rankProgram->setup(*_match_data, _rankEnv.queryEnvironment()); return true; } @@ -780,12 +785,13 @@ RankSetupTest::testFeatureNormalization() { // RANK context MatchDataLayout layout; QueryEnvironment queryEnv; + MatchData::UP match_data = layout.createMatchData(); RankProgram::UP firstPhaseProgram = rankSetup.create_first_phase_program(); RankProgram::UP secondPhaseProgram = rankSetup.create_second_phase_program(); RankProgram::UP summaryProgram = rankSetup.create_summary_program(); - firstPhaseProgram->setup(layout, queryEnv); - secondPhaseProgram->setup(layout, queryEnv); - summaryProgram->setup(layout, queryEnv); + firstPhaseProgram->setup(*match_data, queryEnv); + secondPhaseProgram->setup(*match_data, queryEnv); + summaryProgram->setup(*match_data, queryEnv); EXPECT_APPROX(2.0, Utils::getScoreFeature(*firstPhaseProgram, 1), 0.001); EXPECT_APPROX(4.0, Utils::getScoreFeature(*secondPhaseProgram, 1), 0.001); @@ -831,8 +837,9 @@ RankSetupTest::testFeatureNormalization() { // DUMP context MatchDataLayout layout; QueryEnvironment queryEnv; + MatchData::UP match_data = layout.createMatchData(); RankProgram::UP rankProgram = rankSetup.create_dump_program(); - rankProgram->setup(layout, queryEnv); + rankProgram->setup(*match_data, queryEnv); { // dump seed features std::map<vespalib::string, feature_t> actual = Utils::getSeedFeatures(*rankProgram, 1); diff --git a/searchlib/src/tests/sortresults/sorttest.cpp b/searchlib/src/tests/sortresults/sorttest.cpp index c4c3fd0596a..6bcdca6b2af 100644 --- a/searchlib/src/tests/sortresults/sorttest.cpp +++ b/searchlib/src/tests/sortresults/sorttest.cpp @@ -1,7 +1,4 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// Copyright (C) 2001-2003 Fast Search & Transfer ASA -// Copyright (C) 2003 Overture Services Norway AS - #include <vespa/searchlib/common/bitvector.h> #include <vespa/searchlib/common/sortresults.h> diff --git a/searchlib/src/tests/stackdumpiterator/stackdumpiteratortest.cpp b/searchlib/src/tests/stackdumpiterator/stackdumpiteratortest.cpp index dbf0534e14a..d3a99dc439a 100644 --- a/searchlib/src/tests/stackdumpiterator/stackdumpiteratortest.cpp +++ b/searchlib/src/tests/stackdumpiterator/stackdumpiteratortest.cpp @@ -1,6 +1,4 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// Copyright (C) 2001-2003 Fast Search & Transfer ASA -// Copyright (C) 2003 Overture Services Norway AS #include "stackdumpiteratortest.h" #include <vespa/searchlib/parsequery/simplequerystack.h> diff --git a/searchlib/src/tests/stackdumpiterator/stackdumpiteratortest.h b/searchlib/src/tests/stackdumpiterator/stackdumpiteratortest.h index 394e1d59ebd..bca1dd45893 100644 --- a/searchlib/src/tests/stackdumpiterator/stackdumpiteratortest.h +++ b/searchlib/src/tests/stackdumpiterator/stackdumpiteratortest.h @@ -1,6 +1,4 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// Copyright (C) 2001-2003 Fast Search & Transfer ASA -// Copyright (C) 2003 Overture Services Norway AS #pragma once diff --git a/searchlib/src/tests/stringenum/stringenum_test.cpp b/searchlib/src/tests/stringenum/stringenum_test.cpp index 41f6e5edbb5..a40c009e543 100644 --- a/searchlib/src/tests/stringenum/stringenum_test.cpp +++ b/searchlib/src/tests/stringenum/stringenum_test.cpp @@ -1,7 +1,4 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// Copyright (C) 2001-2003 Fast Search & Transfer ASA -// Copyright (C) 2003 Overture Services Norway AS - #include <vespa/log/log.h> LOG_SETUP("stringenum"); diff --git a/searchlib/src/tests/url/testurl.cpp b/searchlib/src/tests/url/testurl.cpp index 7ac67480a2e..207f77825c5 100644 --- a/searchlib/src/tests/url/testurl.cpp +++ b/searchlib/src/tests/url/testurl.cpp @@ -1,6 +1,4 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// Copyright (C) 2000-2003 Fast Search & Transfer ASA -// Copyright (C) 2003 Overture Services Norway AS #include <vespa/searchlib/util/url.h> #include <cstdio> diff --git a/searchlib/src/vespa/searchlib/attribute/attributevector.cpp b/searchlib/src/vespa/searchlib/attribute/attributevector.cpp index b21f0e0dfff..8b3af9bddd3 100644 --- a/searchlib/src/vespa/searchlib/attribute/attributevector.cpp +++ b/searchlib/src/vespa/searchlib/attribute/attributevector.cpp @@ -696,6 +696,7 @@ AttributeVector::enableEnumeratedSave(bool enable) { } attribute::IPostingListAttributeBase *AttributeVector::getIPostingListAttributeBase() { return nullptr; } +const attribute::IPostingListAttributeBase *AttributeVector::getIPostingListAttributeBase() const { return nullptr; } const IDocumentWeightAttribute * AttributeVector::asDocumentWeightAttribute() const { return nullptr; } bool AttributeVector::hasPostings() { return getIPostingListAttributeBase() != nullptr; } uint64_t AttributeVector::getUniqueValueCount() const { return getTotalValueCount(); } @@ -804,13 +805,14 @@ uint64_t AttributeVector::getEstimatedSaveByteSize() const { uint64_t headerSize = 4096; - uint64_t totalValueCount = getTotalValueCount(); - uint64_t uniqueValueCount = getUniqueValueCount(); + uint64_t totalValueCount = _status.getNumValues(); + uint64_t uniqueValueCount = _status.getNumUniqueValues(); uint64_t docIdLimit = getCommittedDocIdLimit(); uint64_t datFileSize = 0; uint64_t weightFileSize = 0; uint64_t idxFileSize = 0; uint64_t udatFileSize = 0; + size_t fixedWidth = getFixedWidth(); AddressSpace enumAddressSpace(getEnumStoreAddressSpaceUsage()); if (hasMultiValue()) { @@ -821,8 +823,12 @@ AttributeVector::getEstimatedSaveByteSize() const } if (hasEnum() && getEnumeratedSave()) { datFileSize = headerSize + 4 * totalValueCount; - udatFileSize = headerSize + enumAddressSpace.used() - - 8 * uniqueValueCount; + if (fixedWidth != 0) { + udatFileSize = headerSize + fixedWidth * uniqueValueCount; + } else { + udatFileSize = headerSize + enumAddressSpace.used() + - 8 * uniqueValueCount; + } } else { BasicType::Type basicType(getBasicType()); const Status &status = getStatus(); @@ -844,7 +850,7 @@ AttributeVector::getEstimatedSaveByteSize() const } break; default: - datFileSize = headerSize + getFixedWidth() * totalValueCount; + datFileSize = headerSize + fixedWidth * totalValueCount; break; } } diff --git a/searchlib/src/vespa/searchlib/attribute/attributevector.h b/searchlib/src/vespa/searchlib/attribute/attributevector.h index d9426822603..13ba3f801b7 100644 --- a/searchlib/src/vespa/searchlib/attribute/attributevector.h +++ b/searchlib/src/vespa/searchlib/attribute/attributevector.h @@ -661,6 +661,7 @@ public: bool getEnumeratedSave() const { return _hasEnum && _enableEnumeratedSave; } virtual attribute::IPostingListAttributeBase * getIPostingListAttributeBase(); + virtual const attribute::IPostingListAttributeBase * getIPostingListAttributeBase() const; bool hasPostings(); virtual uint64_t getUniqueValueCount() const; virtual uint64_t getTotalValueCount() const; diff --git a/searchlib/src/vespa/searchlib/attribute/ipostinglistattributebase.h b/searchlib/src/vespa/searchlib/attribute/ipostinglistattributebase.h index 91f2a63976b..8d44c022062 100644 --- a/searchlib/src/vespa/searchlib/attribute/ipostinglistattributebase.h +++ b/searchlib/src/vespa/searchlib/attribute/ipostinglistattributebase.h @@ -22,8 +22,8 @@ public: uint32_t fromLid, uint32_t toLid) = 0; - virtual void - forwardedShrinkLidSpace(uint32_t newSize) = 0; + virtual void forwardedShrinkLidSpace(uint32_t newSize) = 0; + virtual MemoryUsage getMemoryUsage() const = 0; }; diff --git a/searchlib/src/vespa/searchlib/attribute/multinumericpostattribute.h b/searchlib/src/vespa/searchlib/attribute/multinumericpostattribute.h index ac525da419f..68aa8d6673c 100644 --- a/searchlib/src/vespa/searchlib/attribute/multinumericpostattribute.h +++ b/searchlib/src/vespa/searchlib/attribute/multinumericpostattribute.h @@ -104,6 +104,10 @@ public: return this; } + const attribute::IPostingListAttributeBase *getIPostingListAttributeBase() const override { + return this; + } + void fillPostingsFixupEnum(const LoadedEnumAttributeVector &loaded) override { fillPostingsFixupEnumBase(loaded); } diff --git a/searchlib/src/vespa/searchlib/attribute/multistringpostattribute.h b/searchlib/src/vespa/searchlib/attribute/multistringpostattribute.h index 348bef44bf8..04f26d232d5 100644 --- a/searchlib/src/vespa/searchlib/attribute/multistringpostattribute.h +++ b/searchlib/src/vespa/searchlib/attribute/multistringpostattribute.h @@ -105,6 +105,8 @@ public: attribute::IPostingListAttributeBase * getIPostingListAttributeBase() override { return this; } + const attribute::IPostingListAttributeBase * getIPostingListAttributeBase() const override { return this; } + void fillPostingsFixupEnum(const LoadedEnumAttributeVector &loaded) override { fillPostingsFixupEnumBase(loaded); } diff --git a/searchlib/src/vespa/searchlib/attribute/postinglistattribute.cpp b/searchlib/src/vespa/searchlib/attribute/postinglistattribute.cpp index fa24221b888..7750758469b 100644 --- a/searchlib/src/vespa/searchlib/attribute/postinglistattribute.cpp +++ b/searchlib/src/vespa/searchlib/attribute/postinglistattribute.cpp @@ -202,6 +202,12 @@ PostingListAttributeBase<P>::forwardedShrinkLidSpace(uint32_t newSize) (void) _postingList.resizeBitVectors(newSize, newSize); } +template <typename P> +MemoryUsage +PostingListAttributeBase<P>::getMemoryUsage() const +{ + return _postingList.getMemoryUsage(); +} template <typename P, typename LoadedVector, typename LoadedValueType, typename EnumStoreType> diff --git a/searchlib/src/vespa/searchlib/attribute/postinglistattribute.h b/searchlib/src/vespa/searchlib/attribute/postinglistattribute.h index 695c7872255..a8563ef6027 100644 --- a/searchlib/src/vespa/searchlib/attribute/postinglistattribute.h +++ b/searchlib/src/vespa/searchlib/attribute/postinglistattribute.h @@ -65,6 +65,7 @@ protected: uint32_t toLid, EnumStoreComparator &cmp); void forwardedShrinkLidSpace(uint32_t newSize) override; + virtual MemoryUsage getMemoryUsage() const override; public: const PostingList & getPostingList() const { return _postingList; } diff --git a/searchlib/src/vespa/searchlib/attribute/predicate_attribute.h b/searchlib/src/vespa/searchlib/attribute/predicate_attribute.h index 309053e52f1..f318e39f4fa 100644 --- a/searchlib/src/vespa/searchlib/attribute/predicate_attribute.h +++ b/searchlib/src/vespa/searchlib/attribute/predicate_attribute.h @@ -25,7 +25,7 @@ private: /** * Attribute that manages a predicate index. It is not a traditional * attribute in that it doesn't store values for each document, but - * rather keeps an index for boolean search. Summaries are not fetched + * rather keeps an index for predicate search. Summaries are not fetched * from the attribute, but rather using the summary store like a * non-index field. */ diff --git a/searchlib/src/vespa/searchlib/attribute/singlenumericpostattribute.h b/searchlib/src/vespa/searchlib/attribute/singlenumericpostattribute.h index 9235611fd09..b50944492a8 100644 --- a/searchlib/src/vespa/searchlib/attribute/singlenumericpostattribute.h +++ b/searchlib/src/vespa/searchlib/attribute/singlenumericpostattribute.h @@ -95,6 +95,7 @@ public: void fillPostings(LoadedVector & loaded) override { handleFillPostings(loaded); } attribute::IPostingListAttributeBase *getIPostingListAttributeBase() override { return this; } + const attribute::IPostingListAttributeBase *getIPostingListAttributeBase() const override { return this; } void fillPostingsFixupEnum(const LoadedEnumAttributeVector &loaded) override { fillPostingsFixupEnumBase(loaded); } }; diff --git a/searchlib/src/vespa/searchlib/attribute/singlestringpostattribute.h b/searchlib/src/vespa/searchlib/attribute/singlestringpostattribute.h index 506269733ba..de8baf4473f 100644 --- a/searchlib/src/vespa/searchlib/attribute/singlestringpostattribute.h +++ b/searchlib/src/vespa/searchlib/attribute/singlestringpostattribute.h @@ -110,6 +110,10 @@ public: return this; } + const attribute::IPostingListAttributeBase * getIPostingListAttributeBase() const override { + return this; + } + void fillPostingsFixupEnum(const LoadedEnumAttributeVector &loaded) override { fillPostingsFixupEnumBase(loaded); } diff --git a/searchlib/src/vespa/searchlib/bitcompression/countcompression.cpp b/searchlib/src/vespa/searchlib/bitcompression/countcompression.cpp index cec128a3907..fca6c749cfb 100644 --- a/searchlib/src/vespa/searchlib/bitcompression/countcompression.cpp +++ b/searchlib/src/vespa/searchlib/bitcompression/countcompression.cpp @@ -1,6 +1,4 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// Copyright (C) 1998-2003 Fast Search & Transfer ASA -// Copyright (C) 2003 Overture Services Norway AS #include "compression.h" #include "countcompression.h" diff --git a/searchlib/src/vespa/searchlib/bitcompression/posocccompression.cpp b/searchlib/src/vespa/searchlib/bitcompression/posocccompression.cpp index bb8921e8fae..739018b99a5 100644 --- a/searchlib/src/vespa/searchlib/bitcompression/posocccompression.cpp +++ b/searchlib/src/vespa/searchlib/bitcompression/posocccompression.cpp @@ -1,6 +1,4 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// Copyright (C) 2002-2003 Fast Search & Transfer ASA -// Copyright (C) 2003 Overture Services Norway AS #include "compression.h" #include "posocccompression.h" diff --git a/searchlib/src/vespa/searchlib/bitcompression/posocccompression.h b/searchlib/src/vespa/searchlib/bitcompression/posocccompression.h index c0adcc98153..6eb6cac46b6 100644 --- a/searchlib/src/vespa/searchlib/bitcompression/posocccompression.h +++ b/searchlib/src/vespa/searchlib/bitcompression/posocccompression.h @@ -1,6 +1,4 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// Copyright (C) 2002-2003 Fast Search & Transfer ASA -// Copyright (C) 2003 Overture Services Norway AS #pragma once #include <vespa/searchlib/index/docidandfeatures.h> diff --git a/searchlib/src/vespa/searchlib/common/allocatedbitvector.h b/searchlib/src/vespa/searchlib/common/allocatedbitvector.h index 1f895eee93f..6de255c48c9 100644 --- a/searchlib/src/vespa/searchlib/common/allocatedbitvector.h +++ b/searchlib/src/vespa/searchlib/common/allocatedbitvector.h @@ -1,6 +1,4 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// Copyright (C) 1998-2003 Fast Search & Transfer ASA -// Copyright (C) 2003 Overture Services Norway AS #pragma once diff --git a/searchlib/src/vespa/searchlib/common/base.h b/searchlib/src/vespa/searchlib/common/base.h index 0bc2f4401a7..3f462d3e3d1 100644 --- a/searchlib/src/vespa/searchlib/common/base.h +++ b/searchlib/src/vespa/searchlib/common/base.h @@ -1,6 +1,4 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// Copyright (C) 1999-2003 Fast Search & Transfer ASA -// Copyright (C) 2003 Overture Services Norway AS #pragma once diff --git a/searchlib/src/vespa/searchlib/common/documentsummary.cpp b/searchlib/src/vespa/searchlib/common/documentsummary.cpp index 5836c623779..0d00ad1a408 100644 --- a/searchlib/src/vespa/searchlib/common/documentsummary.cpp +++ b/searchlib/src/vespa/searchlib/common/documentsummary.cpp @@ -1,6 +1,4 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// Copyright (C) 1998-2003 Fast Search & Transfer ASA -// Copyright (C) 2003 Overture Services Norway AS #include "documentsummary.h" #include <vespa/fastlib/io/bufferedfile.h> diff --git a/searchlib/src/vespa/searchlib/common/documentsummary.h b/searchlib/src/vespa/searchlib/common/documentsummary.h index 2d88158d5b5..09512d92c1e 100644 --- a/searchlib/src/vespa/searchlib/common/documentsummary.h +++ b/searchlib/src/vespa/searchlib/common/documentsummary.h @@ -1,6 +1,4 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// Copyright (C) 1998-2003 Fast Search & Transfer ASA -// Copyright (C) 2003 Overture Services Norway AS #pragma once diff --git a/searchlib/src/vespa/searchlib/common/gid.h b/searchlib/src/vespa/searchlib/common/gid.h index b4a94c98a9a..33b17fe94e6 100644 --- a/searchlib/src/vespa/searchlib/common/gid.h +++ b/searchlib/src/vespa/searchlib/common/gid.h @@ -1,6 +1,4 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// Copyright (C) 1998-2003 Fast Search & Transfer ASA -// Copyright (C) 2003 Overture Services Norway AS #pragma once diff --git a/searchlib/src/vespa/searchlib/common/growablebitvector.cpp b/searchlib/src/vespa/searchlib/common/growablebitvector.cpp index ede9038fc8d..41c99a41ce9 100644 --- a/searchlib/src/vespa/searchlib/common/growablebitvector.cpp +++ b/searchlib/src/vespa/searchlib/common/growablebitvector.cpp @@ -1,6 +1,4 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// Copyright (C) 1998-2003 Fast Search & Transfer ASA -// Copyright (C) 2003 Overture Services Norway AS #include "growablebitvector.h" diff --git a/searchlib/src/vespa/searchlib/common/growablebitvector.h b/searchlib/src/vespa/searchlib/common/growablebitvector.h index 36c59502804..1c5cd31b235 100644 --- a/searchlib/src/vespa/searchlib/common/growablebitvector.h +++ b/searchlib/src/vespa/searchlib/common/growablebitvector.h @@ -1,6 +1,4 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// Copyright (C) 1998-2003 Fast Search & Transfer ASA -// Copyright (C) 2003 Overture Services Norway AS #pragma once diff --git a/searchlib/src/vespa/searchlib/common/location.cpp b/searchlib/src/vespa/searchlib/common/location.cpp index cda3973b311..6927d9ab6cb 100644 --- a/searchlib/src/vespa/searchlib/common/location.cpp +++ b/searchlib/src/vespa/searchlib/common/location.cpp @@ -1,6 +1,4 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// Copyright (C) 1999-2003 Fast Search & Transfer ASA -// Copyright (C) 2003 Overture Services Norway AS #include "location.h" #include <limits> diff --git a/searchlib/src/vespa/searchlib/common/location.h b/searchlib/src/vespa/searchlib/common/location.h index 4cdb5a81a1a..96821e204e2 100644 --- a/searchlib/src/vespa/searchlib/common/location.h +++ b/searchlib/src/vespa/searchlib/common/location.h @@ -1,5 +1,4 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// Copyright (C) 2004 Overture Services Norway AS #pragma once diff --git a/searchlib/src/vespa/searchlib/common/partialbitvector.h b/searchlib/src/vespa/searchlib/common/partialbitvector.h index 4cfd3e34383..f1d0716ed60 100644 --- a/searchlib/src/vespa/searchlib/common/partialbitvector.h +++ b/searchlib/src/vespa/searchlib/common/partialbitvector.h @@ -1,6 +1,4 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// Copyright (C) 1998-2003 Fast Search & Transfer ASA -// Copyright (C) 2003 Overture Services Norway AS #pragma once diff --git a/searchlib/src/vespa/searchlib/common/rankedhit.h b/searchlib/src/vespa/searchlib/common/rankedhit.h index ed8860e0038..635f6e350a5 100644 --- a/searchlib/src/vespa/searchlib/common/rankedhit.h +++ b/searchlib/src/vespa/searchlib/common/rankedhit.h @@ -1,6 +1,4 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// Copyright (C) 1998-2003 Fast Search & Transfer ASA -// Copyright (C) 2003 Overture Services Norway AS #pragma once diff --git a/searchlib/src/vespa/searchlib/common/reserved.h b/searchlib/src/vespa/searchlib/common/reserved.h index 032bed2fd54..41a540a67c8 100644 --- a/searchlib/src/vespa/searchlib/common/reserved.h +++ b/searchlib/src/vespa/searchlib/common/reserved.h @@ -1,6 +1,4 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// Copyright (C) 2002-2003 Fast Search & Transfer ASA -// Copyright (C) 2003 Overture Services Norway AS #pragma once diff --git a/searchlib/src/vespa/searchlib/common/resultset.cpp b/searchlib/src/vespa/searchlib/common/resultset.cpp index 44e70886956..20c636df77c 100644 --- a/searchlib/src/vespa/searchlib/common/resultset.cpp +++ b/searchlib/src/vespa/searchlib/common/resultset.cpp @@ -1,6 +1,4 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// Copyright (C) 1998-2003 Fast Search & Transfer ASA -// Copyright (C) 2003 Overture Services Norway AS #include "resultset.h" #include "bitvector.h" diff --git a/searchlib/src/vespa/searchlib/common/sortresults.cpp b/searchlib/src/vespa/searchlib/common/sortresults.cpp index 11f6eb306eb..ed86014f7b3 100644 --- a/searchlib/src/vespa/searchlib/common/sortresults.cpp +++ b/searchlib/src/vespa/searchlib/common/sortresults.cpp @@ -1,6 +1,4 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// Copyright (C) 2001-2003 Fast Search & Transfer ASA -// Copyright (C) 2003 Overture Services Norway AS #include "sortresults.h" #include <vespa/searchlib/util/sort.h> diff --git a/searchlib/src/vespa/searchlib/common/sortresults.h b/searchlib/src/vespa/searchlib/common/sortresults.h index 8da7073b4f2..26bdc8eba96 100644 --- a/searchlib/src/vespa/searchlib/common/sortresults.h +++ b/searchlib/src/vespa/searchlib/common/sortresults.h @@ -1,6 +1,4 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// Copyright (C) 2001-2003 Fast Search & Transfer ASA -// Copyright (C) 2003 Overture Services Norway AS #pragma once diff --git a/searchlib/src/vespa/searchlib/config/CMakeLists.txt b/searchlib/src/vespa/searchlib/config/CMakeLists.txt index da78c4c74ad..2f34d228f60 100644 --- a/searchlib/src/vespa/searchlib/config/CMakeLists.txt +++ b/searchlib/src/vespa/searchlib/config/CMakeLists.txt @@ -4,4 +4,4 @@ vespa_add_library(searchlib_sconfig OBJECT DEPENDS ) vespa_generate_config(searchlib_sconfig translogserver.def) -install(FILES translogserver.def DESTINATION var/db/vespa/config_server/serverdb/classes) +install(FILES translogserver.def RENAME searchlib.translogserver.def DESTINATION var/db/vespa/config_server/serverdb/classes) diff --git a/searchlib/src/vespa/searchlib/diskindex/bitvectorfile.cpp b/searchlib/src/vespa/searchlib/diskindex/bitvectorfile.cpp index 1a9107db4bb..defacdb05d7 100644 --- a/searchlib/src/vespa/searchlib/diskindex/bitvectorfile.cpp +++ b/searchlib/src/vespa/searchlib/diskindex/bitvectorfile.cpp @@ -1,6 +1,4 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// Copyright (C) 1998-2003 Fast Search & Transfer ASA -// Copyright (C) 2003 Overture Services Norway AS #include "bitvectorfile.h" #include <vespa/searchlib/index/bitvectorkeys.h> diff --git a/searchlib/src/vespa/searchlib/diskindex/bitvectorfile.h b/searchlib/src/vespa/searchlib/diskindex/bitvectorfile.h index f3b34285085..1b7b522bc52 100644 --- a/searchlib/src/vespa/searchlib/diskindex/bitvectorfile.h +++ b/searchlib/src/vespa/searchlib/diskindex/bitvectorfile.h @@ -1,6 +1,5 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// Copyright (C) 1998-2003 Fast Search & Transfer ASA -// Copyright (C) 2003 Overture Services Norway AS + #pragma once #include <vespa/fastlib/io/bufferedfile.h> diff --git a/searchlib/src/vespa/searchlib/diskindex/bitvectoridxfile.cpp b/searchlib/src/vespa/searchlib/diskindex/bitvectoridxfile.cpp index 51caad0f481..856c527a8b5 100644 --- a/searchlib/src/vespa/searchlib/diskindex/bitvectoridxfile.cpp +++ b/searchlib/src/vespa/searchlib/diskindex/bitvectoridxfile.cpp @@ -1,6 +1,4 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// Copyright (C) 1998-2003 Fast Search & Transfer ASA -// Copyright (C) 2003 Overture Services Norway AS #include "bitvectoridxfile.h" #include <vespa/searchlib/index/bitvectorkeys.h> diff --git a/searchlib/src/vespa/searchlib/diskindex/bitvectoridxfile.h b/searchlib/src/vespa/searchlib/diskindex/bitvectoridxfile.h index 0e19af34f00..0ab40d9c4a3 100644 --- a/searchlib/src/vespa/searchlib/diskindex/bitvectoridxfile.h +++ b/searchlib/src/vespa/searchlib/diskindex/bitvectoridxfile.h @@ -1,6 +1,5 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// Copyright (C) 1998-2003 Fast Search & Transfer ASA -// Copyright (C) 2003 Overture Services Norway AS + #pragma once #include <vespa/fastlib/io/bufferedfile.h> diff --git a/searchlib/src/vespa/searchlib/diskindex/extposocc.cpp b/searchlib/src/vespa/searchlib/diskindex/extposocc.cpp index aaacf0b6b67..89c801f04b5 100644 --- a/searchlib/src/vespa/searchlib/diskindex/extposocc.cpp +++ b/searchlib/src/vespa/searchlib/diskindex/extposocc.cpp @@ -1,7 +1,4 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// Copyright (C) 2002-2003 Fast Search & Transfer ASA -// Copyright (C) 2003 Overture Services Norway AS - #include "extposocc.h" #include "zcposocc.h" diff --git a/searchlib/src/vespa/searchlib/diskindex/extposocc.h b/searchlib/src/vespa/searchlib/diskindex/extposocc.h index 1c89323dd25..12c57402def 100644 --- a/searchlib/src/vespa/searchlib/diskindex/extposocc.h +++ b/searchlib/src/vespa/searchlib/diskindex/extposocc.h @@ -1,6 +1,4 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// Copyright (C) 2002-2003 Fast Search & Transfer ASA -// Copyright (C) 2003 Overture Services Norway AS #pragma once diff --git a/searchlib/src/vespa/searchlib/diskindex/fusion.cpp b/searchlib/src/vespa/searchlib/diskindex/fusion.cpp index 7d4efe2d69d..97d21caef96 100644 --- a/searchlib/src/vespa/searchlib/diskindex/fusion.cpp +++ b/searchlib/src/vespa/searchlib/diskindex/fusion.cpp @@ -1,5 +1,4 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// Copyright (C) 2003 Fast Search & Transfer ASA #include "fusion.h" #include "fieldreader.h" diff --git a/searchlib/src/vespa/searchlib/diskindex/fusion.h b/searchlib/src/vespa/searchlib/diskindex/fusion.h index a99b7936d87..76547ad0e5f 100644 --- a/searchlib/src/vespa/searchlib/diskindex/fusion.h +++ b/searchlib/src/vespa/searchlib/diskindex/fusion.h @@ -1,6 +1,4 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// Copyright (C) 2001-2003 Fast Search & Transfer ASA -// Copyright (C) 2003 Overture Services Norway AS #pragma once diff --git a/searchlib/src/vespa/searchlib/engine/errorcodes.h b/searchlib/src/vespa/searchlib/engine/errorcodes.h index 36ab38a4555..3b2357a1ebb 100644 --- a/searchlib/src/vespa/searchlib/engine/errorcodes.h +++ b/searchlib/src/vespa/searchlib/engine/errorcodes.h @@ -1,6 +1,4 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// Copyright (C) 1999-2003 Fast Search & Transfer ASA -// Copyright (C) 2003 Overture Services Norway AS #pragma once diff --git a/searchlib/src/vespa/searchlib/features/attributematchfeature.cpp b/searchlib/src/vespa/searchlib/features/attributematchfeature.cpp index bf3297f3e14..2ff53951d8b 100644 --- a/searchlib/src/vespa/searchlib/features/attributematchfeature.cpp +++ b/searchlib/src/vespa/searchlib/features/attributematchfeature.cpp @@ -253,7 +253,7 @@ AttributeMatchExecutor<T>::execute(uint32_t docId) template <typename T> void -AttributeMatchExecutor<T>::handle_bind_match_data(MatchData &md) +AttributeMatchExecutor<T>::handle_bind_match_data(const MatchData &md) { _cmp.bind_match_data(md); } diff --git a/searchlib/src/vespa/searchlib/features/attributematchfeature.h b/searchlib/src/vespa/searchlib/features/attributematchfeature.h index 8396493b81d..707572abf9e 100644 --- a/searchlib/src/vespa/searchlib/features/attributematchfeature.h +++ b/searchlib/src/vespa/searchlib/features/attributematchfeature.h @@ -69,12 +69,12 @@ private: feature_t getWeight() const; feature_t getSignificance() const; feature_t getImportance() const { return (getWeight() + getSignificance()) * 0.5; } - void bind_match_data(fef::MatchData &md) { _md = &md; } + void bind_match_data(const fef::MatchData &md) { _md = &md; } }; Computer _cmp; - virtual void handle_bind_match_data(fef::MatchData &md) override; + virtual void handle_bind_match_data(const fef::MatchData &md) override; public: /** diff --git a/searchlib/src/vespa/searchlib/features/element_completeness_feature.cpp b/searchlib/src/vespa/searchlib/features/element_completeness_feature.cpp index c3bb87379ad..18988281cbd 100644 --- a/searchlib/src/vespa/searchlib/features/element_completeness_feature.cpp +++ b/searchlib/src/vespa/searchlib/features/element_completeness_feature.cpp @@ -73,7 +73,7 @@ ElementCompletenessExecutor::execute(uint32_t docId) } void -ElementCompletenessExecutor::handle_bind_match_data(fef::MatchData &md) +ElementCompletenessExecutor::handle_bind_match_data(const fef::MatchData &md) { _md = &md; } diff --git a/searchlib/src/vespa/searchlib/features/element_completeness_feature.h b/searchlib/src/vespa/searchlib/features/element_completeness_feature.h index 013efaedc22..1b835da45d2 100644 --- a/searchlib/src/vespa/searchlib/features/element_completeness_feature.h +++ b/searchlib/src/vespa/searchlib/features/element_completeness_feature.h @@ -82,7 +82,7 @@ private: static bool nextElement(Item &item); - void handle_bind_match_data(fef::MatchData &md) override; + void handle_bind_match_data(const fef::MatchData &md) override; public: ElementCompletenessExecutor(const fef::IQueryEnvironment &env, diff --git a/searchlib/src/vespa/searchlib/features/element_similarity_feature.cpp b/searchlib/src/vespa/searchlib/features/element_similarity_feature.cpp index e3c0217be15..c9eab238f82 100644 --- a/searchlib/src/vespa/searchlib/features/element_similarity_feature.cpp +++ b/searchlib/src/vespa/searchlib/features/element_similarity_feature.cpp @@ -230,7 +230,7 @@ public: bool isPure() override { return _terms.handles.empty(); } - void handle_bind_match_data(fef::MatchData &md) override { + void handle_bind_match_data(const fef::MatchData &md) override { _md = &md; } diff --git a/searchlib/src/vespa/searchlib/features/fieldinfofeature.cpp b/searchlib/src/vespa/searchlib/features/fieldinfofeature.cpp index 85f3fc9b652..960dcc7dfd1 100644 --- a/searchlib/src/vespa/searchlib/features/fieldinfofeature.cpp +++ b/searchlib/src/vespa/searchlib/features/fieldinfofeature.cpp @@ -59,7 +59,7 @@ IndexFieldInfoExecutor::execute(uint32_t docId) } void -IndexFieldInfoExecutor::handle_bind_match_data(fef::MatchData &md) +IndexFieldInfoExecutor::handle_bind_match_data(const fef::MatchData &md) { _md = &md; } @@ -98,7 +98,7 @@ AttrFieldInfoExecutor::execute(uint32_t docId) } void -AttrFieldInfoExecutor::handle_bind_match_data(fef::MatchData &md) +AttrFieldInfoExecutor::handle_bind_match_data(const fef::MatchData &md) { _md = &md; } diff --git a/searchlib/src/vespa/searchlib/features/fieldinfofeature.h b/searchlib/src/vespa/searchlib/features/fieldinfofeature.h index 08cf99ed32a..7ced5d5cb9f 100644 --- a/searchlib/src/vespa/searchlib/features/fieldinfofeature.h +++ b/searchlib/src/vespa/searchlib/features/fieldinfofeature.h @@ -18,7 +18,7 @@ private: uint32_t _fieldHandle; const fef::MatchData *_md; - void handle_bind_match_data(fef::MatchData &md) override; + void handle_bind_match_data(const fef::MatchData &md) override; public: IndexFieldInfoExecutor(feature_t type, feature_t isFilter, @@ -35,7 +35,7 @@ private: uint32_t _fieldHandle; const fef::MatchData *_md; - void handle_bind_match_data(fef::MatchData &md) override; + void handle_bind_match_data(const fef::MatchData &md) override; public: AttrFieldInfoExecutor(feature_t type, uint32_t fieldHandle); diff --git a/searchlib/src/vespa/searchlib/features/fieldlengthfeature.cpp b/searchlib/src/vespa/searchlib/features/fieldlengthfeature.cpp index 0ec1f8bd038..d0680e8fc19 100644 --- a/searchlib/src/vespa/searchlib/features/fieldlengthfeature.cpp +++ b/searchlib/src/vespa/searchlib/features/fieldlengthfeature.cpp @@ -56,7 +56,7 @@ FieldLengthExecutor::execute(uint32_t docId) } void -FieldLengthExecutor::handle_bind_match_data(MatchData &md) +FieldLengthExecutor::handle_bind_match_data(const MatchData &md) { _md = &md; } diff --git a/searchlib/src/vespa/searchlib/features/fieldlengthfeature.h b/searchlib/src/vespa/searchlib/features/fieldlengthfeature.h index a818a65d26e..4988bb97546 100644 --- a/searchlib/src/vespa/searchlib/features/fieldlengthfeature.h +++ b/searchlib/src/vespa/searchlib/features/fieldlengthfeature.h @@ -15,7 +15,7 @@ private: std::vector<fef::TermFieldHandle> _fieldHandles; const fef::MatchData *_md; - virtual void handle_bind_match_data(fef::MatchData &md) override; + virtual void handle_bind_match_data(const fef::MatchData &md) override; public: /** diff --git a/searchlib/src/vespa/searchlib/features/fieldmatchfeature.cpp b/searchlib/src/vespa/searchlib/features/fieldmatchfeature.cpp index bea6e032e81..f5f8652461e 100644 --- a/searchlib/src/vespa/searchlib/features/fieldmatchfeature.cpp +++ b/searchlib/src/vespa/searchlib/features/fieldmatchfeature.cpp @@ -86,7 +86,7 @@ FieldMatchExecutor::execute(uint32_t docId) } void -FieldMatchExecutor::handle_bind_match_data(fef::MatchData &md) +FieldMatchExecutor::handle_bind_match_data(const fef::MatchData &md) { _splitter.bind_match_data(md); } diff --git a/searchlib/src/vespa/searchlib/features/fieldmatchfeature.h b/searchlib/src/vespa/searchlib/features/fieldmatchfeature.h index 287af60b021..71ac1023df6 100644 --- a/searchlib/src/vespa/searchlib/features/fieldmatchfeature.h +++ b/searchlib/src/vespa/searchlib/features/fieldmatchfeature.h @@ -19,7 +19,7 @@ private: const fieldmatch::Params & _params; fieldmatch::Computer _cmp; - void handle_bind_match_data(fef::MatchData &md) override; + void handle_bind_match_data(const fef::MatchData &md) override; public: /** diff --git a/searchlib/src/vespa/searchlib/features/fieldtermmatchfeature.cpp b/searchlib/src/vespa/searchlib/features/fieldtermmatchfeature.cpp index 0f71d5dcbec..a7a00bee956 100644 --- a/searchlib/src/vespa/searchlib/features/fieldtermmatchfeature.cpp +++ b/searchlib/src/vespa/searchlib/features/fieldtermmatchfeature.cpp @@ -63,7 +63,7 @@ FieldTermMatchExecutor::execute(uint32_t docId) } void -FieldTermMatchExecutor::handle_bind_match_data(fef::MatchData &md) +FieldTermMatchExecutor::handle_bind_match_data(const fef::MatchData &md) { _md = &md; } diff --git a/searchlib/src/vespa/searchlib/features/fieldtermmatchfeature.h b/searchlib/src/vespa/searchlib/features/fieldtermmatchfeature.h index 02dfa17ba33..0f89f7eeb75 100644 --- a/searchlib/src/vespa/searchlib/features/fieldtermmatchfeature.h +++ b/searchlib/src/vespa/searchlib/features/fieldtermmatchfeature.h @@ -27,7 +27,7 @@ private: fef::TermFieldHandle _fieldHandle; const fef::MatchData *_md; - void handle_bind_match_data(fef::MatchData &md) override; + void handle_bind_match_data(const fef::MatchData &md) override; }; /** diff --git a/searchlib/src/vespa/searchlib/features/flow_completeness_feature.cpp b/searchlib/src/vespa/searchlib/features/flow_completeness_feature.cpp index 5c93566698e..c46d602983d 100644 --- a/searchlib/src/vespa/searchlib/features/flow_completeness_feature.cpp +++ b/searchlib/src/vespa/searchlib/features/flow_completeness_feature.cpp @@ -234,7 +234,7 @@ FlowCompletenessExecutor::execute(uint32_t) } void -FlowCompletenessExecutor::handle_bind_match_data(fef::MatchData &md) +FlowCompletenessExecutor::handle_bind_match_data(const fef::MatchData &md) { _md = &md; } diff --git a/searchlib/src/vespa/searchlib/features/flow_completeness_feature.h b/searchlib/src/vespa/searchlib/features/flow_completeness_feature.h index cdfa0356307..db5c53beac8 100644 --- a/searchlib/src/vespa/searchlib/features/flow_completeness_feature.h +++ b/searchlib/src/vespa/searchlib/features/flow_completeness_feature.h @@ -62,7 +62,7 @@ private: static bool nextElement(Item &item); - void handle_bind_match_data(fef::MatchData &md) override; + void handle_bind_match_data(const fef::MatchData &md) override; public: FlowCompletenessExecutor(const fef::IQueryEnvironment &env, diff --git a/searchlib/src/vespa/searchlib/features/item_raw_score_feature.cpp b/searchlib/src/vespa/searchlib/features/item_raw_score_feature.cpp index a7476185000..45baf646656 100644 --- a/searchlib/src/vespa/searchlib/features/item_raw_score_feature.cpp +++ b/searchlib/src/vespa/searchlib/features/item_raw_score_feature.cpp @@ -23,7 +23,7 @@ ItemRawScoreExecutor::execute(uint32_t docId) } void -ItemRawScoreExecutor::handle_bind_match_data(MatchData &md) +ItemRawScoreExecutor::handle_bind_match_data(const MatchData &md) { _md = &md; } @@ -42,7 +42,7 @@ SimpleItemRawScoreExecutor::execute(uint32_t docId) } void -SimpleItemRawScoreExecutor::handle_bind_match_data(MatchData &md) +SimpleItemRawScoreExecutor::handle_bind_match_data(const MatchData &md) { _md = &md; } diff --git a/searchlib/src/vespa/searchlib/features/item_raw_score_feature.h b/searchlib/src/vespa/searchlib/features/item_raw_score_feature.h index 3aa1672b05d..d6a55f29632 100644 --- a/searchlib/src/vespa/searchlib/features/item_raw_score_feature.h +++ b/searchlib/src/vespa/searchlib/features/item_raw_score_feature.h @@ -16,7 +16,7 @@ private: HandleVector _handles; const fef::MatchData *_md; - void handle_bind_match_data(fef::MatchData &md) override; + void handle_bind_match_data(const fef::MatchData &md) override; public: ItemRawScoreExecutor(HandleVector handles) @@ -30,7 +30,7 @@ private: fef::TermFieldHandle _handle; const fef::MatchData *_md; - void handle_bind_match_data(fef::MatchData &md) override; + void handle_bind_match_data(const fef::MatchData &md) override; public: SimpleItemRawScoreExecutor(fef::TermFieldHandle handle) diff --git a/searchlib/src/vespa/searchlib/features/jarowinklerdistancefeature.cpp b/searchlib/src/vespa/searchlib/features/jarowinklerdistancefeature.cpp index 3f4817d61d2..a5e3e2da5ba 100644 --- a/searchlib/src/vespa/searchlib/features/jarowinklerdistancefeature.cpp +++ b/searchlib/src/vespa/searchlib/features/jarowinklerdistancefeature.cpp @@ -59,7 +59,7 @@ JaroWinklerDistanceExecutor::execute(uint32_t docId) } void -JaroWinklerDistanceExecutor::handle_bind_match_data(fef::MatchData &md) +JaroWinklerDistanceExecutor::handle_bind_match_data(const fef::MatchData &md) { _md = &md; } diff --git a/searchlib/src/vespa/searchlib/features/jarowinklerdistancefeature.h b/searchlib/src/vespa/searchlib/features/jarowinklerdistancefeature.h index 0497d2a2f73..a8d6eacf0eb 100644 --- a/searchlib/src/vespa/searchlib/features/jarowinklerdistancefeature.h +++ b/searchlib/src/vespa/searchlib/features/jarowinklerdistancefeature.h @@ -43,7 +43,7 @@ private: std::vector<fef::TermFieldHandle> _termFieldHandles; // The handles of all query terms. const fef::MatchData *_md; - void handle_bind_match_data(fef::MatchData &md) override; + void handle_bind_match_data(const fef::MatchData &md) override; }; /** diff --git a/searchlib/src/vespa/searchlib/features/matchcountfeature.cpp b/searchlib/src/vespa/searchlib/features/matchcountfeature.cpp index 51a7a952781..fd453e17eb1 100644 --- a/searchlib/src/vespa/searchlib/features/matchcountfeature.cpp +++ b/searchlib/src/vespa/searchlib/features/matchcountfeature.cpp @@ -36,7 +36,7 @@ MatchCountExecutor::execute(uint32_t docId) } void -MatchCountExecutor::handle_bind_match_data(MatchData &md) +MatchCountExecutor::handle_bind_match_data(const MatchData &md) { _md = &md; } diff --git a/searchlib/src/vespa/searchlib/features/matchcountfeature.h b/searchlib/src/vespa/searchlib/features/matchcountfeature.h index c4123016e16..eadb5064c57 100644 --- a/searchlib/src/vespa/searchlib/features/matchcountfeature.h +++ b/searchlib/src/vespa/searchlib/features/matchcountfeature.h @@ -18,7 +18,7 @@ private: std::vector<fef::TermFieldHandle> _handles; const fef::MatchData *_md; - virtual void handle_bind_match_data(fef::MatchData &md) override; + virtual void handle_bind_match_data(const fef::MatchData &md) override; public: MatchCountExecutor(uint32_t fieldId, const fef::IQueryEnvironment &env); diff --git a/searchlib/src/vespa/searchlib/features/matchesfeature.cpp b/searchlib/src/vespa/searchlib/features/matchesfeature.cpp index c2e36551648..f4788ee74c8 100644 --- a/searchlib/src/vespa/searchlib/features/matchesfeature.cpp +++ b/searchlib/src/vespa/searchlib/features/matchesfeature.cpp @@ -40,7 +40,7 @@ MatchesExecutor::execute(uint32_t docId) } void -MatchesExecutor::handle_bind_match_data(MatchData &md) +MatchesExecutor::handle_bind_match_data(const MatchData &md) { _md = &md; } diff --git a/searchlib/src/vespa/searchlib/features/matchesfeature.h b/searchlib/src/vespa/searchlib/features/matchesfeature.h index fe82929d2a7..c6366349dd9 100644 --- a/searchlib/src/vespa/searchlib/features/matchesfeature.h +++ b/searchlib/src/vespa/searchlib/features/matchesfeature.h @@ -18,7 +18,7 @@ private: std::vector<fef::TermFieldHandle> _handles; const fef::MatchData *_md; - void handle_bind_match_data(fef::MatchData &md) override; + void handle_bind_match_data(const fef::MatchData &md) override; public: MatchesExecutor(uint32_t fieldId, diff --git a/searchlib/src/vespa/searchlib/features/native_dot_product_feature.cpp b/searchlib/src/vespa/searchlib/features/native_dot_product_feature.cpp index 6986703eaf6..cb8136e8b7f 100644 --- a/searchlib/src/vespa/searchlib/features/native_dot_product_feature.cpp +++ b/searchlib/src/vespa/searchlib/features/native_dot_product_feature.cpp @@ -35,7 +35,7 @@ NativeDotProductExecutor::execute(uint32_t docId) } void -NativeDotProductExecutor::handle_bind_match_data(fef::MatchData &md) +NativeDotProductExecutor::handle_bind_match_data(const fef::MatchData &md) { _md = &md; } diff --git a/searchlib/src/vespa/searchlib/features/native_dot_product_feature.h b/searchlib/src/vespa/searchlib/features/native_dot_product_feature.h index cc29ecd5e6b..a71d23f3158 100644 --- a/searchlib/src/vespa/searchlib/features/native_dot_product_feature.h +++ b/searchlib/src/vespa/searchlib/features/native_dot_product_feature.h @@ -17,7 +17,7 @@ private: std::vector<Pair> _pairs; const fef::MatchData *_md; - void handle_bind_match_data(fef::MatchData &md) override; + void handle_bind_match_data(const fef::MatchData &md) override; public: NativeDotProductExecutor(const fef::IQueryEnvironment &env, uint32_t fieldId); diff --git a/searchlib/src/vespa/searchlib/features/nativeattributematchfeature.cpp b/searchlib/src/vespa/searchlib/features/nativeattributematchfeature.cpp index 4243bb83e03..1e6423f9de8 100644 --- a/searchlib/src/vespa/searchlib/features/nativeattributematchfeature.cpp +++ b/searchlib/src/vespa/searchlib/features/nativeattributematchfeature.cpp @@ -73,7 +73,7 @@ NativeAttributeMatchExecutorMulti::execute(uint32_t docId) } void -NativeAttributeMatchExecutorMulti::handle_bind_match_data(MatchData &md) +NativeAttributeMatchExecutorMulti::handle_bind_match_data(const MatchData &md) { _md = &md; } @@ -88,7 +88,7 @@ NativeAttributeMatchExecutorSingle::execute(uint32_t docId) } void -NativeAttributeMatchExecutorSingle::handle_bind_match_data(MatchData &md) +NativeAttributeMatchExecutorSingle::handle_bind_match_data(const MatchData &md) { _md = &md; } diff --git a/searchlib/src/vespa/searchlib/features/nativeattributematchfeature.h b/searchlib/src/vespa/searchlib/features/nativeattributematchfeature.h index 4ebdc871bb5..e47cbed0344 100644 --- a/searchlib/src/vespa/searchlib/features/nativeattributematchfeature.h +++ b/searchlib/src/vespa/searchlib/features/nativeattributematchfeature.h @@ -57,7 +57,7 @@ private: std::vector<CachedTermData> _queryTermData; const fef::MatchData *_md; - void handle_bind_match_data(fef::MatchData &md) override; + void handle_bind_match_data(const fef::MatchData &md) override; public: NativeAttributeMatchExecutorMulti(const Precomputed & setup) : _divisor(setup.second), _queryTermData(setup.first), _md(nullptr) { } void execute(uint32_t docId) override; @@ -69,7 +69,7 @@ private: CachedTermData _queryTermData; const fef::MatchData *_md; - void handle_bind_match_data(fef::MatchData &md) override; + void handle_bind_match_data(const fef::MatchData &md) override; public: NativeAttributeMatchExecutorSingle(const Precomputed & setup) : diff --git a/searchlib/src/vespa/searchlib/features/nativefieldmatchfeature.cpp b/searchlib/src/vespa/searchlib/features/nativefieldmatchfeature.cpp index e4b633d54dd..84ad17e1cb0 100644 --- a/searchlib/src/vespa/searchlib/features/nativefieldmatchfeature.cpp +++ b/searchlib/src/vespa/searchlib/features/nativefieldmatchfeature.cpp @@ -83,7 +83,7 @@ NativeFieldMatchExecutor::execute(uint32_t docId) } void -NativeFieldMatchExecutor::handle_bind_match_data(fef::MatchData &md) +NativeFieldMatchExecutor::handle_bind_match_data(const fef::MatchData &md) { _md = &md; } diff --git a/searchlib/src/vespa/searchlib/features/nativefieldmatchfeature.h b/searchlib/src/vespa/searchlib/features/nativefieldmatchfeature.h index d47dece1654..cb676e5b775 100644 --- a/searchlib/src/vespa/searchlib/features/nativefieldmatchfeature.h +++ b/searchlib/src/vespa/searchlib/features/nativefieldmatchfeature.h @@ -73,7 +73,7 @@ private: return table->get(index); } - virtual void handle_bind_match_data(fef::MatchData &md) override; + virtual void handle_bind_match_data(const fef::MatchData &md) override; public: NativeFieldMatchExecutor(const fef::IQueryEnvironment & env, diff --git a/searchlib/src/vespa/searchlib/features/nativeproximityfeature.cpp b/searchlib/src/vespa/searchlib/features/nativeproximityfeature.cpp index 3c92db6b937..a31d9207e05 100644 --- a/searchlib/src/vespa/searchlib/features/nativeproximityfeature.cpp +++ b/searchlib/src/vespa/searchlib/features/nativeproximityfeature.cpp @@ -97,7 +97,7 @@ NativeProximityExecutor::execute(uint32_t docId) } void -NativeProximityExecutor::handle_bind_match_data(fef::MatchData &md) +NativeProximityExecutor::handle_bind_match_data(const fef::MatchData &md) { _md = &md; } diff --git a/searchlib/src/vespa/searchlib/features/nativeproximityfeature.h b/searchlib/src/vespa/searchlib/features/nativeproximityfeature.h index cda6776d42e..f98fe2bfd22 100644 --- a/searchlib/src/vespa/searchlib/features/nativeproximityfeature.h +++ b/searchlib/src/vespa/searchlib/features/nativeproximityfeature.h @@ -62,7 +62,7 @@ private: feature_t calculateScoreForField(const FieldSetup & fs, uint32_t docId); feature_t calculateScoreForPair(const TermPair & pair, uint32_t fieldId, uint32_t docId); - virtual void handle_bind_match_data(fef::MatchData &md) override; + virtual void handle_bind_match_data(const fef::MatchData &md) override; public: NativeProximityExecutor(const fef::IQueryEnvironment & env, const NativeProximityParams & params); diff --git a/searchlib/src/vespa/searchlib/features/proximityfeature.cpp b/searchlib/src/vespa/searchlib/features/proximityfeature.cpp index e6303ace100..f625e30f378 100644 --- a/searchlib/src/vespa/searchlib/features/proximityfeature.cpp +++ b/searchlib/src/vespa/searchlib/features/proximityfeature.cpp @@ -51,7 +51,7 @@ ProximityExecutor::execute(uint32_t docId) } void -ProximityExecutor::handle_bind_match_data(fef::MatchData &md) +ProximityExecutor::handle_bind_match_data(const fef::MatchData &md) { _md = &md; } diff --git a/searchlib/src/vespa/searchlib/features/proximityfeature.h b/searchlib/src/vespa/searchlib/features/proximityfeature.h index 5a46bafe869..14a80628004 100644 --- a/searchlib/src/vespa/searchlib/features/proximityfeature.h +++ b/searchlib/src/vespa/searchlib/features/proximityfeature.h @@ -41,7 +41,7 @@ private: bool findBest(const fef::TermFieldMatchData &matchA, const fef::TermFieldMatchData &matchB); - void handle_bind_match_data(fef::MatchData &md) override; + void handle_bind_match_data(const fef::MatchData &md) override; }; /** diff --git a/searchlib/src/vespa/searchlib/features/querycompletenessfeature.cpp b/searchlib/src/vespa/searchlib/features/querycompletenessfeature.cpp index af44f75e33f..b4b6a1b0eb4 100644 --- a/searchlib/src/vespa/searchlib/features/querycompletenessfeature.cpp +++ b/searchlib/src/vespa/searchlib/features/querycompletenessfeature.cpp @@ -62,7 +62,7 @@ QueryCompletenessExecutor::execute(uint32_t docId) } void -QueryCompletenessExecutor::handle_bind_match_data(fef::MatchData &md) +QueryCompletenessExecutor::handle_bind_match_data(const fef::MatchData &md) { _md = &md; } diff --git a/searchlib/src/vespa/searchlib/features/querycompletenessfeature.h b/searchlib/src/vespa/searchlib/features/querycompletenessfeature.h index 5e30563f6ec..a5e6bbdc2bb 100644 --- a/searchlib/src/vespa/searchlib/features/querycompletenessfeature.h +++ b/searchlib/src/vespa/searchlib/features/querycompletenessfeature.h @@ -38,7 +38,7 @@ private: std::vector<fef::TermFieldHandle> _fieldHandles; const fef::MatchData *_md; - void handle_bind_match_data(fef::MatchData &md) override; + void handle_bind_match_data(const fef::MatchData &md) override; }; /** diff --git a/searchlib/src/vespa/searchlib/features/raw_score_feature.cpp b/searchlib/src/vespa/searchlib/features/raw_score_feature.cpp index 2fa59111d00..02e44e781d4 100644 --- a/searchlib/src/vespa/searchlib/features/raw_score_feature.cpp +++ b/searchlib/src/vespa/searchlib/features/raw_score_feature.cpp @@ -35,7 +35,7 @@ RawScoreExecutor::execute(uint32_t docId) } void -RawScoreExecutor::handle_bind_match_data(fef::MatchData &md) +RawScoreExecutor::handle_bind_match_data(const fef::MatchData &md) { _md = &md; } diff --git a/searchlib/src/vespa/searchlib/features/raw_score_feature.h b/searchlib/src/vespa/searchlib/features/raw_score_feature.h index 8bb310fa18e..0eceba16ffe 100644 --- a/searchlib/src/vespa/searchlib/features/raw_score_feature.h +++ b/searchlib/src/vespa/searchlib/features/raw_score_feature.h @@ -14,7 +14,7 @@ private: std::vector<fef::TermFieldHandle> _handles; const fef::MatchData *_md; - void handle_bind_match_data(fef::MatchData &md) override; + void handle_bind_match_data(const fef::MatchData &md) override; public: RawScoreExecutor(const fef::IQueryEnvironment &env, uint32_t fieldId); void execute(uint32_t docId) override; diff --git a/searchlib/src/vespa/searchlib/features/reverseproximityfeature.cpp b/searchlib/src/vespa/searchlib/features/reverseproximityfeature.cpp index 0f35e1becf8..c27936332d2 100644 --- a/searchlib/src/vespa/searchlib/features/reverseproximityfeature.cpp +++ b/searchlib/src/vespa/searchlib/features/reverseproximityfeature.cpp @@ -90,7 +90,7 @@ ReverseProximityExecutor::execute(uint32_t docId) } void -ReverseProximityExecutor::handle_bind_match_data(fef::MatchData &md) +ReverseProximityExecutor::handle_bind_match_data(const fef::MatchData &md) { _md = &md; } diff --git a/searchlib/src/vespa/searchlib/features/reverseproximityfeature.h b/searchlib/src/vespa/searchlib/features/reverseproximityfeature.h index c16448435f2..776dde39f77 100644 --- a/searchlib/src/vespa/searchlib/features/reverseproximityfeature.h +++ b/searchlib/src/vespa/searchlib/features/reverseproximityfeature.h @@ -39,7 +39,7 @@ private: fef::TermFieldHandle _termB; // Handle to the second query term. const fef::MatchData *_md; - void handle_bind_match_data(fef::MatchData &md) override; + void handle_bind_match_data(const fef::MatchData &md) override; }; /** diff --git a/searchlib/src/vespa/searchlib/features/subqueries_feature.cpp b/searchlib/src/vespa/searchlib/features/subqueries_feature.cpp index 4bbb7af9dfb..6c52b6edb76 100644 --- a/searchlib/src/vespa/searchlib/features/subqueries_feature.cpp +++ b/searchlib/src/vespa/searchlib/features/subqueries_feature.cpp @@ -37,7 +37,7 @@ void SubqueriesExecutor::execute(uint32_t docId) { } void -SubqueriesExecutor::handle_bind_match_data(fef::MatchData &md) +SubqueriesExecutor::handle_bind_match_data(const fef::MatchData &md) { _md = &md; } diff --git a/searchlib/src/vespa/searchlib/features/subqueries_feature.h b/searchlib/src/vespa/searchlib/features/subqueries_feature.h index 60bec51cb79..136d8d90f94 100644 --- a/searchlib/src/vespa/searchlib/features/subqueries_feature.h +++ b/searchlib/src/vespa/searchlib/features/subqueries_feature.h @@ -12,7 +12,7 @@ class SubqueriesExecutor : public fef::FeatureExecutor { std::vector<fef::TermFieldHandle> _handles; const fef::MatchData *_md; - void handle_bind_match_data(fef::MatchData &md) override; + void handle_bind_match_data(const fef::MatchData &md) override; public: SubqueriesExecutor(const fef::IQueryEnvironment &env, uint32_t fieldId); void execute(uint32_t docId) override; diff --git a/searchlib/src/vespa/searchlib/features/term_field_md_feature.cpp b/searchlib/src/vespa/searchlib/features/term_field_md_feature.cpp index 97d267071cf..8b0eecd9cff 100644 --- a/searchlib/src/vespa/searchlib/features/term_field_md_feature.cpp +++ b/searchlib/src/vespa/searchlib/features/term_field_md_feature.cpp @@ -65,7 +65,7 @@ TermFieldMdExecutor::execute(uint32_t docId) } void -TermFieldMdExecutor::handle_bind_match_data(MatchData &md) +TermFieldMdExecutor::handle_bind_match_data(const MatchData &md) { _md = &md; } diff --git a/searchlib/src/vespa/searchlib/features/term_field_md_feature.h b/searchlib/src/vespa/searchlib/features/term_field_md_feature.h index f819989fc08..df9ab982dcb 100644 --- a/searchlib/src/vespa/searchlib/features/term_field_md_feature.h +++ b/searchlib/src/vespa/searchlib/features/term_field_md_feature.h @@ -21,7 +21,7 @@ class TermFieldMdExecutor : public fef::FeatureExecutor { const fef::MatchData *_md; void execute(uint32_t docId) override; - void handle_bind_match_data(fef::MatchData &md) override; + void handle_bind_match_data(const fef::MatchData &md) override; public: TermFieldMdExecutor(const fef::IQueryEnvironment &env, uint32_t fieldId); }; diff --git a/searchlib/src/vespa/searchlib/features/termdistancefeature.cpp b/searchlib/src/vespa/searchlib/features/termdistancefeature.cpp index a67c6a0be7c..0408f6d4b75 100644 --- a/searchlib/src/vespa/searchlib/features/termdistancefeature.cpp +++ b/searchlib/src/vespa/searchlib/features/termdistancefeature.cpp @@ -42,7 +42,7 @@ TermDistanceExecutor::execute(uint32_t docId) } void -TermDistanceExecutor::handle_bind_match_data(fef::MatchData &md) +TermDistanceExecutor::handle_bind_match_data(const fef::MatchData &md) { _md = &md; } diff --git a/searchlib/src/vespa/searchlib/features/termdistancefeature.h b/searchlib/src/vespa/searchlib/features/termdistancefeature.h index 5c4542a1f0d..8e153072bc2 100644 --- a/searchlib/src/vespa/searchlib/features/termdistancefeature.h +++ b/searchlib/src/vespa/searchlib/features/termdistancefeature.h @@ -31,7 +31,7 @@ private: QueryTerm _termB; const fef::MatchData *_md; - virtual void handle_bind_match_data(fef::MatchData &md) override; + virtual void handle_bind_match_data(const fef::MatchData &md) override; public: TermDistanceExecutor(const fef::IQueryEnvironment & env, diff --git a/searchlib/src/vespa/searchlib/features/termeditdistancefeature.cpp b/searchlib/src/vespa/searchlib/features/termeditdistancefeature.cpp index 05bdbb4475b..5990d62cb25 100644 --- a/searchlib/src/vespa/searchlib/features/termeditdistancefeature.cpp +++ b/searchlib/src/vespa/searchlib/features/termeditdistancefeature.cpp @@ -156,7 +156,7 @@ TermEditDistanceExecutor::execute(uint32_t docId) } void -TermEditDistanceExecutor::handle_bind_match_data(fef::MatchData &md) +TermEditDistanceExecutor::handle_bind_match_data(const fef::MatchData &md) { _md = &md; } diff --git a/searchlib/src/vespa/searchlib/features/termeditdistancefeature.h b/searchlib/src/vespa/searchlib/features/termeditdistancefeature.h index 07d625726e0..ea5974495ea 100644 --- a/searchlib/src/vespa/searchlib/features/termeditdistancefeature.h +++ b/searchlib/src/vespa/searchlib/features/termeditdistancefeature.h @@ -88,7 +88,7 @@ private: */ void logRow(const std::vector<TedCell> &row, size_t numCols); - void handle_bind_match_data(fef::MatchData &md) override; + void handle_bind_match_data(const fef::MatchData &md) override; private: const TermEditDistanceConfig &_config; // The config for this executor. diff --git a/searchlib/src/vespa/searchlib/features/text_similarity_feature.cpp b/searchlib/src/vespa/searchlib/features/text_similarity_feature.cpp index 31876e4f1c9..a50f042b552 100644 --- a/searchlib/src/vespa/searchlib/features/text_similarity_feature.cpp +++ b/searchlib/src/vespa/searchlib/features/text_similarity_feature.cpp @@ -158,7 +158,7 @@ TextSimilarityExecutor::execute(uint32_t docId) } void -TextSimilarityExecutor::handle_bind_match_data(fef::MatchData &md) +TextSimilarityExecutor::handle_bind_match_data(const fef::MatchData &md) { _md = &md; } diff --git a/searchlib/src/vespa/searchlib/features/text_similarity_feature.h b/searchlib/src/vespa/searchlib/features/text_similarity_feature.h index e948589fc97..a660f3e3e30 100644 --- a/searchlib/src/vespa/searchlib/features/text_similarity_feature.h +++ b/searchlib/src/vespa/searchlib/features/text_similarity_feature.h @@ -35,7 +35,7 @@ private: vespalib::PriorityQueue<Item> _queue; const fef::MatchData *_md; - void handle_bind_match_data(fef::MatchData &md) override; + void handle_bind_match_data(const fef::MatchData &md) override; public: TextSimilarityExecutor(const fef::IQueryEnvironment &env, uint32_t field_id); diff --git a/searchlib/src/vespa/searchlib/fef/featureexecutor.cpp b/searchlib/src/vespa/searchlib/fef/featureexecutor.cpp index f8268d66e47..02d69fbc5ca 100644 --- a/searchlib/src/vespa/searchlib/fef/featureexecutor.cpp +++ b/searchlib/src/vespa/searchlib/fef/featureexecutor.cpp @@ -28,7 +28,7 @@ FeatureExecutor::handle_bind_outputs(vespalib::ArrayRef<NumberOrObject>) } void -FeatureExecutor::handle_bind_match_data(MatchData &) +FeatureExecutor::handle_bind_match_data(const MatchData &) { } @@ -47,7 +47,7 @@ FeatureExecutor::bind_outputs(vespalib::ArrayRef<NumberOrObject> outputs) } void -FeatureExecutor::bind_match_data(MatchData &md) +FeatureExecutor::bind_match_data(const MatchData &md) { handle_bind_match_data(md); } diff --git a/searchlib/src/vespa/searchlib/fef/featureexecutor.h b/searchlib/src/vespa/searchlib/fef/featureexecutor.h index 1d8e4bff0c4..c8219fada3b 100644 --- a/searchlib/src/vespa/searchlib/fef/featureexecutor.h +++ b/searchlib/src/vespa/searchlib/fef/featureexecutor.h @@ -98,7 +98,7 @@ private: protected: virtual void handle_bind_inputs(vespalib::ConstArrayRef<LazyValue> inputs); virtual void handle_bind_outputs(vespalib::ArrayRef<NumberOrObject> outputs); - virtual void handle_bind_match_data(MatchData &md); + virtual void handle_bind_match_data(const MatchData &md); /** * Execute this feature executor for the given document. @@ -117,7 +117,7 @@ public: // bind order per executor: inputs, outputs, match_data void bind_inputs(vespalib::ConstArrayRef<LazyValue> inputs); void bind_outputs(vespalib::ArrayRef<NumberOrObject> outputs); - void bind_match_data(MatchData &md); + void bind_match_data(const MatchData &md); const Inputs &inputs() const { return _inputs; } const Outputs &outputs() const { return _outputs; } diff --git a/searchlib/src/vespa/searchlib/fef/featureoverrider.cpp b/searchlib/src/vespa/searchlib/fef/featureoverrider.cpp index ad623c65c07..5209065fd62 100644 --- a/searchlib/src/vespa/searchlib/fef/featureoverrider.cpp +++ b/searchlib/src/vespa/searchlib/fef/featureoverrider.cpp @@ -40,7 +40,7 @@ FeatureOverrider::execute(uint32_t docId) } void -FeatureOverrider::handle_bind_match_data(MatchData &md) +FeatureOverrider::handle_bind_match_data(const MatchData &md) { _executor.bind_match_data(md); } diff --git a/searchlib/src/vespa/searchlib/fef/featureoverrider.h b/searchlib/src/vespa/searchlib/fef/featureoverrider.h index 3f2075ffbcb..89c397abfdb 100644 --- a/searchlib/src/vespa/searchlib/fef/featureoverrider.h +++ b/searchlib/src/vespa/searchlib/fef/featureoverrider.h @@ -24,7 +24,7 @@ private: uint32_t _outputIdx; feature_t _value; - virtual void handle_bind_match_data(MatchData &md) override; + virtual void handle_bind_match_data(const MatchData &md) override; virtual void handle_bind_inputs(vespalib::ConstArrayRef<LazyValue> inputs) override; virtual void handle_bind_outputs(vespalib::ArrayRef<NumberOrObject> outputs) override; diff --git a/searchlib/src/vespa/searchlib/fef/matchdata.cpp b/searchlib/src/vespa/searchlib/fef/matchdata.cpp index 4ce2a7c9299..0c589749112 100644 --- a/searchlib/src/vespa/searchlib/fef/matchdata.cpp +++ b/searchlib/src/vespa/searchlib/fef/matchdata.cpp @@ -11,6 +11,15 @@ MatchData::MatchData(const Params &cparams) { } +void +MatchData::soft_reset() +{ + for (auto &tfmd: _termFields) { + tfmd.resetOnlyDocId(TermFieldMatchData::invalidId()).tagAsNeeded(); + } + _termwise_limit = 1.0; +} + MatchData::UP MatchData::makeTestInstance(uint32_t numTermFields, uint32_t fieldIdLimit) { diff --git a/searchlib/src/vespa/searchlib/fef/matchdata.h b/searchlib/src/vespa/searchlib/fef/matchdata.h index 472b34a823f..5b06f4d37e0 100644 --- a/searchlib/src/vespa/searchlib/fef/matchdata.h +++ b/searchlib/src/vespa/searchlib/fef/matchdata.h @@ -58,6 +58,15 @@ public: **/ explicit MatchData(const Params &cparams); + /** + * Reset this match data in such a way that it can be re-used with + * either the same search iterator tree or with a new search + * iterator tree where the only difference in interaction with the + * match data is which terms are unpacked. Note that this will + * reset some properties, but not all. Use with caution. + **/ + void soft_reset(); + MatchData(const MatchData &rhs) = delete; MatchData & operator=(const MatchData &rhs) = delete; diff --git a/searchlib/src/vespa/searchlib/fef/rank_program.cpp b/searchlib/src/vespa/searchlib/fef/rank_program.cpp index fa4582d46b7..33a65b48a1c 100644 --- a/searchlib/src/vespa/searchlib/fef/rank_program.cpp +++ b/searchlib/src/vespa/searchlib/fef/rank_program.cpp @@ -109,7 +109,7 @@ RankProgram::run_const(FeatureExecutor *executor) } void -RankProgram::unbox(BlueprintResolver::FeatureRef seed) +RankProgram::unbox(BlueprintResolver::FeatureRef seed, const MatchData &md) { FeatureExecutor *input_executor = _executors[seed.executor]; const NumberOrObject *input_value = input_executor->outputs().get_raw(seed.output); @@ -122,7 +122,7 @@ RankProgram::unbox(BlueprintResolver::FeatureRef seed) FeatureExecutor &unboxer = _hot_stash.create<UnboxingExecutor>(); unboxer.bind_inputs(inputs); unboxer.bind_outputs(outputs); - unboxer.bind_match_data(*_match_data); + unboxer.bind_match_data(md); _unboxed_seeds.emplace(input_value, LazyValue(&outputs[0], &unboxer)); } } @@ -153,7 +153,6 @@ RankProgram::resolve(const BlueprintResolver::FeatureMap &features, bool unbox_s RankProgram::RankProgram(BlueprintResolver::SP resolver) : _resolver(resolver), - _match_data(), _hot_stash(32768), _cold_stash(), _executors(), @@ -165,12 +164,11 @@ RankProgram::RankProgram(BlueprintResolver::SP resolver) RankProgram::~RankProgram() {} void -RankProgram::setup(const MatchDataLayout &mdl_in, +RankProgram::setup(const MatchData &md, const IQueryEnvironment &queryEnv, const Properties &featureOverrides) { assert(_executors.empty()); - _match_data = mdl_in.createMatchData(); std::vector<Override> overrides = prepare_overrides(_resolver->getFeatureMap(), featureOverrides); auto override = overrides.begin(); auto override_end = overrides.end(); @@ -204,7 +202,7 @@ RankProgram::setup(const MatchDataLayout &mdl_in, } executor->bind_inputs(inputs); executor->bind_outputs(outputs); - executor->bind_match_data(*_match_data); + executor->bind_match_data(md); _executors.push_back(executor); if (is_const) { run_const(executor); @@ -213,7 +211,7 @@ RankProgram::setup(const MatchDataLayout &mdl_in, for (const auto &seed_entry: _resolver->getSeedMap()) { auto seed = seed_entry.second; if (specs[seed.executor].output_types[seed.output]) { - unbox(seed); + unbox(seed, md); } } assert(_executors.size() == specs.size()); diff --git a/searchlib/src/vespa/searchlib/fef/rank_program.h b/searchlib/src/vespa/searchlib/fef/rank_program.h index 7cf593e4a4e..3a92fc874a4 100644 --- a/searchlib/src/vespa/searchlib/fef/rank_program.h +++ b/searchlib/src/vespa/searchlib/fef/rank_program.h @@ -6,7 +6,6 @@ #include "featureexecutor.h" #include "properties.h" #include "matchdata.h" -#include "matchdatalayout.h" #include "feature_resolver.h" #include <vespa/vespalib/stllike/string.h> #include <vespa/vespalib/util/array.h> @@ -21,11 +20,10 @@ namespace fef { * values. In order to access (and thereby calculate) output features * you typically use the get_seeds function to resolve the predefined * set of output features. Each feature value will be wrapped in a - * LazyValue object that can be realized for a specific docid. The - * rank program also owns the MatchData used to store unpacked - * term-field match information. Note that you need unpack any - * relevant posting information into the MatchData object before - * trying to resolve lazy values. + * LazyValue object that can be realized for a specific docid. Note + * that you need unpack any relevant posting information into the + * MatchData object passed to the setup function before trying to + * resolve lazy values. **/ class RankProgram { @@ -37,7 +35,6 @@ private: using ValueSet = std::set<const NumberOrObject *>; BlueprintResolver::SP _resolver; - MatchData::UP _match_data; vespalib::Stash _hot_stash; vespalib::Stash _cold_stash; std::vector<FeatureExecutor *> _executors; @@ -47,7 +44,7 @@ private: bool check_const(const NumberOrObject *value) const { return (_is_const.count(value) == 1); } bool check_const(FeatureExecutor *executor, const std::vector<BlueprintResolver::FeatureRef> &inputs) const; void run_const(FeatureExecutor *executor); - void unbox(BlueprintResolver::FeatureRef seed); + void unbox(BlueprintResolver::FeatureRef seed, const MatchData &md); FeatureResolver resolve(const BlueprintResolver::FeatureMap &features, bool unbox_seeds) const; public: @@ -66,21 +63,13 @@ public: /** * Set up this rank program by creating the needed feature * executors and wiring them together. This function will also - * create the MatchData to be used for iterator unpacking as well - * as pre-calculating all constant features. + * pre-calculate all constant features. **/ - void setup(const MatchDataLayout &mdl, + void setup(const MatchData &md, const IQueryEnvironment &queryEnv, const Properties &featureOverrides = Properties()); /** - * Expose the MatchData used when creating search iterators as it - * is where all iterators should unpack their match information. - **/ - MatchData &match_data() { return *_match_data; } - const MatchData &match_data() const { return *_match_data; } - - /** * Obtain the names and storage locations of all seed features for * this rank program. Programs for ranking phases will only have a * single seed while programs used for summary features or diff --git a/searchlib/src/vespa/searchlib/fef/termfieldmatchdata.h b/searchlib/src/vespa/searchlib/fef/termfieldmatchdata.h index 2d91764f7c9..018af889557 100644 --- a/searchlib/src/vespa/searchlib/fef/termfieldmatchdata.h +++ b/searchlib/src/vespa/searchlib/fef/termfieldmatchdata.h @@ -258,6 +258,13 @@ public: } /** + * Tag that this instance is used for ranking. + */ + void tagAsNeeded() { + _fieldId = _fieldId & ~0x2000; + } + + /** * Special docId value indicating that no data has been saved yet. * This should match (or be above) endId() in search::queryeval::SearchIterator. * diff --git a/searchlib/src/vespa/searchlib/fef/test/featuretest.cpp b/searchlib/src/vespa/searchlib/fef/test/featuretest.cpp index 2652abf97a4..e5a6232ed54 100644 --- a/searchlib/src/vespa/searchlib/fef/test/featuretest.cpp +++ b/searchlib/src/vespa/searchlib/fef/test/featuretest.cpp @@ -22,6 +22,7 @@ FeatureTest::FeatureTest(BlueprintFactory &factory, _layout(layout), _overrides(overrides), _resolver(new BlueprintResolver(factory, indexEnv)), + _match_data(_layout.createMatchData()), _rankProgram(new RankProgram(_resolver)), _doneSetup(false) { @@ -42,6 +43,7 @@ FeatureTest::FeatureTest(BlueprintFactory &factory, _layout(layout), _overrides(overrides), _resolver(new BlueprintResolver(factory, indexEnv)), + _match_data(_layout.createMatchData()), _rankProgram(new RankProgram(_resolver)), _doneSetup(false) { @@ -68,7 +70,7 @@ FeatureTest::setup() return false; } - _rankProgram->setup(_layout, _queryEnv, _overrides); + _rankProgram->setup(*_match_data, _queryEnv, _overrides); _doneSetup = true; return true; } @@ -77,7 +79,7 @@ MatchDataBuilder::UP FeatureTest::createMatchDataBuilder() { if (_doneSetup) { - return MatchDataBuilder::UP(new MatchDataBuilder(_queryEnv, _rankProgram->match_data())); + return MatchDataBuilder::UP(new MatchDataBuilder(_queryEnv, *_match_data)); } LOG(warning, "Match data not initialized."); return MatchDataBuilder::UP(); @@ -135,6 +137,7 @@ void FeatureTest::clear() { _resolver = BlueprintResolver::SP(new BlueprintResolver(_factory, _indexEnv)); + _match_data = _layout.createMatchData(); _rankProgram.reset(new RankProgram(_resolver)); _doneSetup = false; } diff --git a/searchlib/src/vespa/searchlib/fef/test/featuretest.h b/searchlib/src/vespa/searchlib/fef/test/featuretest.h index 2bce63d357d..e354884c110 100644 --- a/searchlib/src/vespa/searchlib/fef/test/featuretest.h +++ b/searchlib/src/vespa/searchlib/fef/test/featuretest.h @@ -116,6 +116,7 @@ private: MatchDataLayout &_layout; const Properties &_overrides; BlueprintResolver::SP _resolver; + MatchData::UP _match_data; RankProgram::UP _rankProgram; bool _doneSetup; diff --git a/searchlib/src/vespa/searchlib/index/bitvectorkeys.h b/searchlib/src/vespa/searchlib/index/bitvectorkeys.h index a916d07228a..a6cd0022371 100644 --- a/searchlib/src/vespa/searchlib/index/bitvectorkeys.h +++ b/searchlib/src/vespa/searchlib/index/bitvectorkeys.h @@ -1,6 +1,4 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// Copyright (C) 1998-2003 Fast Search & Transfer ASA -// Copyright (C) 2003 Overture Services Norway AS #pragma once diff --git a/searchlib/src/vespa/searchlib/parsequery/simplequerystack.cpp b/searchlib/src/vespa/searchlib/parsequery/simplequerystack.cpp index 6274bd083de..a7b41476ec6 100644 --- a/searchlib/src/vespa/searchlib/parsequery/simplequerystack.cpp +++ b/searchlib/src/vespa/searchlib/parsequery/simplequerystack.cpp @@ -1,12 +1,5 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -/** - * Creation date: 2000-05-15 - * Implementation of the simple query stack. - * - * Copyright (C) 1997-2003 Fast Search & Transfer ASA - * Copyright (C) 2003 Overture Services Norway AS - * ALL RIGHTS RESERVED - */ + #include "simplequerystack.h" #include <vespa/vespalib/util/compress.h> #include <vespa/vespalib/objects/nbo.h> diff --git a/searchlib/src/vespa/searchlib/parsequery/simplequerystack.h b/searchlib/src/vespa/searchlib/parsequery/simplequerystack.h index 1defce4c757..a9d99ca93f8 100644 --- a/searchlib/src/vespa/searchlib/parsequery/simplequerystack.h +++ b/searchlib/src/vespa/searchlib/parsequery/simplequerystack.h @@ -1,13 +1,5 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -/** - * Creation date: 2000-05-15 - * - * Declaration of the SimpleQueryStack - * - * Copyright (C) 1997-2003 Fast Search & Transfer ASA - * Copyright (C) 2003 Overture Services Norway AS - * ALL RIGHTS RESERVED - */ + #pragma once #include <vespa/searchlib/parsequery/parse.h> diff --git a/searchlib/src/vespa/searchlib/parsequery/stackdumpiterator.cpp b/searchlib/src/vespa/searchlib/parsequery/stackdumpiterator.cpp index d6a4181b1de..ec34b2d3a84 100644 --- a/searchlib/src/vespa/searchlib/parsequery/stackdumpiterator.cpp +++ b/searchlib/src/vespa/searchlib/parsequery/stackdumpiterator.cpp @@ -1,11 +1,5 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -/** - * Implementation of the simple query stack dump iterator. - * - * Copyright (C) 1997-2003 Fast Search & Transfer ASA - * Copyright (C) 2003 Overture Services Norway AS - * ALL RIGHTS RESERVED - */ + #include "stackdumpiterator.h" #include <vespa/vespalib/util/compress.h> #include <vespa/vespalib/objects/nbo.h> diff --git a/searchlib/src/vespa/searchlib/parsequery/stackdumpiterator.h b/searchlib/src/vespa/searchlib/parsequery/stackdumpiterator.h index ee0aec4b297..dbb4becc819 100644 --- a/searchlib/src/vespa/searchlib/parsequery/stackdumpiterator.h +++ b/searchlib/src/vespa/searchlib/parsequery/stackdumpiterator.h @@ -1,11 +1,5 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -/** - * Declaration of the SimpleQueryStack dump iterator - * - * Copyright (C) 1997-2003 Fast Search & Transfer ASA - * Copyright (C) 2003 Overture Services Norway AS - * ALL RIGHTS RESERVED - */ + #pragma once #include <vespa/searchlib/parsequery/parse.h> diff --git a/searchlib/src/vespa/searchlib/predicate/predicate_hash.h b/searchlib/src/vespa/searchlib/predicate/predicate_hash.h index 5b295e8bb41..938b1bc5542 100644 --- a/searchlib/src/vespa/searchlib/predicate/predicate_hash.h +++ b/searchlib/src/vespa/searchlib/predicate/predicate_hash.h @@ -7,7 +7,7 @@ namespace search { namespace predicate { /** - * Hash function coming from the RISE code base, used in boolean search. + * Hash function used for predicate fields. */ struct PredicateHash { static uint64_t hash64(vespalib::stringref aKey) { diff --git a/searchlib/src/vespa/searchlib/queryeval/iterators.h b/searchlib/src/vespa/searchlib/queryeval/iterators.h index 7dfed494d69..79c91f6be39 100644 --- a/searchlib/src/vespa/searchlib/queryeval/iterators.h +++ b/searchlib/src/vespa/searchlib/queryeval/iterators.h @@ -1,6 +1,4 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// Copyright (C) 2002-2003 Fast Search & Transfer ASA -// Copyright (C) 2003 Overture Services Norway AS #pragma once diff --git a/searchlib/src/vespa/searchlib/queryeval/termwise_search.cpp b/searchlib/src/vespa/searchlib/queryeval/termwise_search.cpp index 13b1c8f1da6..e7df5aa6bae 100644 --- a/searchlib/src/vespa/searchlib/queryeval/termwise_search.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/termwise_search.cpp @@ -11,17 +11,27 @@ template <bool IS_STRICT> struct TermwiseSearch : public SearchIterator { SearchIterator::UP search; - BitVector::UP result; + BitVector::UP result; + uint32_t my_beginid; + uint32_t my_first_hit; + + bool same_range(uint32_t beginid, uint32_t endid) const { + return ((beginid == my_beginid) && endid == getEndId()); + } TermwiseSearch(SearchIterator::UP search_in) - : search(std::move(search_in)), result() {} + : search(std::move(search_in)), result(), my_beginid(0), my_first_hit(0) {} Trinary is_strict() const override { return IS_STRICT ? Trinary::True : Trinary::False; } void initRange(uint32_t beginid, uint32_t endid) override { - SearchIterator::initRange(beginid, endid); - search->initRange(beginid, endid); - setDocId(std::max(getDocId(), search->getDocId())); - result = search->get_hits(beginid); + if (!same_range(beginid, endid)) { + my_beginid = beginid; + SearchIterator::initRange(beginid, endid); + search->initRange(beginid, endid); + my_first_hit = std::max(getDocId(), search->getDocId()); + result = search->get_hits(beginid); + } + setDocId(my_first_hit); } void doSeek(uint32_t docid) override { if (__builtin_expect(isAtEnd(docid), false)) { diff --git a/searchlib/src/vespa/searchlib/test/datastore/memstats.h b/searchlib/src/vespa/searchlib/test/datastore/memstats.h index 08960a2bb89..c7cc4be9a92 100644 --- a/searchlib/src/vespa/searchlib/test/datastore/memstats.h +++ b/searchlib/src/vespa/searchlib/test/datastore/memstats.h @@ -1,5 +1,4 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// Copyright 2017 Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once diff --git a/searchlib/src/vespa/searchlib/util/comprbuffer.h b/searchlib/src/vespa/searchlib/util/comprbuffer.h index bb568073cbd..61928976acc 100644 --- a/searchlib/src/vespa/searchlib/util/comprbuffer.h +++ b/searchlib/src/vespa/searchlib/util/comprbuffer.h @@ -1,6 +1,4 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// Copyright (C) 1999-2003 Fast Search & Transfer ASA -// Copyright (C) 2003 Overture Services Norway AS #pragma once diff --git a/searchlib/src/vespa/searchlib/util/comprfile.cpp b/searchlib/src/vespa/searchlib/util/comprfile.cpp index 194bed95d58..8b17fd85753 100644 --- a/searchlib/src/vespa/searchlib/util/comprfile.cpp +++ b/searchlib/src/vespa/searchlib/util/comprfile.cpp @@ -1,6 +1,4 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// Copyright (C) 2002-2003 Fast Search & Transfer ASA -// Copyright (C) 2003 Overture Services Norway AS #include "comprfile.h" #include <vespa/vespalib/objects/nbostream.h> diff --git a/searchlib/src/vespa/searchlib/util/dirtraverse.cpp b/searchlib/src/vespa/searchlib/util/dirtraverse.cpp index 57c7eee164f..c414e9c82fd 100644 --- a/searchlib/src/vespa/searchlib/util/dirtraverse.cpp +++ b/searchlib/src/vespa/searchlib/util/dirtraverse.cpp @@ -1,6 +1,4 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// Copyright (C) 2002-2003 Fast Search & Transfer ASA -// Copyright (C) 2003 Overture Services Norway AS #include "dirtraverse.h" #include <vespa/fastos/file.h> diff --git a/searchlib/src/vespa/searchlib/util/dirtraverse.h b/searchlib/src/vespa/searchlib/util/dirtraverse.h index ef37cfe871c..f2bf9cd6ffc 100644 --- a/searchlib/src/vespa/searchlib/util/dirtraverse.h +++ b/searchlib/src/vespa/searchlib/util/dirtraverse.h @@ -1,6 +1,4 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// Copyright (C) 2002-2003 Fast Search & Transfer ASA -// Copyright (C) 2003 Overture Services Norway AS #pragma once diff --git a/searchlib/src/vespa/searchlib/util/filekit.cpp b/searchlib/src/vespa/searchlib/util/filekit.cpp index 137210eb758..210a9c30134 100644 --- a/searchlib/src/vespa/searchlib/util/filekit.cpp +++ b/searchlib/src/vespa/searchlib/util/filekit.cpp @@ -1,6 +1,4 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// Copyright (C) 1998-2003 Fast Search & Transfer ASA -// Copyright (C) 2003 Overture Services Norway AS #include <vespa/searchlib/util/filekit.h> #include <vespa/vespalib/util/error.h> diff --git a/searchlib/src/vespa/searchlib/util/filekit.h b/searchlib/src/vespa/searchlib/util/filekit.h index 0c755ab50e5..70acf19c70c 100644 --- a/searchlib/src/vespa/searchlib/util/filekit.h +++ b/searchlib/src/vespa/searchlib/util/filekit.h @@ -1,6 +1,4 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// Copyright (C) 1998-2003 Fast Search & Transfer ASA -// Copyright (C) 2003 Overture Services Norway AS #pragma once diff --git a/searchlib/src/vespa/searchlib/util/rawbuf.cpp b/searchlib/src/vespa/searchlib/util/rawbuf.cpp index 7a23614df73..cf019014fd1 100644 --- a/searchlib/src/vespa/searchlib/util/rawbuf.cpp +++ b/searchlib/src/vespa/searchlib/util/rawbuf.cpp @@ -1,6 +1,4 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// Copyright (C) 1998-2003 Fast Search & Transfer ASA -// Copyright (C) 2003 Overture Services Norway AS #include "rawbuf.h" #include <vespa/vespalib/util/compress.h> diff --git a/searchlib/src/vespa/searchlib/util/rawbuf.h b/searchlib/src/vespa/searchlib/util/rawbuf.h index 123ceeaf5f7..ffebd035950 100644 --- a/searchlib/src/vespa/searchlib/util/rawbuf.h +++ b/searchlib/src/vespa/searchlib/util/rawbuf.h @@ -1,6 +1,4 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// Copyright (C) 1998-2003 Fast Search & Transfer ASA -// Copyright (C) 2003 Overture Services Norway AS #pragma once diff --git a/searchlib/src/vespa/searchlib/util/sort.h b/searchlib/src/vespa/searchlib/util/sort.h index 2362c219114..cc6eea32c6e 100644 --- a/searchlib/src/vespa/searchlib/util/sort.h +++ b/searchlib/src/vespa/searchlib/util/sort.h @@ -1,7 +1,4 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// Copyright (C) 1998-2003 Fast Search & Transfer ASA -// Copyright (C) 2003 Overture Services Norway AS - #pragma once diff --git a/searchlib/src/vespa/searchlib/util/stringenum.cpp b/searchlib/src/vespa/searchlib/util/stringenum.cpp index 2a29754443a..60238c32cc6 100644 --- a/searchlib/src/vespa/searchlib/util/stringenum.cpp +++ b/searchlib/src/vespa/searchlib/util/stringenum.cpp @@ -1,6 +1,4 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// Copyright (C) 2001-2003 Fast Search & Transfer ASA -// Copyright (C) 2003 Overture Services Norway AS #include "stringenum.h" #include <vespa/fastlib/io/bufferedfile.h> diff --git a/searchlib/src/vespa/searchlib/util/stringenum.h b/searchlib/src/vespa/searchlib/util/stringenum.h index 72040e5399c..44b3afca539 100644 --- a/searchlib/src/vespa/searchlib/util/stringenum.h +++ b/searchlib/src/vespa/searchlib/util/stringenum.h @@ -1,6 +1,4 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// Copyright (C) 2001-2003 Fast Search & Transfer ASA -// Copyright (C) 2003 Overture Services Norway AS #pragma once diff --git a/searchlib/src/vespa/searchlib/util/url.cpp b/searchlib/src/vespa/searchlib/util/url.cpp index 9c9a5ca5b51..6959f1fdff7 100644 --- a/searchlib/src/vespa/searchlib/util/url.cpp +++ b/searchlib/src/vespa/searchlib/util/url.cpp @@ -1,6 +1,4 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// Copyright (C) 2000-2003 Fast Search & Transfer ASA -// Copyright (C) 2003 Overture Services Norway AS #include "url.h" diff --git a/searchlib/src/vespa/searchlib/util/url.h b/searchlib/src/vespa/searchlib/util/url.h index 3bb81c266b7..6cf6f38af3b 100644 --- a/searchlib/src/vespa/searchlib/util/url.h +++ b/searchlib/src/vespa/searchlib/util/url.h @@ -1,6 +1,5 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// Copyright (C) 2000-2003 Fast Search & Transfer ASA -// Copyright (C) 2003 Overture Services Norway AS + #pragma once #ifndef MAX_URL_LEN diff --git a/searchsummary/src/tests/docsumformat/docsum-parse.cpp b/searchsummary/src/tests/docsumformat/docsum-parse.cpp index bc53d0f7d0d..d3862913261 100644 --- a/searchsummary/src/tests/docsumformat/docsum-parse.cpp +++ b/searchsummary/src/tests/docsumformat/docsum-parse.cpp @@ -1,7 +1,4 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// Copyright (C) 2001-2003 Fast Search & Transfer ASA -// Copyright (C) 2003 Overture Services Norway AS - #include <vespa/log/log.h> LOG_SETUP("docsum-parse"); diff --git a/searchsummary/src/tests/extractkeywords/extractkeywordstest.cpp b/searchsummary/src/tests/extractkeywords/extractkeywordstest.cpp index 07aa3e0fd14..eac6f65a48e 100644 --- a/searchsummary/src/tests/extractkeywords/extractkeywordstest.cpp +++ b/searchsummary/src/tests/extractkeywords/extractkeywordstest.cpp @@ -1,6 +1,4 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// Copyright (C) 2001-2003 Fast Search & Transfer ASA -// Copyright (C) 2003 Overture Services Norway AS #include "extractkeywordstest.h" #include <vespa/searchsummary/docsummary/keywordextractor.h> diff --git a/searchsummary/src/tests/extractkeywords/extractkeywordstest.h b/searchsummary/src/tests/extractkeywords/extractkeywordstest.h index 55873ecb0a6..e3c79bfc202 100644 --- a/searchsummary/src/tests/extractkeywords/extractkeywordstest.h +++ b/searchsummary/src/tests/extractkeywords/extractkeywordstest.h @@ -1,6 +1,4 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// Copyright (C) 2001-2003 Fast Search & Transfer ASA -// Copyright (C) 2003 Overture Services Norway AS #pragma once diff --git a/searchsummary/src/tests/extractkeywords/runtests.sh b/searchsummary/src/tests/extractkeywords/runtests.sh index 2d27e09ab0e..2b1cf3f228e 100755 --- a/searchsummary/src/tests/extractkeywords/runtests.sh +++ b/searchsummary/src/tests/extractkeywords/runtests.sh @@ -1,13 +1,6 @@ #!/bin/bash # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -# -# $Id$ -# -# Copyright (C) 2000-2003 Fast Search & Transfer ASA -# Copyright (C) 2003 Overture Services Norway AS -# -# All Rights Reserved -# + set -e if $VALGRIND ./searchsummary_extractkeywordstest_app - diff --git a/searchsummary/src/vespa/searchsummary/config/CMakeLists.txt b/searchsummary/src/vespa/searchsummary/config/CMakeLists.txt index 4d14cfd9fdf..7f6dc5c4cdd 100644 --- a/searchsummary/src/vespa/searchsummary/config/CMakeLists.txt +++ b/searchsummary/src/vespa/searchsummary/config/CMakeLists.txt @@ -4,4 +4,4 @@ vespa_add_library(searchsummary_config OBJECT DEPENDS ) vespa_generate_config(searchsummary_config juniperrc.def) -install(FILES juniperrc.def DESTINATION var/db/vespa/config_server/serverdb/classes) +install(FILES juniperrc.def RENAME vespa.config.search.summary.juniperrc.def DESTINATION var/db/vespa/config_server/serverdb/classes) diff --git a/searchsummary/src/vespa/searchsummary/docsummary/docsumconfig.h b/searchsummary/src/vespa/searchsummary/docsummary/docsumconfig.h index 16e83601186..57c82c97a93 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/docsumconfig.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/docsumconfig.h @@ -1,6 +1,4 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// Copyright (C) 1998-2003 Fast Search & Transfer ASA -// Copyright (C) 2003 Overture Services Norway AS #pragma once diff --git a/searchsummary/src/vespa/searchsummary/docsummary/docsumstate.h b/searchsummary/src/vespa/searchsummary/docsummary/docsumstate.h index ebff2264f29..5e01d698fb6 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/docsumstate.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/docsumstate.h @@ -1,6 +1,4 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// Copyright (C) 1998-2003 Fast Search & Transfer ASA -// Copyright (C) 2003 Overture Services Norway AS #pragma once diff --git a/searchsummary/src/vespa/searchsummary/docsummary/docsumstore.h b/searchsummary/src/vespa/searchsummary/docsummary/docsumstore.h index 4b3e27799fe..21ce81c483b 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/docsumstore.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/docsumstore.h @@ -1,6 +1,4 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// Copyright (C) 1998-2003 Fast Search & Transfer ASA -// Copyright (C) 2003 Overture Services Norway AS #pragma once diff --git a/searchsummary/src/vespa/searchsummary/docsummary/getdocsumargs.h b/searchsummary/src/vespa/searchsummary/docsummary/getdocsumargs.h index beca4134946..44cb674597f 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/getdocsumargs.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/getdocsumargs.h @@ -1,6 +1,4 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// Copyright (C) 2003 Overture Services Norway AS -// Copyright (C) 1999-2003 Fast Search & Transfer ASA #pragma once diff --git a/searchsummary/src/vespa/searchsummary/docsummary/keywordextractor.h b/searchsummary/src/vespa/searchsummary/docsummary/keywordextractor.h index 3b6779f6c52..6c9ed675225 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/keywordextractor.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/keywordextractor.h @@ -1,6 +1,4 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// Copyright (C) 1998-2003 Fast Search & Transfer ASA -// Copyright (C) 2003 Overture Services Norway AS #pragma once diff --git a/searchsummary/src/vespa/searchsummary/docsummary/resultpacker.h b/searchsummary/src/vespa/searchsummary/docsummary/resultpacker.h index 3dd643406d7..e4d49b2eb60 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/resultpacker.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/resultpacker.h @@ -1,6 +1,4 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// Copyright (C) 2001-2003 Fast Search & Transfer ASA -// Copyright (C) 2003 Overture Services Norway AS #pragma once diff --git a/staging_vespalib/src/vespa/vespalib/util/rand48.h b/staging_vespalib/src/vespa/vespalib/util/rand48.h index 64d78c2f8b3..441f8e6e10f 100644 --- a/staging_vespalib/src/vespa/vespalib/util/rand48.h +++ b/staging_vespalib/src/vespa/vespalib/util/rand48.h @@ -1,6 +1,4 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// Copyright (C) 2003 Fast Search & Transfer ASA -// Copyright (C) 2003 Overture Services Norway AS #pragma once diff --git a/storage/src/vespa/storage/bucketdb/CMakeLists.txt b/storage/src/vespa/storage/bucketdb/CMakeLists.txt index 3f0a27a5bb4..5c818631d54 100644 --- a/storage/src/vespa/storage/bucketdb/CMakeLists.txt +++ b/storage/src/vespa/storage/bucketdb/CMakeLists.txt @@ -17,6 +17,6 @@ vespa_add_library(storage_bucketdb OBJECT storage_storageconfig ) vespa_generate_config(storage_bucketdb stor-bucketdb.def) -install(FILES stor-bucketdb.def DESTINATION var/db/vespa/config_server/serverdb/classes) +install(FILES stor-bucketdb.def RENAME vespa.config.content.core.stor-bucketdb.def DESTINATION var/db/vespa/config_server/serverdb/classes) vespa_generate_config(storage_bucketdb stor-bucket-init.def) -install(FILES stor-bucket-init.def DESTINATION var/db/vespa/config_server/serverdb/classes) +install(FILES stor-bucket-init.def RENAME vespa.config.content.core.stor-bucket-init.def DESTINATION var/db/vespa/config_server/serverdb/classes) diff --git a/storage/src/vespa/storage/config/CMakeLists.txt b/storage/src/vespa/storage/config/CMakeLists.txt index 9b7af8c6024..0f3f392f324 100644 --- a/storage/src/vespa/storage/config/CMakeLists.txt +++ b/storage/src/vespa/storage/config/CMakeLists.txt @@ -5,26 +5,26 @@ vespa_add_library(storage_storageconfig OBJECT DEPENDS ) vespa_generate_config(storage_storageconfig stor-communicationmanager.def) -install(FILES stor-communicationmanager.def DESTINATION var/db/vespa/config_server/serverdb/classes) +install(FILES stor-communicationmanager.def RENAME vespa.config.content.core.stor-communicationmanager.def DESTINATION var/db/vespa/config_server/serverdb/classes) vespa_generate_config(storage_storageconfig stor-distributormanager.def) -install(FILES stor-distributormanager.def DESTINATION var/db/vespa/config_server/serverdb/classes) +install(FILES stor-distributormanager.def RENAME vespa.config.content.core.stor-distributormanager.def DESTINATION var/db/vespa/config_server/serverdb/classes) vespa_generate_config(storage_storageconfig stor-server.def) -install(FILES stor-server.def DESTINATION var/db/vespa/config_server/serverdb/classes) +install(FILES stor-server.def RENAME vespa.config.content.core.stor-server.def DESTINATION var/db/vespa/config_server/serverdb/classes) vespa_generate_config(storage_storageconfig stor-status.def) -install(FILES stor-status.def DESTINATION var/db/vespa/config_server/serverdb/classes) +install(FILES stor-status.def RENAME vespa.config.content.core.stor-status.def DESTINATION var/db/vespa/config_server/serverdb/classes) vespa_generate_config(storage_storageconfig stor-messageforwarder.def) -install(FILES stor-messageforwarder.def DESTINATION var/db/vespa/config_server/serverdb/classes) +install(FILES stor-messageforwarder.def RENAME vespa.config.content.core.stor-messageforwarder.def DESTINATION var/db/vespa/config_server/serverdb/classes) vespa_generate_config(storage_storageconfig stor-opslogger.def) -install(FILES stor-opslogger.def DESTINATION var/db/vespa/config_server/serverdb/classes) +install(FILES stor-opslogger.def RENAME vespa.config.content.core.stor-opslogger.def DESTINATION var/db/vespa/config_server/serverdb/classes) vespa_generate_config(storage_storageconfig stor-visitordispatcher.def) -install(FILES stor-visitordispatcher.def DESTINATION var/db/vespa/config_server/serverdb/classes) +install(FILES stor-visitordispatcher.def RENAME vespa.config.content.core.stor-visitordispatcher.def DESTINATION var/db/vespa/config_server/serverdb/classes) vespa_generate_config(storage_storageconfig stor-integritychecker.def) -install(FILES stor-integritychecker.def DESTINATION var/db/vespa/config_server/serverdb/classes) +install(FILES stor-integritychecker.def RENAME vespa.config.content.core.stor-integritychecker.def DESTINATION var/db/vespa/config_server/serverdb/classes) vespa_generate_config(storage_storageconfig stor-bucketmover.def) -install(FILES stor-bucketmover.def DESTINATION var/db/vespa/config_server/serverdb/classes) +install(FILES stor-bucketmover.def RENAME vespa.config.content.core.stor-bucketmover.def DESTINATION var/db/vespa/config_server/serverdb/classes) vespa_generate_config(storage_storageconfig stor-bouncer.def) -install(FILES stor-bouncer.def DESTINATION var/db/vespa/config_server/serverdb/classes) +install(FILES stor-bouncer.def RENAME vespa.config.content.core.stor-bouncer.def DESTINATION var/db/vespa/config_server/serverdb/classes) vespa_generate_config(storage_storageconfig stor-prioritymapping.def) -install(FILES stor-prioritymapping.def DESTINATION var/db/vespa/config_server/serverdb/classes) +install(FILES stor-prioritymapping.def RENAME vespa.config.content.core.stor-prioritymapping.def DESTINATION var/db/vespa/config_server/serverdb/classes) vespa_generate_config(storage_storageconfig rpc-provider.def) -install(FILES rpc-provider.def DESTINATION var/db/vespa/config_server/serverdb/classes) +install(FILES rpc-provider.def RENAME vespa.config.content.core.rpc-provider.def DESTINATION var/db/vespa/config_server/serverdb/classes) diff --git a/storage/src/vespa/storage/storageserver/communicationmanager.cpp b/storage/src/vespa/storage/storageserver/communicationmanager.cpp index ae8f3290fef..c90b18038c2 100644 --- a/storage/src/vespa/storage/storageserver/communicationmanager.cpp +++ b/storage/src/vespa/storage/storageserver/communicationmanager.cpp @@ -404,6 +404,11 @@ void CommunicationManager::configure(std::unique_ptr<CommunicationManagerConfig> params.setListenPort(config->mbusport); } + using CompressionConfig = vespalib::compression::CompressionConfig; + CompressionConfig::Type compressionType = CompressionConfig::toType( + CommunicationManagerConfig::Mbus::Compress::getTypeName(config->mbus.compress.type).c_str()); + params.setCompressionConfig(CompressionConfig(compressionType, config->mbus.compress.level, + 90, config->mbus.compress.limit)); // Configure messagebus here as we for legacy reasons have // config here. _mbus = std::make_unique<mbus::RPCMessageBus>( diff --git a/storage/src/vespa/storage/visiting/CMakeLists.txt b/storage/src/vespa/storage/visiting/CMakeLists.txt index 9bd18510859..ee10bbd58ab 100644 --- a/storage/src/vespa/storage/visiting/CMakeLists.txt +++ b/storage/src/vespa/storage/visiting/CMakeLists.txt @@ -17,4 +17,4 @@ vespa_add_library(storage_visitor OBJECT storage_storageconfig ) vespa_generate_config(storage_visitor stor-visitor.def) -install(FILES stor-visitor.def DESTINATION var/db/vespa/config_server/serverdb/classes) +install(FILES stor-visitor.def RENAME vespa.config.content.core.stor-visitor.def DESTINATION var/db/vespa/config_server/serverdb/classes) diff --git a/streamingvisitors/src/vespa/searchvisitor/rankprocessor.cpp b/streamingvisitors/src/vespa/searchvisitor/rankprocessor.cpp index f0c2a8d4cc9..dc9cfdc7efd 100644 --- a/streamingvisitors/src/vespa/searchvisitor/rankprocessor.cpp +++ b/streamingvisitors/src/vespa/searchvisitor/rankprocessor.cpp @@ -89,6 +89,7 @@ RankProcessor::initQueryEnvironment() terms[i].getTerm()->index().c_str(), terms[i].getTerm()->getTerm()); } } + _match_data = _mdLayout.createMatchData(); } void @@ -100,7 +101,7 @@ RankProcessor::initHitCollector(size_t wantedHitCount) void RankProcessor::setupRankProgram(RankProgram &program) { - program.setup(_mdLayout, _queryEnv, search::fef::Properties()); + program.setup(*_match_data, _queryEnv, search::fef::Properties()); } void @@ -137,6 +138,7 @@ RankProcessor::RankProcessor(RankManager::Snapshot::SP snapshot, _query(query), _queryEnv(location, snapshot->getIndexEnvironment(rankProfile), queryProperties, attrMgr), _mdLayout(), + _match_data(), _rankProgram(), _docId(TermFieldMatchData::invalidId()), _score(0.0), @@ -182,13 +184,13 @@ copyTermFieldMatchData(const std::vector<search::fef::TermFieldMatchData> &src, class RankProgramWrapper : public HitCollector::IRankProgram { private: - RankProgram &_rankProgram; + MatchData &_match_data; public: - RankProgramWrapper(RankProgram &rankProgram) : _rankProgram(rankProgram) {} + RankProgramWrapper(MatchData &match_data) : _match_data(match_data) {} virtual void run(uint32_t docid, const std::vector<search::fef::TermFieldMatchData> &matchData) override { // Prepare the match data object used by the rank program with earlier unpacked match data. - copyTermFieldMatchData(matchData, _rankProgram.match_data()); + copyTermFieldMatchData(matchData, _match_data); (void) docid; } }; @@ -202,7 +204,7 @@ RankProcessor::calculateFeatureSet() RankProgram &rankProgram = *(_summaryProgram.get() != nullptr ? _summaryProgram : _rankProgram); search::fef::FeatureResolver resolver(rankProgram.get_seeds()); LOG(debug, "Feature handles: numNames(%ld)", resolver.num_features()); - RankProgramWrapper wrapper(rankProgram); + RankProgramWrapper wrapper(*_match_data); FeatureSet::SP sf = _hitCollector->getFeatureSet(wrapper, resolver); LOG(debug, "Feature set: numFeatures(%u), numDocs(%u)", sf->numFeatures(), sf->numDocs()); return sf; @@ -217,9 +219,8 @@ RankProcessor::fillSearchResult(vdslib::SearchResult & searchResult) void RankProcessor::unpackMatchData(uint32_t docId) { - MatchData &matchData = _rankProgram->match_data(); _docId = docId; - unpackMatchData(matchData); + unpackMatchData(*_match_data); } void diff --git a/streamingvisitors/src/vespa/searchvisitor/rankprocessor.h b/streamingvisitors/src/vespa/searchvisitor/rankprocessor.h index f1fb917afcf..bab63c36522 100644 --- a/streamingvisitors/src/vespa/searchvisitor/rankprocessor.h +++ b/streamingvisitors/src/vespa/searchvisitor/rankprocessor.h @@ -29,6 +29,7 @@ private: QueryEnvironment _queryEnv; search::fef::MatchDataLayout _mdLayout; + search::fef::MatchData::UP _match_data; search::fef::RankProgram::UP _rankProgram; uint32_t _docId; double _score; @@ -67,7 +68,7 @@ public: void runRankProgram(uint32_t docId); search::FeatureSet::SP calculateFeatureSet(); void fillSearchResult(vdslib::SearchResult & searchResult); - const search::fef::MatchData &getMatchData() const { return _rankProgram->match_data(); } + const search::fef::MatchData &getMatchData() const { return *_match_data; } void setRankScore(double score) { _score = score; } double getRankScore() const { return _score; } HitCollector & getHitCollector() { return *_hitCollector; } diff --git a/vespabase/src/rhel-prestart.sh b/vespabase/src/rhel-prestart.sh index 069a41235b2..c00557243a7 100755 --- a/vespabase/src/rhel-prestart.sh +++ b/vespabase/src/rhel-prestart.sh @@ -84,7 +84,6 @@ fixdir root wheel 1777 var/run fixdir ${VESPA_USER} wheel 1777 var/crash fixdir ${VESPA_USER} wheel 1777 logs/vespa fixdir ${VESPA_USER} wheel 1777 tmp/vespa -fixdir ${VESPA_USER} wheel 755 libdata/yell/exception fixdir ${VESPA_USER} wheel 755 libexec/vespa/plugins/qrs fixdir ${VESPA_USER} wheel 755 logs/vespa/configserver fixdir ${VESPA_USER} wheel 755 logs/vespa/qrs @@ -95,8 +94,6 @@ fixdir ${VESPA_USER} wheel 755 var/db/vespa/tmp fixdir ${VESPA_USER} wheel 755 var/db/vespa/config_server fixdir ${VESPA_USER} wheel 755 var/db/vespa/config_server/serverdb fixdir ${VESPA_USER} wheel 755 var/db/vespa/config_server/serverdb/applications -fixdir ${VESPA_USER} wheel 755 var/db/vespa/config_server/serverdb/configs -fixdir ${VESPA_USER} wheel 755 var/db/vespa/config_server/serverdb/configs/application fixdir ${VESPA_USER} wheel 755 var/db/vespa/index fixdir ${VESPA_USER} wheel 755 var/db/vespa/logcontrol fixdir ${VESPA_USER} wheel 755 var/db/vespa/search diff --git a/vespabase/src/vespa-start-configserver.sh b/vespabase/src/vespa-start-configserver.sh index cba6cbfd740..38a4249982e 100755 --- a/vespabase/src/vespa-start-configserver.sh +++ b/vespabase/src/vespa-start-configserver.sh @@ -61,11 +61,8 @@ findroot ${VESPA_HOME}/bin/vespa-prestart.sh || exit 1 willrun=${VESPA_HOME}/libexec/vespa/start-configserver if [ -x $willrun ]; then - echo "Starting $willrun as a background process." >&2 - exec </dev/null - exec >/dev/null 2>&1 - nohup $willrun & - exit 0 + echo "Running $willrun" >&2 + exec $willrun fi echo "FATAL cannot run: $willrun" exit 1 diff --git a/vespabase/src/vespa-start-services.sh b/vespabase/src/vespa-start-services.sh index e3cdf855850..fbc110dbdee 100755 --- a/vespabase/src/vespa-start-services.sh +++ b/vespabase/src/vespa-start-services.sh @@ -61,11 +61,8 @@ findroot ${VESPA_HOME}/bin/vespa-prestart.sh || exit 1 willrun=${VESPA_HOME}/libexec/vespa/start-vespa-base.sh if [ -x $willrun ]; then - echo "Starting $willrun as a background process." >&2 - exec </dev/null - exec >/dev/null 2>&1 - nohup $willrun & - exit 0 + echo "Running $willrun" >&2 + exec $willrun fi echo "FATAL cannot run: $willrun" exit 1 diff --git a/vespaclient/src/vespa/vespaclient/vesparoute/application.cpp b/vespaclient/src/vespa/vespaclient/vesparoute/application.cpp index 1a518b4c819..fe4600aea58 100644 --- a/vespaclient/src/vespa/vespaclient/vesparoute/application.cpp +++ b/vespaclient/src/vespa/vespaclient/vesparoute/application.cpp @@ -4,16 +4,18 @@ #include <vespa/document/config/config-documenttypes.h> #include <vespa/document/repo/documenttyperepo.h> -#include <vespa/document/util/stringutil.h> #include <vespa/documentapi/messagebus/documentprotocol.h> #include <vespa/messagebus/configagent.h> #include <vespa/messagebus/routing/routingtable.h> #include <vespa/messagebus/routing/routedirective.h> #include <vespa/messagebus/rpcmessagebus.h> +#include <vespa/messagebus/network/rpcsendv1.h> +#include <vespa/messagebus/network/rpcsendv2.h> #include <vespa/slobrok/sbmirror.h> #include <vespa/config/common/exceptions.h> #include <vespa/config/helper/configgetter.hpp> #include <vespa/vespalib/util/stringfmt.h> +#include <vespa/fnet/frt/supervisor.h> using config::ConfigGetter; using document::DocumenttypesConfig; @@ -528,9 +530,8 @@ Application::isService(FRT_Supervisor &frt, const std::string &spec) const FRT_StringValue *retList = req->GetReturn()->GetValue(2)._string_array._pt; for (uint32_t i = 0; i < numMethods; ++i) { - if (strcmp(methods[i]._str, mbus::RPCSendV1::METHOD_NAME) == 0 && - strcmp(argList[i]._str, mbus::RPCSendV1::METHOD_PARAMS) == 0 && - strcmp(retList[i]._str, mbus::RPCSendV1::METHOD_RETURN) == 0) { + if (mbus::RPCSendV1::isCompatible(methods[i]._str,argList[i]._str, retList[i]._str) || + mbus::RPCSendV2::isCompatible(methods[i]._str,argList[i]._str, retList[i]._str)) { ret = true; break; } diff --git a/vespajlib/src/main/java/com/yahoo/concurrent/Lock.java b/vespajlib/src/main/java/com/yahoo/concurrent/lock/Lock.java index 3b1919a987d..1273a298592 100644 --- a/vespajlib/src/main/java/com/yahoo/concurrent/Lock.java +++ b/vespajlib/src/main/java/com/yahoo/concurrent/lock/Lock.java @@ -1,5 +1,5 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.concurrent; +package com.yahoo.concurrent.lock; import java.util.concurrent.locks.ReentrantLock; diff --git a/vespajlib/src/main/java/com/yahoo/concurrent/lock/Locking.java b/vespajlib/src/main/java/com/yahoo/concurrent/lock/Locking.java new file mode 100644 index 00000000000..4d9f45e959e --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/concurrent/lock/Locking.java @@ -0,0 +1,25 @@ +package com.yahoo.concurrent.lock; + +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.locks.ReentrantLock; + +/** + * @author valerijf + */ +public class Locking { + private final Map<Class<?>, ReentrantLock> locks = new ConcurrentHashMap<>(); + + /** + * Locks class. This will block until the lock is acquired. + * Users of this <b>must</b> close any lock acquired. + * + * @param key the key to lock + * @return the acquired lock + */ + public Lock lock(Class<?> key) { + ReentrantLock lock = locks.computeIfAbsent(key, k -> new ReentrantLock(true)); + lock.lock(); + return new Lock(lock); + } +} diff --git a/vespajlib/src/main/java/com/yahoo/concurrent/Locks.java b/vespajlib/src/main/java/com/yahoo/concurrent/lock/Locks.java index 2ac53bdb963..0308b26c903 100644 --- a/vespajlib/src/main/java/com/yahoo/concurrent/Locks.java +++ b/vespajlib/src/main/java/com/yahoo/concurrent/lock/Locks.java @@ -1,5 +1,5 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.concurrent; +package com.yahoo.concurrent.lock; import com.google.common.util.concurrent.UncheckedTimeoutException; @@ -51,7 +51,7 @@ public class Locks<TYPE> { public Lock lock(TYPE key, long timeout, TimeUnit timeoutUnit) { try { ReentrantLock lock = locks.computeIfAbsent(key, k -> new ReentrantLock(true)); - boolean acquired = lock.tryLock(timeout, TimeUnit.MILLISECONDS); + boolean acquired = lock.tryLock(timeout, timeoutUnit); if ( ! acquired) throw new UncheckedTimeoutException("Timed out waiting for the lock to " + key); return new Lock(lock); diff --git a/vespajlib/src/main/java/com/yahoo/concurrent/lock/package-info.java b/vespajlib/src/main/java/com/yahoo/concurrent/lock/package-info.java new file mode 100644 index 00000000000..326124e600f --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/concurrent/lock/package-info.java @@ -0,0 +1,5 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.concurrent.lock; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/vespajlib/src/main/java/com/yahoo/text/Lowercase.java b/vespajlib/src/main/java/com/yahoo/text/Lowercase.java index 61efcd6a683..2ee88ebe5cb 100644 --- a/vespajlib/src/main/java/com/yahoo/text/Lowercase.java +++ b/vespajlib/src/main/java/com/yahoo/text/Lowercase.java @@ -8,7 +8,7 @@ import java.util.Locale; * which is not to be handled as natural language data, e.g. field names or * configuration paramaters. * - * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + * @author Steinar Knutsen */ public final class Lowercase { diff --git a/vespalib/fix-hg.pl b/vespalib/fix-hg.pl deleted file mode 100755 index fe5eb1101de..00000000000 --- a/vespalib/fix-hg.pl +++ /dev/null @@ -1,50 +0,0 @@ -#!/usr/bin/perl -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -foreach $arg ( @ARGV ) { - $hgd = $arg; # maybe: . "_"; - $hgd =~ s{\W}{_}g; - $hgd =~ tr{a-z}{A-Z}; - $hgd =~ s{^_*}{H_}; - -# print "arg $arg header guard $hgd\n"; - - open(FOO, $arg) or die "Cannot open '$arg'\n"; - $backup = $arg . ".orig"; - rename ($arg, $backup); - - open(ARGVOUT, ">$arg") or die "cannot write to '$arg'\n"; - select(ARGVOUT); - - my $eic = 0; - my $cnt = 1; - - while (<FOO>) { - ++$eic if m{#endif} ; - } - seek FOO, 0, 0; - while (<FOO>) { - if ($cnt == 1 and m{^#ifndef}) { - s{\s.*}{ $hgd}; - ++$cnt; - } - if ($cnt == 2 and m{^#define}) { - s{\s.*}{ $hgd}; - ++$cnt; - } - if ( m{#endif} ) { - --$eic; - if ($eic == 0) { - s{.*#endif.*}{#endif // header guard}; - } - } - print; - } - close(FOO); - select(STDOUT); - close(ARGVOUT); - - unlink($backup); -} - -exit 0; diff --git a/vespalib/fix-hg.sh b/vespalib/fix-hg.sh deleted file mode 100755 index 57c7d48bc58..00000000000 --- a/vespalib/fix-hg.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/sh -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -td=${0%.sh}.pl -if [ -f "$td" ]; then - find . -name '*.h' -o -name '*.hpp' | xargs $td -else - echo "Could not find '${td}'" -fi diff --git a/vespalib/src/vespa/vespalib/hwaccelrated/generic.h b/vespalib/src/vespa/vespalib/hwaccelrated/generic.h index abb2b48b7c0..4722179dd66 100644 --- a/vespalib/src/vespa/vespalib/hwaccelrated/generic.h +++ b/vespalib/src/vespa/vespalib/hwaccelrated/generic.h @@ -1,6 +1,4 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// Copyright (C) 2003 Fast Search & Transfer ASA -// Copyright (C) 2003 Overture Services Norway AS #pragma once diff --git a/vespalib/src/vespa/vespalib/util/alignedmemory.h b/vespalib/src/vespa/vespalib/util/alignedmemory.h index 7ff0dfbc90e..e99417547c1 100644 --- a/vespalib/src/vespa/vespalib/util/alignedmemory.h +++ b/vespalib/src/vespa/vespalib/util/alignedmemory.h @@ -1,5 +1,4 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// Copyright (C) 2009 Yahoo #pragma once diff --git a/vespalib/src/vespa/vespalib/util/arrayqueue.hpp b/vespalib/src/vespa/vespalib/util/arrayqueue.hpp index 6c26fd3a0c4..17a8c02dbf5 100644 --- a/vespalib/src/vespa/vespalib/util/arrayqueue.hpp +++ b/vespalib/src/vespa/vespalib/util/arrayqueue.hpp @@ -1,5 +1,4 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// Copyright (C) 2009 Yahoo #pragma once diff --git a/vespalib/src/vespa/vespalib/util/barrier.h b/vespalib/src/vespa/vespalib/util/barrier.h index 9935f238618..20768c3abcf 100644 --- a/vespalib/src/vespa/vespalib/util/barrier.h +++ b/vespalib/src/vespa/vespalib/util/barrier.h @@ -1,5 +1,4 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// Copyright (C) 2010 Yahoo #pragma once diff --git a/vespalib/src/vespa/vespalib/util/blockingthreadstackexecutor.h b/vespalib/src/vespa/vespalib/util/blockingthreadstackexecutor.h index 39656039d55..73a142ae42c 100644 --- a/vespalib/src/vespa/vespalib/util/blockingthreadstackexecutor.h +++ b/vespalib/src/vespa/vespalib/util/blockingthreadstackexecutor.h @@ -1,5 +1,4 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// Copyright (C) 2010 Yahoo #pragma once diff --git a/vespalib/src/vespa/vespalib/util/bobhash.h b/vespalib/src/vespa/vespalib/util/bobhash.h index 1b1d0095be9..2aff09929b2 100644 --- a/vespalib/src/vespa/vespalib/util/bobhash.h +++ b/vespalib/src/vespa/vespalib/util/bobhash.h @@ -5,21 +5,14 @@ namespace vespalib { -/** - * @class BobHash - * - * @brief Hash function based on - * http://burtleburtle.net/bob/hash/index.html - * by Bob Jenkins, 1996. bob_jenkins@burtleburtle.net. You may use this - * code any way you wish, private, educational, or commercial. It's free. - * - * @author Michael Susag - * @date Creation date: 2003-04-16 - * @version $Id$ +/* + * Hash function based on + * http://burtleburtle.net/bob/hash/index.html + * by Bob Jenkins, 1996. bob_jenkins@burtleburtle.net. You may use this + * code any way you wish, private, educational, or commercial. It's free. * - * Copyright (c) 2003 Fast Search & Transfer ASA - * ALL RIGHTS RESERVED - **/ + * Author: Michael Susag + */ diff --git a/vespalib/src/vespa/vespalib/util/delegatelist.hpp b/vespalib/src/vespa/vespalib/util/delegatelist.hpp index 5e2bacb5cf3..716474f66f7 100644 --- a/vespalib/src/vespa/vespalib/util/delegatelist.hpp +++ b/vespalib/src/vespa/vespalib/util/delegatelist.hpp @@ -1,5 +1,4 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// Copyright (C) 2009 Yahoo #pragma once diff --git a/vespalib/src/vespa/vespalib/util/eventbarrier.hpp b/vespalib/src/vespa/vespalib/util/eventbarrier.hpp index 0e8ca2c274d..79dcf131985 100644 --- a/vespalib/src/vespa/vespalib/util/eventbarrier.hpp +++ b/vespalib/src/vespa/vespalib/util/eventbarrier.hpp @@ -1,5 +1,4 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// Copyright (C) 2009 Yahoo #pragma once diff --git a/vespalib/src/vespa/vespalib/util/exception.h b/vespalib/src/vespa/vespalib/util/exception.h index c7067101967..ff5f02f33da 100644 --- a/vespalib/src/vespa/vespalib/util/exception.h +++ b/vespalib/src/vespa/vespalib/util/exception.h @@ -1,5 +1,4 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// Copyright (C) 2006 Yahoo #pragma once diff --git a/vespalib/src/vespa/vespalib/util/executor.h b/vespalib/src/vespa/vespalib/util/executor.h index c686ab90382..ef5cb4b84fa 100644 --- a/vespalib/src/vespa/vespalib/util/executor.h +++ b/vespalib/src/vespa/vespalib/util/executor.h @@ -1,5 +1,4 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// Copyright (C) 2010 Yahoo #pragma once diff --git a/vespalib/src/vespa/vespalib/util/guard.h b/vespalib/src/vespa/vespalib/util/guard.h index 390a8734d0c..bc45ff404f9 100644 --- a/vespalib/src/vespa/vespalib/util/guard.h +++ b/vespalib/src/vespa/vespalib/util/guard.h @@ -1,5 +1,4 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// Copyright (C) 2006 Yahoo #pragma once diff --git a/vespalib/src/vespa/vespalib/util/macro.h b/vespalib/src/vespa/vespalib/util/macro.h index 4f4a4a44f4b..16c66880e5d 100644 --- a/vespalib/src/vespa/vespalib/util/macro.h +++ b/vespalib/src/vespa/vespalib/util/macro.h @@ -1,5 +1,4 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// Copyright (C) 2006 Yahoo #pragma once diff --git a/vespalib/src/vespa/vespalib/util/memory.h b/vespalib/src/vespa/vespalib/util/memory.h index 1326236723a..7f51c4b553f 100644 --- a/vespalib/src/vespa/vespalib/util/memory.h +++ b/vespalib/src/vespa/vespalib/util/memory.h @@ -1,5 +1,4 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// Copyright (C) 2006 Yahoo #pragma once diff --git a/vespalib/src/vespa/vespalib/util/optimized.h b/vespalib/src/vespa/vespalib/util/optimized.h index f4fd1376033..28dee703a5f 100644 --- a/vespalib/src/vespa/vespalib/util/optimized.h +++ b/vespalib/src/vespa/vespalib/util/optimized.h @@ -1,6 +1,4 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// Copyright (C) 2003 Fast Search & Transfer ASA -// Copyright (C) 2003 Overture Services Norway AS #pragma once diff --git a/vespalib/src/vespa/vespalib/util/ptrholder.h b/vespalib/src/vespa/vespalib/util/ptrholder.h index 7a1ef8b0ccd..3e807a8a93d 100644 --- a/vespalib/src/vespa/vespalib/util/ptrholder.h +++ b/vespalib/src/vespa/vespalib/util/ptrholder.h @@ -1,5 +1,4 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// Copyright (C) 2004 Overture Services Norway AS #pragma once diff --git a/vespalib/src/vespa/vespalib/util/rwlock.h b/vespalib/src/vespa/vespalib/util/rwlock.h index 1a4f56df469..9b1f9b93289 100644 --- a/vespalib/src/vespa/vespalib/util/rwlock.h +++ b/vespalib/src/vespa/vespalib/util/rwlock.h @@ -1,5 +1,4 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// Copyright (C) 2006 Yahoo #pragma once diff --git a/vespalib/src/vespa/vespalib/util/sequence.h b/vespalib/src/vespa/vespalib/util/sequence.h index 99fbf4cad3c..e0ac526cd06 100644 --- a/vespalib/src/vespa/vespalib/util/sequence.h +++ b/vespalib/src/vespa/vespalib/util/sequence.h @@ -1,5 +1,4 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// Copyright (C) 2010 Yahoo #pragma once diff --git a/vespalib/src/vespa/vespalib/util/slaveproc.h b/vespalib/src/vespa/vespalib/util/slaveproc.h index 589301d7ad1..72b20cfb069 100644 --- a/vespalib/src/vespa/vespalib/util/slaveproc.h +++ b/vespalib/src/vespa/vespalib/util/slaveproc.h @@ -1,5 +1,4 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// Copyright (C) 2004 Overture Services Norway AS #pragma once diff --git a/vespalib/src/vespa/vespalib/util/sync.h b/vespalib/src/vespa/vespalib/util/sync.h index ccadbfb8dfc..f961c280174 100644 --- a/vespalib/src/vespa/vespalib/util/sync.h +++ b/vespalib/src/vespa/vespalib/util/sync.h @@ -1,5 +1,4 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// Copyright (C) 2006 Yahoo #pragma once diff --git a/vespalib/src/vespa/vespalib/util/syncable.h b/vespalib/src/vespa/vespalib/util/syncable.h index f7fee3dd862..0b009263646 100644 --- a/vespalib/src/vespa/vespalib/util/syncable.h +++ b/vespalib/src/vespa/vespalib/util/syncable.h @@ -1,5 +1,4 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// Copyright (C) 2010 Yahoo #pragma once diff --git a/vespalib/src/vespa/vespalib/util/threadexecutor.h b/vespalib/src/vespa/vespalib/util/threadexecutor.h index c202a0a2373..158805288e9 100644 --- a/vespalib/src/vespa/vespalib/util/threadexecutor.h +++ b/vespalib/src/vespa/vespalib/util/threadexecutor.h @@ -1,5 +1,4 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// Copyright (C) 2010 Yahoo #pragma once diff --git a/vespalib/src/vespa/vespalib/util/threadstackexecutorbase.h b/vespalib/src/vespa/vespalib/util/threadstackexecutorbase.h index ee142659027..5048a9c0436 100644 --- a/vespalib/src/vespa/vespalib/util/threadstackexecutorbase.h +++ b/vespalib/src/vespa/vespalib/util/threadstackexecutorbase.h @@ -1,5 +1,4 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// Copyright (C) 2010 Yahoo #pragma once diff --git a/vespalog/src/logger/runserver.cpp b/vespalog/src/logger/runserver.cpp index 7cbe3711889..efd45288199 100644 --- a/vespalog/src/logger/runserver.cpp +++ b/vespalog/src/logger/runserver.cpp @@ -51,15 +51,15 @@ public: int readPid(); void writePid(); bool writeOpen(); - bool isRunning(); - bool isMine(); + bool anotherRunning(); + bool canStealLock(); void cleanUp(); }; void PidFile::cleanUp() { - if (isMine() || !isRunning()) remove(_pidfile); + if (!anotherRunning()) remove(_pidfile); if (_fd >= 0) close(_fd); _fd = -1; } @@ -75,7 +75,6 @@ PidFile::writeOpen() strerror(errno)); return false; } - // XXX should we use locking or not? if (flock(_fd, LOCK_EX | LOCK_NB) != 0) { fprintf(stderr, "could not lock pidfile %s: %s\n", _pidfile, strerror(errno)); @@ -117,18 +116,34 @@ PidFile::readPid() } bool -PidFile::isRunning() +PidFile::anotherRunning() { int pid = readPid(); - if (pid < 1) return false; + if (pid < 1 || pid == getpid()) { + // no valid pid, or my own pid + return false; + } + if (canStealLock()) { + return false; + } return (kill(pid, 0) == 0 || errno == EPERM); } bool -PidFile::isMine() +PidFile::canStealLock() { - int pid = readPid(); - return (pid == getpid()); + int flags = O_WRONLY | O_NONBLOCK; + int desc = open(_pidfile, flags, 0644); + if (desc < 0) { + return false; + } + if (flock(desc, LOCK_EX | LOCK_NB) != 0) { + close(desc); + return false; + } + flock(desc, LOCK_UN | LOCK_NB); + close(desc); + return true; } using namespace ns_log; @@ -351,7 +366,7 @@ int main(int argc, char *argv[]) PidFile mypf(pidfile); if (doStop) { - if (mypf.isRunning()) { + if (mypf.anotherRunning()) { int pid = mypf.readPid(); if (killcmd != NULL) { fprintf(stdout, "%s was running with pid %d, running '%s' to stop it\n", @@ -399,7 +414,7 @@ int main(int argc, char *argv[]) usage(argv[0], 1); } - if (mypf.isRunning()) { + if (mypf.anotherRunning()) { fprintf(stderr, "runserver already running with pid %d\n", mypf.readPid()); exit(0); diff --git a/vespalog/src/main/java/com/yahoo/log/InvalidLogFormatException.java b/vespalog/src/main/java/com/yahoo/log/InvalidLogFormatException.java index ed09b6b0eaa..37d43b37952 100644 --- a/vespalog/src/main/java/com/yahoo/log/InvalidLogFormatException.java +++ b/vespalog/src/main/java/com/yahoo/log/InvalidLogFormatException.java @@ -5,7 +5,7 @@ package com.yahoo.log; * This (checked) exception is used to flag invalid log messages, * primarily for use in the factory methods of LogMessage. * - * @author <a href="mailto:borud@yahoo-inc.com">Bjorn Borud</a> + * @author Bjorn Borud */ public class InvalidLogFormatException extends Exception { diff --git a/vespalog/src/main/java/com/yahoo/log/LogMessage.java b/vespalog/src/main/java/com/yahoo/log/LogMessage.java index 608290026b6..1e2b9dfdab0 100644 --- a/vespalog/src/main/java/com/yahoo/log/LogMessage.java +++ b/vespalog/src/main/java/com/yahoo/log/LogMessage.java @@ -15,7 +15,7 @@ import com.yahoo.log.event.MalformedEventException; * chosen the name LogMessage to avoid confusion with LogRecord * which is used in java.util.logging. * - * @author <a href="mailto:borud@yahoo-inc.com">Bjorn Borud</a> + * @author Bjorn Borud */ public class LogMessage { diff --git a/vespalog/src/main/java/com/yahoo/log/LogSetup.java b/vespalog/src/main/java/com/yahoo/log/LogSetup.java index 50b0032d6b7..12b1411b17e 100644 --- a/vespalog/src/main/java/com/yahoo/log/LogSetup.java +++ b/vespalog/src/main/java/com/yahoo/log/LogSetup.java @@ -187,7 +187,7 @@ public class LogSetup { public boolean isLoggable(LogRecord record) { if (record.getLoggerName() == null) return true; if (!record.getLoggerName().startsWith("org.apache.zookeeper.") && - !record.getLoggerName().startsWith("com.netflix.curator")) { + !record.getLoggerName().startsWith("org.apache.curator")) { return true; } fileHandler.publish(record); diff --git a/vespalog/src/main/java/com/yahoo/log/event/Collection.java b/vespalog/src/main/java/com/yahoo/log/event/Collection.java index 52d0cbe5ab5..efd70742e2f 100644 --- a/vespalog/src/main/java/com/yahoo/log/event/Collection.java +++ b/vespalog/src/main/java/com/yahoo/log/event/Collection.java @@ -3,7 +3,7 @@ package com.yahoo.log.event; /** * - * @author <a href="mailto:borud@yahoo-inc.com">Bjorn Borud</a> + * @author Bjorn Borud */ public class Collection extends Event { public Collection () { diff --git a/vespalog/src/main/java/com/yahoo/log/event/Count.java b/vespalog/src/main/java/com/yahoo/log/event/Count.java index 4e422b3ec8c..175e8b9f69f 100644 --- a/vespalog/src/main/java/com/yahoo/log/event/Count.java +++ b/vespalog/src/main/java/com/yahoo/log/event/Count.java @@ -3,7 +3,7 @@ package com.yahoo.log.event; /** * - * @author <a href="mailto:borud@yahoo-inc.com">Bjorn Borud</a> + * @author Bjorn Borud */ public class Count extends Event { public Count () { diff --git a/vespalog/src/main/java/com/yahoo/log/event/Crash.java b/vespalog/src/main/java/com/yahoo/log/event/Crash.java index 12de9879df7..7732938b809 100644 --- a/vespalog/src/main/java/com/yahoo/log/event/Crash.java +++ b/vespalog/src/main/java/com/yahoo/log/event/Crash.java @@ -3,7 +3,7 @@ package com.yahoo.log.event; /** * - * @author <a href="mailto:borud@yahoo-inc.com">Bjorn Borud</a> + * @author Bjorn Borud */ public class Crash extends Event { public Crash () { diff --git a/vespalog/src/main/java/com/yahoo/log/event/Event.java b/vespalog/src/main/java/com/yahoo/log/event/Event.java index d3bd621c3ac..76fdf16c4ef 100644 --- a/vespalog/src/main/java/com/yahoo/log/event/Event.java +++ b/vespalog/src/main/java/com/yahoo/log/event/Event.java @@ -27,7 +27,7 @@ import java.util.regex.Pattern; * logging events, rather than making events him/herself and stuffing * them through the logging API. * - * @author <a href="mailto:borud@yahoo-inc.com">Bjorn Borud</a> + * @author Bjorn Borud */ public abstract class Event implements Serializable { private static Logger log = Logger.getLogger(Event.class.getName()); diff --git a/vespalog/src/main/java/com/yahoo/log/event/Progress.java b/vespalog/src/main/java/com/yahoo/log/event/Progress.java index 398dd10a8e6..e16dbce7b2d 100644 --- a/vespalog/src/main/java/com/yahoo/log/event/Progress.java +++ b/vespalog/src/main/java/com/yahoo/log/event/Progress.java @@ -3,7 +3,7 @@ package com.yahoo.log.event; /** * - * @author <a href="mailto:borud@yahoo-inc.com">Bjorn Borud</a> + * @author Bjorn Borud */ public class Progress extends Event { public Progress () { diff --git a/vespalog/src/main/java/com/yahoo/log/event/Reloaded.java b/vespalog/src/main/java/com/yahoo/log/event/Reloaded.java index cdd2bd8c261..8f976588446 100644 --- a/vespalog/src/main/java/com/yahoo/log/event/Reloaded.java +++ b/vespalog/src/main/java/com/yahoo/log/event/Reloaded.java @@ -3,7 +3,7 @@ package com.yahoo.log.event; /** * - * @author <a href="mailto:borud@yahoo-inc.com">Bjorn Borud</a> + * @author Bjorn Borud */ public class Reloaded extends Event { public Reloaded () { diff --git a/vespalog/src/main/java/com/yahoo/log/event/Reloading.java b/vespalog/src/main/java/com/yahoo/log/event/Reloading.java index 60f0d0aa729..617e2881b0c 100644 --- a/vespalog/src/main/java/com/yahoo/log/event/Reloading.java +++ b/vespalog/src/main/java/com/yahoo/log/event/Reloading.java @@ -3,7 +3,7 @@ package com.yahoo.log.event; /** * - * @author <a href="mailto:borud@yahoo-inc.com">Bjorn Borud</a> + * @author Bjorn Borud */ public class Reloading extends Event { public Reloading () { diff --git a/vespalog/src/main/java/com/yahoo/log/event/Started.java b/vespalog/src/main/java/com/yahoo/log/event/Started.java index 228dce3894b..4fa3853daaf 100644 --- a/vespalog/src/main/java/com/yahoo/log/event/Started.java +++ b/vespalog/src/main/java/com/yahoo/log/event/Started.java @@ -3,7 +3,7 @@ package com.yahoo.log.event; /** * - * @author <a href="mailto:borud@yahoo-inc.com">Bjorn Borud</a> + * @author Bjorn Borud */ public class Started extends Event { public Started () { diff --git a/vespalog/src/main/java/com/yahoo/log/event/Starting.java b/vespalog/src/main/java/com/yahoo/log/event/Starting.java index f8c8f04b5f8..8c3dee28271 100644 --- a/vespalog/src/main/java/com/yahoo/log/event/Starting.java +++ b/vespalog/src/main/java/com/yahoo/log/event/Starting.java @@ -3,7 +3,7 @@ package com.yahoo.log.event; /** * - * @author <a href="mailto:borud@yahoo-inc.com">Bjorn Borud</a> + * @author Bjorn Borud */ public class Starting extends Event { public Starting () { diff --git a/vespalog/src/main/java/com/yahoo/log/event/Stopped.java b/vespalog/src/main/java/com/yahoo/log/event/Stopped.java index 10e627ee85b..7964a956b28 100644 --- a/vespalog/src/main/java/com/yahoo/log/event/Stopped.java +++ b/vespalog/src/main/java/com/yahoo/log/event/Stopped.java @@ -3,7 +3,7 @@ package com.yahoo.log.event; /** * - * @author <a href="mailto:borud@yahoo-inc.com">Bjorn Borud</a> + * @author Bjorn Borud */ public class Stopped extends Event { public Stopped () { diff --git a/vespalog/src/main/java/com/yahoo/log/event/Stopping.java b/vespalog/src/main/java/com/yahoo/log/event/Stopping.java index 3868db7016d..1bc2f91f4e9 100644 --- a/vespalog/src/main/java/com/yahoo/log/event/Stopping.java +++ b/vespalog/src/main/java/com/yahoo/log/event/Stopping.java @@ -3,7 +3,7 @@ package com.yahoo.log.event; /** * - * @author <a href="mailto:borud@yahoo-inc.com">Bjorn Borud</a> + * @author Bjorn Borud */ public class Stopping extends Event { public Stopping () { diff --git a/vespalog/src/main/java/com/yahoo/log/event/Unknown.java b/vespalog/src/main/java/com/yahoo/log/event/Unknown.java index cf807f98fa7..e9fa1a2e530 100644 --- a/vespalog/src/main/java/com/yahoo/log/event/Unknown.java +++ b/vespalog/src/main/java/com/yahoo/log/event/Unknown.java @@ -3,7 +3,7 @@ package com.yahoo.log.event; /** * - * @author <a href="mailto:borud@yahoo-inc.com">Bjorn Borud </a> + * @author Bjorn Borud */ public class Unknown extends Event { public Unknown() { diff --git a/vespalog/src/main/java/com/yahoo/log/event/Value.java b/vespalog/src/main/java/com/yahoo/log/event/Value.java index 8214a111426..f694627bb12 100644 --- a/vespalog/src/main/java/com/yahoo/log/event/Value.java +++ b/vespalog/src/main/java/com/yahoo/log/event/Value.java @@ -3,7 +3,7 @@ package com.yahoo.log.event; /** * - * @author <a href="mailto:borud@yahoo-inc.com">Bjorn Borud</a> + * @author Bjorn Borud */ public class Value extends Event { public Value () { diff --git a/vespalog/src/test/java/com/yahoo/log/LogLevelTestCase.java b/vespalog/src/test/java/com/yahoo/log/LogLevelTestCase.java index 1b1c75ca518..0a19b900b97 100644 --- a/vespalog/src/test/java/com/yahoo/log/LogLevelTestCase.java +++ b/vespalog/src/test/java/com/yahoo/log/LogLevelTestCase.java @@ -12,7 +12,7 @@ import static org.junit.Assert.*; /** * Testcases for LogLevel * - * @author <a href="mailto:borud@yahoo-inc.com">Bjorn Borud</a> + * @author Bjorn Borud */ public class LogLevelTestCase { diff --git a/vespalog/src/test/java/com/yahoo/log/LogSetupTestCase.java b/vespalog/src/test/java/com/yahoo/log/LogSetupTestCase.java index bbb3e039562..70eb65b1463 100644 --- a/vespalog/src/test/java/com/yahoo/log/LogSetupTestCase.java +++ b/vespalog/src/test/java/com/yahoo/log/LogSetupTestCase.java @@ -18,7 +18,7 @@ import static org.hamcrest.CoreMatchers.is; /** * Make sure we can install the logging stuff properly. * - * @author <a href="mailto:borud@yahoo-inc.com">Bjorn Borud</a> + * @author Bjorn Borud */ public class LogSetupTestCase { // For testing zookeeper log records @@ -45,7 +45,7 @@ public class LogSetupTestCase { zookeeperLogRecord.setMillis(1107011348029L); curatorLogRecord = new LogRecord(Level.WARNING, "curator log record"); - curatorLogRecord.setLoggerName("com.netflix.curator.utils.DefaultTracerDriver"); + curatorLogRecord.setLoggerName("org.apache.curator.utils.DefaultTracerDriver"); curatorLogRecord.setMillis(1107011348029L); hostname = Util.getHostName(); @@ -63,7 +63,7 @@ public class LogSetupTestCase { zookeeperLogRecordError.setMillis(1107011348029L); curatorLogRecordError = new LogRecord(Level.SEVERE, "curator log record"); - curatorLogRecordError.setLoggerName("com.netflix.curator.utils.DefaultTracerDriver"); + curatorLogRecordError.setLoggerName("org.apache.curator.utils.DefaultTracerDriver"); curatorLogRecordError.setMillis(1107011348029L); notzookeeperLogRecord = new LogRecord(Level.WARNING, "not zookeeper log record"); diff --git a/vespalog/src/test/java/com/yahoo/log/UtilTestCase.java b/vespalog/src/test/java/com/yahoo/log/UtilTestCase.java index e7f93ea2611..760382b5c62 100644 --- a/vespalog/src/test/java/com/yahoo/log/UtilTestCase.java +++ b/vespalog/src/test/java/com/yahoo/log/UtilTestCase.java @@ -6,7 +6,7 @@ import org.junit.Test; import static org.junit.Assert.*; /** - * @author <a href="mailto:borud@yahoo-inc.com">Bjorn Borud</a> + * @author Bjorn Borud */ public class UtilTestCase { diff --git a/vespalog/src/test/java/com/yahoo/log/VespaFormatterTestCase.java b/vespalog/src/test/java/com/yahoo/log/VespaFormatterTestCase.java index 7ceae03e9ab..5dac2e34786 100644 --- a/vespalog/src/test/java/com/yahoo/log/VespaFormatterTestCase.java +++ b/vespalog/src/test/java/com/yahoo/log/VespaFormatterTestCase.java @@ -12,7 +12,7 @@ import static org.junit.Assert.*; import static org.hamcrest.CoreMatchers.is; /** - * @author <a href="mailto:borud@yahoo-inc.com">Bjorn Borud</a> + * @author Bjorn Borud */ public class VespaFormatterTestCase { diff --git a/vespalog/src/test/java/com/yahoo/log/VespaLogHandlerTestCase.java b/vespalog/src/test/java/com/yahoo/log/VespaLogHandlerTestCase.java index 4518253561f..3e51d194a5e 100644 --- a/vespalog/src/test/java/com/yahoo/log/VespaLogHandlerTestCase.java +++ b/vespalog/src/test/java/com/yahoo/log/VespaLogHandlerTestCase.java @@ -18,7 +18,7 @@ import static org.hamcrest.core.Is.is; import static org.junit.Assert.*; /** - * @author <a href="mailto:borud@yahoo-inc.com">Bjorn Borud</a> + * @author Bjorn Borud */ public class VespaLogHandlerTestCase { protected static String hostname; diff --git a/vsm/src/vespa/vsm/config/CMakeLists.txt b/vsm/src/vespa/vsm/config/CMakeLists.txt index 43c8fe0218b..ea65d8c8fb4 100644 --- a/vsm/src/vespa/vsm/config/CMakeLists.txt +++ b/vsm/src/vespa/vsm/config/CMakeLists.txt @@ -4,8 +4,8 @@ vespa_add_library(vsm_vconfig OBJECT DEPENDS ) vespa_generate_config(vsm_vconfig vsmfields.def) -install(FILES vsmfields.def DESTINATION var/db/vespa/config_server/serverdb/classes) +install(FILES vsmfields.def RENAME vespa.config.search.vsm.vsmfields.def DESTINATION var/db/vespa/config_server/serverdb/classes) vespa_generate_config(vsm_vconfig vsm.def) -install(FILES vsm.def DESTINATION var/db/vespa/config_server/serverdb/classes) +install(FILES vsm.def RENAME vespa.config.search.vsm.vsm.def DESTINATION var/db/vespa/config_server/serverdb/classes) vespa_generate_config(vsm_vconfig vsmsummary.def) -install(FILES vsmsummary.def DESTINATION var/db/vespa/config_server/serverdb/classes) +install(FILES vsmsummary.def RENAME vespa.config.search.vsm.vsmsummary.def DESTINATION var/db/vespa/config_server/serverdb/classes) diff --git a/vsm/src/vespa/vsm/vsm/docsumconfig.h b/vsm/src/vespa/vsm/vsm/docsumconfig.h index 147a0c3c274..b6a3db45472 100644 --- a/vsm/src/vespa/vsm/vsm/docsumconfig.h +++ b/vsm/src/vespa/vsm/vsm/docsumconfig.h @@ -1,6 +1,4 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// Copyright (C) 1998-2003 Fast Search & Transfer ASA -// Copyright (C) 2003 Overture Services Norway AS #pragma once diff --git a/vsm/src/vespa/vsm/vsm/docsumfilter.h b/vsm/src/vespa/vsm/vsm/docsumfilter.h index f8f2d350698..a3f8a8539f1 100644 --- a/vsm/src/vespa/vsm/vsm/docsumfilter.h +++ b/vsm/src/vespa/vsm/vsm/docsumfilter.h @@ -1,6 +1,4 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// Copyright (C) 1998-2003 Fast Search & Transfer ASA -// Copyright (C) 2003 Overture Services Norway AS #pragma once diff --git a/vsm/src/vespa/vsm/vsm/vsm-adapter.h b/vsm/src/vespa/vsm/vsm/vsm-adapter.h index d8cb81afc07..2bdd0248be9 100644 --- a/vsm/src/vespa/vsm/vsm/vsm-adapter.h +++ b/vsm/src/vespa/vsm/vsm/vsm-adapter.h @@ -1,6 +1,4 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// Copyright (C) 1998-2003 Fast Search & Transfer ASA -// Copyright (C) 2003 Overture Services Norway AS #pragma once diff --git a/zkfacade/src/main/java/com/yahoo/vespa/curator/mock/MockCurator.java b/zkfacade/src/main/java/com/yahoo/vespa/curator/mock/MockCurator.java index ccd02e5c6d6..d3950e4c221 100644 --- a/zkfacade/src/main/java/com/yahoo/vespa/curator/mock/MockCurator.java +++ b/zkfacade/src/main/java/com/yahoo/vespa/curator/mock/MockCurator.java @@ -4,8 +4,8 @@ package com.yahoo.vespa.curator.mock; import com.google.common.util.concurrent.UncheckedTimeoutException; import com.google.inject.Inject; import com.yahoo.collections.Pair; -import com.yahoo.concurrent.Lock; -import com.yahoo.concurrent.Locks; +import com.yahoo.concurrent.lock.Lock; +import com.yahoo.concurrent.lock.Locks; import com.yahoo.path.Path; import static com.yahoo.vespa.curator.mock.MemoryFileSystem.Node; |