diff options
75 files changed, 1051 insertions, 382 deletions
diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java index 47160bdd4a2..d0ec98e3297 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java @@ -54,8 +54,10 @@ public interface ModelContext { boolean isFirstTimeDeployment(); boolean useDedicatedNodeForLogserver(); boolean useFdispatchByDefault(); + boolean dispatchWithProtobuf(); boolean useAdaptiveDispatch(); - boolean useSeparateServiceTypeForLogserverContainer(); + // TODO: Remove when 7.33 is the oldest model in use + default boolean useSeparateServiceTypeForLogserverContainer() { return true; } boolean enableMetricsProxyContainer(); } diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java index 89433cdf7c0..fe6d683adf8 100644 --- a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java +++ b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java @@ -35,8 +35,8 @@ public class TestProperties implements ModelContext.Properties { private boolean isFirstTimeDeployment = false; private boolean useDedicatedNodeForLogserver = false; private boolean useFdispatchByDefault = true; + private boolean dispatchWithProtobuf = true; private boolean useAdaptiveDispatch = false; - private boolean useSeparateServiceTypeForLogserverContainer = false; private boolean enableMetricsProxyContainer = false; @@ -54,7 +54,7 @@ public class TestProperties implements ModelContext.Properties { @Override public boolean useAdaptiveDispatch() { return useAdaptiveDispatch; } @Override public boolean useDedicatedNodeForLogserver() { return useDedicatedNodeForLogserver; } @Override public boolean useFdispatchByDefault() { return useFdispatchByDefault; } - @Override public boolean useSeparateServiceTypeForLogserverContainer() { return useSeparateServiceTypeForLogserverContainer; } + @Override public boolean dispatchWithProtobuf() { return dispatchWithProtobuf; } @Override public boolean enableMetricsProxyContainer() { return enableMetricsProxyContainer; } public TestProperties setApplicationId(ApplicationId applicationId) { @@ -87,11 +87,6 @@ public class TestProperties implements ModelContext.Properties { return this; } - public TestProperties setUseSeparateServiceTypeForLogserverContainer(boolean useSeparateServiceTypeForLogserverContainer) { - this.useSeparateServiceTypeForLogserverContainer = useSeparateServiceTypeForLogserverContainer; - return this; - } - public TestProperties setEnableMetricsProxyContainer(boolean enableMetricsProxyContainer) { this.enableMetricsProxyContainer = enableMetricsProxyContainer; return this; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/LogserverContainer.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/LogserverContainer.java index 3d8773dcc2a..09bc29b181a 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/LogserverContainer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/LogserverContainer.java @@ -5,6 +5,7 @@ import com.yahoo.config.model.producer.AbstractConfigProducer; import com.yahoo.vespa.model.container.Container; import com.yahoo.vespa.model.container.ContainerCluster; import com.yahoo.config.model.api.container.ContainerServiceType; +import com.yahoo.vespa.model.container.component.AccessLogComponent; import com.yahoo.vespa.model.container.component.Handler; /** @@ -18,6 +19,7 @@ public class LogserverContainer extends Container { public LogserverContainer(AbstractConfigProducer parent, boolean useSeparateServiceTypeForLogserverContainer) { super(parent, "" + 0, 0); this.useSeparateServiceTypeForLogserverContainer = useSeparateServiceTypeForLogserverContainer; + addComponent(new AccessLogComponent(AccessLogComponent.AccessLogType.jsonAccessLog, ((LogserverContainerCluster) parent).getName(), true)); } @Override @@ -25,5 +27,4 @@ public class LogserverContainer extends Container { return useSeparateServiceTypeForLogserverContainer ? ContainerServiceType.LOGSERVER_CONTAINER : ContainerServiceType.CONTAINER; } - } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/component/FileStatusHandlerComponent.java b/config-model/src/main/java/com/yahoo/vespa/model/container/component/FileStatusHandlerComponent.java index dc04aa54ba0..ee61b34987a 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/component/FileStatusHandlerComponent.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/component/FileStatusHandlerComponent.java @@ -6,6 +6,7 @@ import com.yahoo.osgi.provider.model.ComponentModel; /** * Sets up VipStatusHandler that answers OK when a certain file is present. + * * @author Tony Vaagenes */ public class FileStatusHandlerComponent extends Handler implements VipStatusConfig.Producer { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java index 9caf7fbdc9e..ee82d0cd719 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java @@ -107,6 +107,7 @@ public class IndexedSearchCluster extends SearchCluster private final DispatchGroup rootDispatch; private DispatchSpec dispatchSpec; private final boolean useFdispatchByDefault; + private final boolean dispatchWithProtobuf; private final boolean useAdaptiveDispatch; private List<SearchNode> searchNodes = new ArrayList<>(); @@ -126,6 +127,7 @@ public class IndexedSearchCluster extends SearchCluster dispatchParent = new SimpleConfigProducer(this, "dispatchers"); rootDispatch = new DispatchGroup(this); useFdispatchByDefault = deployState.getProperties().useFdispatchByDefault(); + dispatchWithProtobuf = deployState.getProperties().dispatchWithProtobuf(); useAdaptiveDispatch = deployState.getProperties().useAdaptiveDispatch(); } @@ -437,6 +439,7 @@ public class IndexedSearchCluster extends SearchCluster builder.maxNodesDownPerGroup(rootDispatch.getMaxNodesDownPerFixedRow()); builder.useMultilevelDispatch(useMultilevelDispatchSetup()); builder.useFdispatchByDefault(useFdispatchByDefault); + builder.dispatchWithProtobuf(dispatchWithProtobuf); builder.useLocalNode(tuning.dispatch.useLocalNode); builder.searchableCopies(rootDispatch.getSearchableCopies()); if (searchCoverage != null) { diff --git a/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java b/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java index f395ef680da..6c9b9fdc084 100644 --- a/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java +++ b/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java @@ -956,27 +956,7 @@ public class ModelProvisioningTest { " </container>" + "</services>"; boolean useDedicatedNodeForLogserver = false; - boolean useSeparateServiceTypeForLogserverContainer = false; - testContainerOnLogserverHost(services, useDedicatedNodeForLogserver, useSeparateServiceTypeForLogserverContainer); - } - - @Test - public void testLogserverContainerWhenDedicatedLogserverSeparateServiceType() { - String services = - "<?xml version='1.0' encoding='utf-8' ?>\n" + - "<services>" + - " <admin version='4.0'>" + - " <logservers>" + - " <nodes count='1' dedicated='true'/>" + - " </logservers>" + - " </admin>" + - " <container version='1.0' id='foo'>" + - " <nodes count='1'/>" + - " </container>" + - "</services>"; - boolean useDedicatedNodeForLogserver = false; - boolean useSeparateServiceTypeForLogserverContainer = true; - testContainerOnLogserverHost(services, useDedicatedNodeForLogserver, useSeparateServiceTypeForLogserverContainer); + testContainerOnLogserverHost(services, useDedicatedNodeForLogserver); } @Test @@ -989,8 +969,7 @@ public class ModelProvisioningTest { " </container>" + "</services>"; boolean useDedicatedNodeForLogserver = true; - boolean useSeparateServiceTypeForLogserverContainer = false; - testContainerOnLogserverHost(services, useDedicatedNodeForLogserver, useSeparateServiceTypeForLogserverContainer); + testContainerOnLogserverHost(services, useDedicatedNodeForLogserver); } @Test @@ -1833,11 +1812,10 @@ public class ModelProvisioningTest { // Tests that a container is allocated on logserver host and that // it is able to get config - private void testContainerOnLogserverHost(String services, boolean useDedicatedNodeForLogserver, boolean useSeparateServiceTypeForLogserverContainer) { + private void testContainerOnLogserverHost(String services, boolean useDedicatedNodeForLogserver) { int numberOfHosts = 2; VespaModelTester tester = new VespaModelTester(); tester.useDedicatedNodeForLogserver(useDedicatedNodeForLogserver); - tester.useSeparateServiceTypeForLogserverContainer(useSeparateServiceTypeForLogserverContainer); tester.addHosts(numberOfHosts); VespaModel model = tester.createModel(Zone.defaultZone(), services, true); @@ -1847,9 +1825,7 @@ public class ModelProvisioningTest { Logserver logserver = admin.getLogserver(); HostResource hostResource = logserver.getHostResource(); assertNotNull(hostResource.getService("logserver")); - String containerServiceType = useSeparateServiceTypeForLogserverContainer - ? ContainerServiceType.LOGSERVER_CONTAINER.serviceName - : ContainerServiceType.CONTAINER.serviceName; + String containerServiceType = ContainerServiceType.LOGSERVER_CONTAINER.serviceName; assertNotNull(hostResource.getService(containerServiceType)); // Test that the container gets config diff --git a/config-model/src/test/java/com/yahoo/vespa/model/admin/DedicatedAdminV4Test.java b/config-model/src/test/java/com/yahoo/vespa/model/admin/DedicatedAdminV4Test.java index f004cc8ff2a..3896b6d799a 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/admin/DedicatedAdminV4Test.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/admin/DedicatedAdminV4Test.java @@ -28,6 +28,7 @@ import java.util.stream.Collectors; import java.util.stream.IntStream; import static com.yahoo.config.model.api.container.ContainerServiceType.CONTAINER; +import static com.yahoo.config.model.api.container.ContainerServiceType.LOGSERVER_CONTAINER; import static com.yahoo.config.model.api.container.ContainerServiceType.METRICS_PROXY_CONTAINER; import static com.yahoo.config.model.api.container.ContainerServiceType.QRSERVER; import static org.junit.Assert.assertEquals; @@ -77,7 +78,7 @@ public class DedicatedAdminV4Test { assertHostContainsServices(model, "hosts/myhost0", "slobrok", "logd"); assertHostContainsServices(model, "hosts/myhost1", "slobrok", "logd"); // Note: A container is always added on logserver host - assertHostContainsServices(model, "hosts/myhost2", "logserver", "logd", CONTAINER.serviceName); + assertHostContainsServices(model, "hosts/myhost2", "logserver", "logd", LOGSERVER_CONTAINER.serviceName); Monitoring monitoring = model.getAdmin().getMonitoring(); assertEquals("vespa.routing", monitoring.getClustername()); @@ -157,7 +158,7 @@ public class DedicatedAdminV4Test { assertHostContainsServices(model, "hosts/myhost0", "logd", "logforwarder", "slobrok"); assertHostContainsServices(model, "hosts/myhost1", "logd", "logforwarder", "slobrok"); // Note: A container is always added on logserver host - assertHostContainsServices(model, "hosts/myhost2", "logd", "logforwarder", "logserver", CONTAINER.serviceName); + assertHostContainsServices(model, "hosts/myhost2", "logd", "logforwarder", "logserver", LOGSERVER_CONTAINER.serviceName); Set<String> configIds = model.getConfigIds(); // 1 logforwarder on each host @@ -200,7 +201,7 @@ public class DedicatedAdminV4Test { .properties(new TestProperties().setHostedVespa(true))); assertEquals(1, model.getHosts().size()); // Should create a container on the same node as logserver - assertHostContainsServices(model, "hosts/myhost0", "slobrok", "logd", "logserver", CONTAINER.serviceName); + assertHostContainsServices(model, "hosts/myhost0", "slobrok", "logd", "logserver", LOGSERVER_CONTAINER.serviceName); } private Set<String> serviceNames(VespaModel model, String hostname) { diff --git a/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTester.java b/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTester.java index 00bc82bbe02..801e138f3c7 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTester.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTester.java @@ -47,7 +47,6 @@ public class VespaModelTester { private Map<String, Collection<Host>> hostsByFlavor = new HashMap<>(); private ApplicationId applicationId = ApplicationId.defaultId(); private boolean useDedicatedNodeForLogserver = false; - private boolean useSeparateServiceTypeForLogserverContainer = false; private boolean enableMetricsProxyContainer = false; public VespaModelTester() { @@ -99,10 +98,6 @@ public class VespaModelTester { this.useDedicatedNodeForLogserver = useDedicatedNodeForLogserver; } - public void useSeparateServiceTypeForLogserverContainer(boolean useSeparateServiceTypeForLogserverContainer) { - this.useSeparateServiceTypeForLogserverContainer = useSeparateServiceTypeForLogserverContainer; - } - public void enableMetricsProxyContainer(boolean enableMetricsProxyContainer) { this.enableMetricsProxyContainer = enableMetricsProxyContainer; } @@ -148,7 +143,6 @@ public class VespaModelTester { .setHostedVespa(hosted) .setApplicationId(applicationId) .setUseDedicatedNodeForLogserver(useDedicatedNodeForLogserver) - .setUseSeparateServiceTypeForLogserverContainer(useSeparateServiceTypeForLogserverContainer) .setEnableMetricsProxyContainer(enableMetricsProxyContainer); DeployState deployState = new DeployState.Builder() diff --git a/configdefinitions/src/vespa/dispatch.def b/configdefinitions/src/vespa/dispatch.def index f0a4b0d8419..7d5979bcdf1 100644 --- a/configdefinitions/src/vespa/dispatch.def +++ b/configdefinitions/src/vespa/dispatch.def @@ -19,6 +19,9 @@ distributionPolicy enum { ROUNDROBIN, ADAPTIVE } default=ROUNDROBIN # Should fdispatch be used as the default dispatcher useFdispatchByDefault bool default=true +# Should protobuf/jrt be preferred over fs4 +dispatchWithProtobuf bool default=false + # Is multi-level dispatch configured for this cluster useMultilevelDispatch bool default=false diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java index 780135c893e..0279d175488 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java @@ -131,7 +131,7 @@ public class ModelContextImpl implements ModelContext { private final boolean useDedicatedNodeForLogserver; private final boolean useFdispatchByDefault; private final boolean useAdaptiveDispatch; - private final boolean useSeparateServiceTypeForLogserverContainer; + private final boolean dispatchWithProtobuf; private final boolean enableMetricsProxyContainer; public Properties(ApplicationId applicationId, @@ -161,9 +161,9 @@ public class ModelContextImpl implements ModelContext { .with(FetchVector.Dimension.APPLICATION_ID, applicationId.serializedForm()).value(); this.useFdispatchByDefault = Flags.USE_FDISPATCH_BY_DEFAULT.bindTo(flagSource) .with(FetchVector.Dimension.APPLICATION_ID, applicationId.serializedForm()).value(); - this.useAdaptiveDispatch = Flags.USE_ADAPTIVE_DISPATCH.bindTo(flagSource) + this.dispatchWithProtobuf = Flags.DISPATCH_WITH_PROTOBUF.bindTo(flagSource) .with(FetchVector.Dimension.APPLICATION_ID, applicationId.serializedForm()).value(); - this.useSeparateServiceTypeForLogserverContainer = Flags.USE_SEPARATE_SERVICE_TYPE_FOR_LOGSERVER_CONTAINER.bindTo(flagSource) + this.useAdaptiveDispatch = Flags.USE_ADAPTIVE_DISPATCH.bindTo(flagSource) .with(FetchVector.Dimension.APPLICATION_ID, applicationId.serializedForm()).value(); this.enableMetricsProxyContainer = Flags.ENABLE_METRICS_PROXY_CONTAINER.bindTo(flagSource) .with(FetchVector.Dimension.APPLICATION_ID, applicationId.serializedForm()).value(); @@ -213,10 +213,10 @@ public class ModelContextImpl implements ModelContext { public boolean useFdispatchByDefault() { return useFdispatchByDefault; } @Override - public boolean useAdaptiveDispatch() { return useAdaptiveDispatch; } + public boolean dispatchWithProtobuf() { return dispatchWithProtobuf; } @Override - public boolean useSeparateServiceTypeForLogserverContainer() { return useSeparateServiceTypeForLogserverContainer; } + public boolean useAdaptiveDispatch() { return useAdaptiveDispatch; } @Override public boolean enableMetricsProxyContainer() { return enableMetricsProxyContainer; } diff --git a/container-core/CMakeLists.txt b/container-core/CMakeLists.txt index 1a2bbabaed3..43225e38aee 100644 --- a/container-core/CMakeLists.txt +++ b/container-core/CMakeLists.txt @@ -12,3 +12,5 @@ install_config_definition(src/main/resources/configdefinitions/qr.def container. install_config_definition(src/main/resources/configdefinitions/servlet-config.def container.servlet.servlet-config.def) install_config_definition(src/main/resources/configdefinitions/threadpool.def container.handler.threadpool.def) install_config_definition(src/main/resources/configdefinitions/vip-status.def container.core.vip-status.def) + +vespa_install_script(src/main/sh/vespa-load-balancer-status libexec/vespa) diff --git a/container-core/src/main/sh/vespa-load-balancer-status b/container-core/src/main/sh/vespa-load-balancer-status new file mode 100755 index 00000000000..e93337333f3 --- /dev/null +++ b/container-core/src/main/sh/vespa-load-balancer-status @@ -0,0 +1,214 @@ +#!/bin/bash +# +# Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +# BEGIN environment bootstrap section +# Do not edit between here and END as this section should stay identical in all scripts + +findpath () { + myname=${0} + mypath=${myname%/*} + myname=${myname##*/} + if [ "$mypath" ] && [ -d "$mypath" ]; then + return + fi + mypath=$(pwd) + if [ -f "${mypath}/${myname}" ]; then + return + fi + echo "FATAL: Could not figure out the path where $myname lives from $0" + exit 1 +} + +COMMON_ENV=libexec/vespa/common-env.sh + +source_common_env () { + if [ "$VESPA_HOME" ] && [ -d "$VESPA_HOME" ]; then + export VESPA_HOME + common_env=$VESPA_HOME/$COMMON_ENV + if [ -f "$common_env" ]; then + . $common_env + return + fi + fi + return 1 +} + +findroot () { + source_common_env && return + if [ "$VESPA_HOME" ]; then + echo "FATAL: bad VESPA_HOME value '$VESPA_HOME'" + exit 1 + fi + if [ "$ROOT" ] && [ -d "$ROOT" ]; then + VESPA_HOME="$ROOT" + source_common_env && return + fi + findpath + while [ "$mypath" ]; do + VESPA_HOME=${mypath} + source_common_env && return + mypath=${mypath%/*} + done + echo "FATAL: missing VESPA_HOME environment variable" + echo "Could not locate $COMMON_ENV anywhere" + exit 1 +} + +findhost () { + if [ "${VESPA_HOSTNAME}" = "" ]; then + VESPA_HOSTNAME=$(vespa-detect-hostname || hostname -f || hostname || echo "localhost") || exit 1 + fi + validate="${VESPA_HOME}/bin/vespa-validate-hostname" + if [ -f "$validate" ]; then + "$validate" "${VESPA_HOSTNAME}" || exit 1 + fi + export VESPA_HOSTNAME +} + +findroot +findhost + +# END environment bootstrap section + +set -eu + +declare LB_STATUS_DIR="$VESPA_HOME"/var/vespa/load-balancer +declare LB_STATUS_FILE="$LB_STATUS_DIR"/status.html +declare LB_OPERATOR_LOG="$LB_STATUS_DIR"/operator.log + +function Usage { + cat <<EOF +Usage: ${0##*/} COMMAND [-u USER] [-f] +Make jdisc container stop serving /status.html. + +Useful when jdisc container is behind a load balancer: The load balancer can be +set up to monitor the health of /status.html requests, and remove bad backends +from serving. + +Command: + get Return info on the current in/out status. + in Undo 'out'. This command is a no-op if 1. status is already in, or 2. + if the the user that set it out is different from USER and -f was NOT + specified. + out Stop answering OK on /status.html requests against jdisc container. + Note: The jdisc container may not answer OK for other reasons too. + +Options: + -u USER Set the user agent. The user setting the status in, must match + the user that set it out. Defaults to current user. + -f Force-set status: Ignore any mismatch on user. +EOF + + exit 0 +} + +function PrintPair { + printf "%-19s %s\n" "$1:" "$2" +} + +function IsIn { + if [ -r "$LB_STATUS_FILE" ]; then + return 0 + else + return 1 + fi +} + +function DifferentUserSetOut { + local user="$1" + + if [ -r "$LB_OPERATOR_LOG" ]; then + local out_user + out_user=$(< "$LB_OPERATOR_LOG") + if [ "$user" != "$out_user" ]; then + return 0 + fi + fi + + return 1 +} + +function GetCommand { + if IsIn; then + PrintPair "VIP status" IN + else + PrintPair "VIP status" OUT + fi + PrintPair "Status file" "$LB_STATUS_FILE" + + if [ -r "$LB_OPERATOR_LOG" ]; then + PrintPair "Last modified" "$(stat -c %y "$LB_OPERATOR_LOG")" + PrintPair "Last modified by" "$(< "$LB_OPERATOR_LOG")" + fi +} + +function InCommand { + local user="$1" + local force="$2" + + if ! $force; then + if IsIn || DifferentUserSetOut "$user"; then + return + fi + fi + + mkdir -p "$LB_STATUS_DIR" + echo "$user" > "$LB_OPERATOR_LOG" + echo OK > "$LB_STATUS_FILE" +} + +function OutCommand { + local user="$1" + local force="$2" + + if ! $force && ! IsIn; then + return + fi + + mkdir -p "$LB_STATUS_DIR" + echo "$user" > "$LB_OPERATOR_LOG" + rm -f "$LB_STATUS_FILE" +} + +function Main { + if (($# == 0)); then + Usage + fi + + local command= + local user="${SUDO_USER:-${USER:-$(id -nu)}}" + local force=false + + # Supports placement of options both before and after command. + while (($# > 0)); do + case "$1" in + -f) + force=true + shift + ;; + -u) + user="$2" + shift 2 + ;; + -*) Usage "Unknown option '$1'" ;; + *) + case "$1" in + get) command="GetCommand" ;; + in) command="InCommand" ;; + out) command="OutCommand" ;; + *) Usage ;; + esac + shift + ;; + esac + done + + if [ -z "$command" ]; then + Usage + fi + + "$command" "$user" "$force" +} + +Main "$@" diff --git a/container-search/src/main/java/com/yahoo/fs4/PongPacket.java b/container-search/src/main/java/com/yahoo/fs4/PongPacket.java index 13fb7d84408..37aaf7067a9 100644 --- a/container-search/src/main/java/com/yahoo/fs4/PongPacket.java +++ b/container-search/src/main/java/com/yahoo/fs4/PongPacket.java @@ -28,7 +28,7 @@ public class PongPacket extends BasicPacket { /** For testing */ public PongPacket(long activeDocuments) { - this.activeDocuments = Optional.of(activeDocuments); + this.activeDocuments = Optional.of(activeDocuments); } private int code; diff --git a/container-search/src/main/java/com/yahoo/prelude/Pong.java b/container-search/src/main/java/com/yahoo/prelude/Pong.java index cef64c293af..a6bc3e7975d 100644 --- a/container-search/src/main/java/com/yahoo/prelude/Pong.java +++ b/container-search/src/main/java/com/yahoo/prelude/Pong.java @@ -1,15 +1,15 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.prelude; +import com.yahoo.fs4.PongPacket; +import com.yahoo.search.result.ErrorMessage; +import com.yahoo.search.statistics.ElapsedTime; + import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.Optional; -import com.yahoo.fs4.PongPacket; -import com.yahoo.search.result.ErrorMessage; -import com.yahoo.search.statistics.ElapsedTime; - /** * An answer from Ping. * @@ -21,20 +21,29 @@ public class Pong { private final List<ErrorMessage> errors = new ArrayList<>(1); private final Optional<PongPacket> pongPacket; private ElapsedTime elapsed = new ElapsedTime(); + private final Optional<Long> activeDocuments; public Pong() { this.pongPacket = Optional.empty(); + this.activeDocuments = Optional.empty(); } - + public Pong(ErrorMessage error) { errors.add(error); this.pongPacket = Optional.empty(); + this.activeDocuments = Optional.empty(); } - + public Pong(PongPacket pongPacket) { this.pongPacket = Optional.of(pongPacket); + this.activeDocuments = Optional.empty(); } - + + public Pong(long activeDocuments) { + this.pongPacket = Optional.empty(); + this.activeDocuments = Optional.of(activeDocuments); + } + public void addError(ErrorMessage error) { errors.add(error); } @@ -49,6 +58,7 @@ public class Pong { /** Returns the number of active documents in the backend responding in this Pong, if available */ public Optional<Long> activeDocuments() { + if (activeDocuments.isPresent()) return activeDocuments; if ( ! pongPacket.isPresent()) return Optional.empty(); return pongPacket.get().getActiveDocuments(); } diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/FS4InvokerFactory.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/FS4InvokerFactory.java index ac99675c9c5..b9af60089f8 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/FS4InvokerFactory.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/FS4InvokerFactory.java @@ -3,13 +3,16 @@ package com.yahoo.prelude.fastsearch; import com.google.common.collect.ImmutableMap; import com.yahoo.fs4.mplex.Backend; +import com.yahoo.prelude.Pong; import com.yahoo.search.Query; import com.yahoo.search.Result; +import com.yahoo.search.cluster.ClusterMonitor; import com.yahoo.search.dispatch.FillInvoker; import com.yahoo.search.dispatch.InterleavedFillInvoker; import com.yahoo.search.dispatch.InvokerFactory; import com.yahoo.search.dispatch.SearchInvoker; import com.yahoo.search.dispatch.searchcluster.Node; +import com.yahoo.search.dispatch.searchcluster.Pinger; import com.yahoo.search.dispatch.searchcluster.SearchCluster; import com.yahoo.search.result.Hit; @@ -20,6 +23,7 @@ import java.util.Iterator; import java.util.Map; import java.util.Optional; import java.util.Set; +import java.util.concurrent.Callable; /** * FS4InvokerFactory constructs {@link FillInvoker} and {@link SearchInvoker} objects that communicate with @@ -97,4 +101,8 @@ public class FS4InvokerFactory extends InvokerFactory { return requiredNodes; } + @Override + public Callable<Pong> createPinger(Node node, ClusterMonitor<Node> monitor) { + return new Pinger(node, monitor, fs4ResourcePool); + } } diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java b/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java index 74d9c38b273..f2dbb1e8557 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java @@ -56,6 +56,7 @@ public class Dispatcher extends AbstractComponent { private final LoadBalancer loadBalancer; private final boolean multilevelDispatch; private final boolean internalDispatchByDefault; + private final boolean dispatchWithProtobuf; private final FS4InvokerFactory fs4InvokerFactory; private final RpcInvokerFactory rpcInvokerFactory; @@ -65,14 +66,12 @@ public class Dispatcher extends AbstractComponent { public Dispatcher(String clusterId, DispatchConfig dispatchConfig, FS4ResourcePool fs4ResourcePool, int containerClusterSize, VipStatus vipStatus, Metric metric) { - this(new SearchCluster(clusterId, dispatchConfig, fs4ResourcePool, containerClusterSize, vipStatus), dispatchConfig, - fs4ResourcePool, new RpcResourcePool(dispatchConfig), metric); + this(new SearchCluster(clusterId, dispatchConfig, containerClusterSize, vipStatus), dispatchConfig, fs4ResourcePool, metric); } - public Dispatcher(SearchCluster searchCluster, DispatchConfig dispatchConfig, FS4ResourcePool fs4ResourcePool, - RpcResourcePool rpcResourcePool, Metric metric) { + public Dispatcher(SearchCluster searchCluster, DispatchConfig dispatchConfig, FS4ResourcePool fs4ResourcePool, Metric metric) { this(searchCluster, dispatchConfig, new FS4InvokerFactory(fs4ResourcePool, searchCluster), - new RpcInvokerFactory(rpcResourcePool, searchCluster), metric); + new RpcInvokerFactory(new RpcResourcePool(dispatchConfig), searchCluster, dispatchConfig.dispatchWithProtobuf()), metric); } public Dispatcher(SearchCluster searchCluster, DispatchConfig dispatchConfig, FS4InvokerFactory fs4InvokerFactory, @@ -82,12 +81,15 @@ public class Dispatcher extends AbstractComponent { dispatchConfig.distributionPolicy() == DispatchConfig.DistributionPolicy.ROUNDROBIN); this.multilevelDispatch = dispatchConfig.useMultilevelDispatch(); this.internalDispatchByDefault = !dispatchConfig.useFdispatchByDefault(); + this.dispatchWithProtobuf = dispatchConfig.dispatchWithProtobuf(); this.fs4InvokerFactory = fs4InvokerFactory; this.rpcInvokerFactory = rpcInvokerFactory; this.metric = metric; this.metricContext = metric.createContext(null); + + searchCluster.startClusterMonitoring(dispatchWithProtobuf ? rpcInvokerFactory : fs4InvokerFactory); } /** Returns the search cluster this dispatches to */ @@ -120,7 +122,8 @@ public class Dispatcher extends AbstractComponent { return Optional.empty(); } - InvokerFactory factory = query.properties().getBoolean(dispatchProtobuf, false) ? rpcInvokerFactory : fs4InvokerFactory; + InvokerFactory factory = query.properties().getBoolean(dispatchProtobuf, dispatchWithProtobuf) + ? rpcInvokerFactory : fs4InvokerFactory; Optional<SearchInvoker> invoker = getSearchPathInvoker(query, factory, searcher); diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/InvokerFactory.java b/container-search/src/main/java/com/yahoo/search/dispatch/InvokerFactory.java index 815a2a257ea..8617c74ec41 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/InvokerFactory.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/InvokerFactory.java @@ -1,9 +1,11 @@ // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.dispatch; +import com.yahoo.prelude.Pong; import com.yahoo.prelude.fastsearch.VespaBackEndSearcher; import com.yahoo.search.Query; import com.yahoo.search.Result; +import com.yahoo.search.cluster.ClusterMonitor; import com.yahoo.search.dispatch.searchcluster.Node; import com.yahoo.search.dispatch.searchcluster.SearchCluster; import com.yahoo.search.result.Coverage; @@ -15,6 +17,7 @@ import java.util.List; import java.util.Optional; import java.util.OptionalInt; import java.util.Set; +import java.util.concurrent.Callable; /** * @author ollivir @@ -30,6 +33,8 @@ public abstract class InvokerFactory { public abstract Optional<FillInvoker> createFillInvoker(VespaBackEndSearcher searcher, Result result); + public abstract Callable<Pong> createPinger(Node node, ClusterMonitor<Node> monitor); + /** * Create a {@link SearchInvoker} for a list of content nodes. * diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/Client.java b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/Client.java index 4422538cff6..cc37df04a62 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/Client.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/Client.java @@ -2,6 +2,8 @@ package com.yahoo.search.dispatch.rpc; import com.yahoo.compress.CompressionType; +import com.yahoo.compress.Compressor; +import com.yahoo.prelude.Pong; import com.yahoo.prelude.fastsearch.FastHit; import java.util.List; diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcInvokerFactory.java b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcInvokerFactory.java index c1b164aaeaa..b0a418241f8 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcInvokerFactory.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcInvokerFactory.java @@ -1,11 +1,13 @@ // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.dispatch.rpc; +import com.yahoo.prelude.Pong; import com.yahoo.prelude.fastsearch.DocumentDatabase; import com.yahoo.prelude.fastsearch.VespaBackEndSearcher; import com.yahoo.processing.request.CompoundName; import com.yahoo.search.Query; import com.yahoo.search.Result; +import com.yahoo.search.cluster.ClusterMonitor; import com.yahoo.search.dispatch.Dispatcher; import com.yahoo.search.dispatch.FillInvoker; import com.yahoo.search.dispatch.InvokerFactory; @@ -14,6 +16,7 @@ import com.yahoo.search.dispatch.searchcluster.Node; import com.yahoo.search.dispatch.searchcluster.SearchCluster; import java.util.Optional; +import java.util.concurrent.Callable; /** * @author ollivir @@ -23,10 +26,12 @@ public class RpcInvokerFactory extends InvokerFactory { private final static CompoundName dispatchSummaries = new CompoundName("dispatch.summaries"); private final RpcResourcePool rpcResourcePool; + private final boolean dispatchWithProtobuf; - public RpcInvokerFactory(RpcResourcePool rpcResourcePool, SearchCluster searchCluster) { + public RpcInvokerFactory(RpcResourcePool rpcResourcePool, SearchCluster searchCluster, boolean dispatchWithProtobuf) { super(searchCluster); this.rpcResourcePool = rpcResourcePool; + this.dispatchWithProtobuf = dispatchWithProtobuf; } @Override @@ -40,7 +45,7 @@ public class RpcInvokerFactory extends InvokerFactory { boolean summaryNeedsQuery = searcher.summaryNeedsQuery(query); - if(query.properties().getBoolean(Dispatcher.dispatchProtobuf, false)) { + if(query.properties().getBoolean(Dispatcher.dispatchProtobuf, dispatchWithProtobuf)) { return Optional.of(new RpcProtobufFillInvoker(rpcResourcePool, searcher.getDocumentDatabase(query), searcher.getServerId(), summaryNeedsQuery)); } @@ -62,4 +67,9 @@ public class RpcInvokerFactory extends InvokerFactory { public void release() { rpcResourcePool.release(); } + + @Override + public Callable<Pong> createPinger(Node node, ClusterMonitor<Node> monitor) { + return new RpcPing(node, monitor, rpcResourcePool); + } } diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcPing.java b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcPing.java new file mode 100644 index 00000000000..f3479e2e4a9 --- /dev/null +++ b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcPing.java @@ -0,0 +1,77 @@ +package com.yahoo.search.dispatch.rpc; + +import ai.vespa.searchlib.searchprotocol.protobuf.SearchProtocol; +import com.google.protobuf.InvalidProtocolBufferException; +import com.yahoo.compress.CompressionType; +import com.yahoo.compress.Compressor; +import com.yahoo.prelude.Pong; +import com.yahoo.search.cluster.ClusterMonitor; +import com.yahoo.search.dispatch.rpc.Client.ProtobufResponse; +import com.yahoo.search.dispatch.rpc.Client.ResponseOrError; +import com.yahoo.search.dispatch.searchcluster.Node; +import com.yahoo.search.result.ErrorMessage; +import com.yahoo.yolean.Exceptions; + +import java.util.concurrent.Callable; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.TimeUnit; + +public class RpcPing implements Callable<Pong> { + private static final String RPC_METHOD = "vespa.searchprotocol.ping"; + private static final CompressionType PING_COMPRESSION = CompressionType.NONE; + + private final Node node; + private final RpcResourcePool resourcePool; + private final ClusterMonitor<Node> clusterMonitor; + + public RpcPing(Node node, ClusterMonitor<Node> clusterMonitor, RpcResourcePool rpcResourcePool) { + this.node = node; + this.resourcePool = rpcResourcePool; + this.clusterMonitor = clusterMonitor; + } + + @Override + public Pong call() throws Exception { + try { + var queue = new LinkedBlockingQueue<ResponseOrError<ProtobufResponse>>(1); + + sendPing(queue); + + var responseOrError = queue.poll(clusterMonitor.getConfiguration().getRequestTimeout(), TimeUnit.MILLISECONDS); + if (responseOrError == null) { + return new Pong(ErrorMessage.createTimeout("Timed out waiting for pong from " + node)); + } else if (responseOrError.error().isPresent()) { + return new Pong(ErrorMessage.createBackendCommunicationError(responseOrError.error().get())); + } + + return decodeReply(responseOrError.response().get()); + } catch (RuntimeException e) { + return new Pong( + ErrorMessage.createBackendCommunicationError("Exception when pinging " + node + ": " + Exceptions.toMessageString(e))); + } + } + + private void sendPing(LinkedBlockingQueue<ResponseOrError<ProtobufResponse>> queue) { + var connection = resourcePool.nodeConnections().get(node.key()); + var ping = SearchProtocol.MonitorRequest.newBuilder().build().toByteArray(); + double timeoutSeconds = ((double) clusterMonitor.getConfiguration().getRequestTimeout()) / 1000.0; + Compressor.Compression compressionResult = resourcePool.compressor().compress(PING_COMPRESSION, ping); + resourcePool.client().request(RPC_METHOD, connection, compressionResult.type(), ping.length, compressionResult.data(), + rsp -> queue.add(rsp), timeoutSeconds); + } + + private Pong decodeReply(ProtobufResponse response) throws InvalidProtocolBufferException { + CompressionType compression = CompressionType.valueOf(response.compression()); + byte[] responseBytes = resourcePool.compressor().decompress(response.compressedPayload(), compression, response.uncompressedSize()); + var reply = SearchProtocol.MonitorReply.parseFrom(responseBytes); + + if (reply.getDistributionKey() != node.key()) { + return new Pong(ErrorMessage.createBackendCommunicationError( + "Expected pong from node id " + node.key() + ", response is from id " + reply.getDistributionKey())); + } else if (!reply.getOnline()) { + return new Pong(ErrorMessage.createBackendCommunicationError("Node id " + node.key() + " reports being offline")); + } else { + return new Pong(reply.getActiveDocs()); + } + } +} diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/Node.java b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/Node.java index 7e0e3117628..a71ce0354f9 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/Node.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/Node.java @@ -53,7 +53,7 @@ public class Node { public boolean isWorking() { return working.get(); } /** Updates the active documents on this node */ - void setActiveDocuments(long activeDocuments) { + public void setActiveDocuments(long activeDocuments) { this.activeDocuments.set(activeDocuments); } diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/Pinger.java b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/Pinger.java index 7c7a9cb1d1c..dea6f741bb0 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/Pinger.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/Pinger.java @@ -15,7 +15,7 @@ import java.util.concurrent.Callable; * @author bratseth * @author ollivir */ -class Pinger implements Callable<Pong> { +public class Pinger implements Callable<Pong> { private final Node node; private final ClusterMonitor<Node> clusterMonitor; private final FS4ResourcePool fs4ResourcePool; @@ -30,8 +30,6 @@ class Pinger implements Callable<Pong> { try { Pong pong = FastSearcher.ping(new Ping(clusterMonitor.getConfiguration().getRequestTimeout()), fs4ResourcePool.getBackend(node.hostname(), node.fs4port()), node.toString()); - if (pong.activeDocuments().isPresent()) - node.setActiveDocuments(pong.activeDocuments().get()); return pong; } catch (RuntimeException e) { return new Pong(ErrorMessage.createBackendCommunicationError("Exception when pinging " + node + ": " diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java index 5c3ef98c523..6c28352f27b 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java @@ -7,12 +7,12 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableMultimap; import com.yahoo.container.handler.VipStatus; import com.yahoo.net.HostName; +import com.yahoo.prelude.Pong; import com.yahoo.search.cluster.ClusterMonitor; import com.yahoo.search.cluster.NodeManager; +import com.yahoo.search.dispatch.InvokerFactory; import com.yahoo.search.result.ErrorMessage; import com.yahoo.vespa.config.search.DispatchConfig; -import com.yahoo.prelude.Pong; -import com.yahoo.prelude.fastsearch.FS4ResourcePool; import java.util.LinkedHashMap; import java.util.List; @@ -46,6 +46,7 @@ public class SearchCluster implements NodeManager<Node> { private final ImmutableList<Group> orderedGroups; private final ClusterMonitor<Node> clusterMonitor; private final VipStatus vipStatus; + private InvokerFactory pingFactory; /** * A search node on this local machine having the entire corpus, which we therefore @@ -57,13 +58,9 @@ public class SearchCluster implements NodeManager<Node> { */ private final Optional<Node> directDispatchTarget; - // Only needed until query requests are moved to rpc - private final FS4ResourcePool fs4ResourcePool; - - public SearchCluster(String clusterId, DispatchConfig dispatchConfig, FS4ResourcePool fs4ResourcePool, int containerClusterSize, VipStatus vipStatus) { + public SearchCluster(String clusterId, DispatchConfig dispatchConfig, int containerClusterSize, VipStatus vipStatus) { this.clusterId = clusterId; this.dispatchConfig = dispatchConfig; - this.fs4ResourcePool = fs4ResourcePool; this.vipStatus = vipStatus; List<Node> nodes = toNodes(dispatchConfig); @@ -89,15 +86,20 @@ public class SearchCluster implements NodeManager<Node> { this.directDispatchTarget = findDirectDispatchTarget(HostName.getLocalhost(), size, containerClusterSize, nodesByHost, groups); - // Set up monitoring of the fs4 interface of the nodes - // We can switch to monitoring the rpc interface instead when we move the query phase to rpc this.clusterMonitor = new ClusterMonitor<>(this); - for (Node node : nodes) { - // cluster monitor will only call working() when the - // node transitions from down to up, so we need to - // register the initial (working) state here: - working(node); - clusterMonitor.add(node, true); + } + + public void startClusterMonitoring(InvokerFactory pingFactory) { + this.pingFactory = pingFactory; + + for (var group : orderedGroups) { + for (var node : group.nodes()) { + // cluster monitor will only call working() when the + // node transitions from down to up, so we need to + // register the initial (working) state here: + working(node); + clusterMonitor.add(node, true); + } } } @@ -251,16 +253,21 @@ public class SearchCluster implements NodeManager<Node> { /** Used by the cluster monitor to manage node status */ @Override public void ping(Node node, Executor executor) { - Pinger pinger = new Pinger(node, clusterMonitor, fs4ResourcePool); - FutureTask<Pong> futurePong = new FutureTask<>(pinger); + if (pingFactory == null) // not initialized yet + return; + FutureTask<Pong> futurePong = new FutureTask<>(pingFactory.createPinger(node, clusterMonitor)); executor.execute(futurePong); Pong pong = getPong(futurePong, node); futurePong.cancel(true); - if (pong.badResponse()) + if (pong.badResponse()) { clusterMonitor.failed(node, pong.getError(0)); - else + } else { + if (pong.activeDocuments().isPresent()) { + node.setActiveDocuments(pong.activeDocuments().get()); + } clusterMonitor.responded(node); + } } /** diff --git a/container-search/src/test/java/com/yahoo/search/dispatch/DispatcherTest.java b/container-search/src/test/java/com/yahoo/search/dispatch/DispatcherTest.java index 859e10dbe2c..25aed879a48 100644 --- a/container-search/src/test/java/com/yahoo/search/dispatch/DispatcherTest.java +++ b/container-search/src/test/java/com/yahoo/search/dispatch/DispatcherTest.java @@ -137,7 +137,7 @@ public class DispatcherTest { public class MockRpcInvokerFactory extends RpcInvokerFactory { public MockRpcInvokerFactory() { - super(null, null); + super(null, null, true); } @Override diff --git a/container-search/src/test/java/com/yahoo/search/dispatch/LoadBalancerTest.java b/container-search/src/test/java/com/yahoo/search/dispatch/LoadBalancerTest.java index 141d87a41ab..1ebf7940f25 100644 --- a/container-search/src/test/java/com/yahoo/search/dispatch/LoadBalancerTest.java +++ b/container-search/src/test/java/com/yahoo/search/dispatch/LoadBalancerTest.java @@ -29,7 +29,7 @@ public class LoadBalancerTest { @Test public void requireThatLoadBalancerServesSingleNodeSetups() { Node n1 = new Node(0, "test-node1", 0, 0); - SearchCluster cluster = new SearchCluster("a", createDispatchConfig(n1), null, 1, null); + SearchCluster cluster = new SearchCluster("a", createDispatchConfig(n1), 1, null); LoadBalancer lb = new LoadBalancer(cluster, true); Optional<Group> grp = lb.takeGroup(null); @@ -43,7 +43,7 @@ public class LoadBalancerTest { public void requireThatLoadBalancerServesMultiGroupSetups() { Node n1 = new Node(0, "test-node1", 0, 0); Node n2 = new Node(1, "test-node2", 1, 1); - SearchCluster cluster = new SearchCluster("a", createDispatchConfig(n1, n2), null, 1, null); + SearchCluster cluster = new SearchCluster("a", createDispatchConfig(n1, n2), 1, null); LoadBalancer lb = new LoadBalancer(cluster, true); Optional<Group> grp = lb.takeGroup(null); @@ -59,7 +59,7 @@ public class LoadBalancerTest { Node n2 = new Node(1, "test-node2", 1, 0); Node n3 = new Node(0, "test-node3", 0, 1); Node n4 = new Node(1, "test-node4", 1, 1); - SearchCluster cluster = new SearchCluster("a", createDispatchConfig(n1, n2, n3, n4), null, 2, null); + SearchCluster cluster = new SearchCluster("a", createDispatchConfig(n1, n2, n3, n4), 2, null); LoadBalancer lb = new LoadBalancer(cluster, true); Optional<Group> grp = lb.takeGroup(null); @@ -70,7 +70,7 @@ public class LoadBalancerTest { public void requireThatLoadBalancerReturnsDifferentGroups() { Node n1 = new Node(0, "test-node1", 0, 0); Node n2 = new Node(1, "test-node2", 1, 1); - SearchCluster cluster = new SearchCluster("a", createDispatchConfig(n1, n2), null, 1, null); + SearchCluster cluster = new SearchCluster("a", createDispatchConfig(n1, n2), 1, null); LoadBalancer lb = new LoadBalancer(cluster, true); // get first group diff --git a/container-search/src/test/java/com/yahoo/search/dispatch/MockSearchCluster.java b/container-search/src/test/java/com/yahoo/search/dispatch/MockSearchCluster.java index 3919bc26bdc..42a22f6f86b 100644 --- a/container-search/src/test/java/com/yahoo/search/dispatch/MockSearchCluster.java +++ b/container-search/src/test/java/com/yahoo/search/dispatch/MockSearchCluster.java @@ -28,7 +28,7 @@ public class MockSearchCluster extends SearchCluster { } public MockSearchCluster(String clusterId, DispatchConfig dispatchConfig, int groups, int nodesPerGroup) { - super(clusterId, dispatchConfig, null, 1, null); + super(clusterId, dispatchConfig, 1, null); ImmutableList.Builder<Group> orderedGroupBuilder = ImmutableList.builder(); ImmutableMap.Builder<Integer, Group> groupBuilder = ImmutableMap.builder(); diff --git a/container-search/src/test/java/com/yahoo/search/dispatch/rpc/FillTestCase.java b/container-search/src/test/java/com/yahoo/search/dispatch/rpc/FillTestCase.java index 2adbd12a2aa..e059008acac 100644 --- a/container-search/src/test/java/com/yahoo/search/dispatch/rpc/FillTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/dispatch/rpc/FillTestCase.java @@ -39,7 +39,7 @@ public class FillTestCase { nodes.put(1, client.createConnection("host1", 123)); nodes.put(2, client.createConnection("host2", 123)); RpcResourcePool rpcResourcePool = new RpcResourcePool(client, nodes); - RpcInvokerFactory factory = new RpcInvokerFactory(rpcResourcePool, null); + RpcInvokerFactory factory = new RpcInvokerFactory(rpcResourcePool, null, true); Query query = new Query(); Result result = new Result(query); @@ -76,7 +76,7 @@ public class FillTestCase { nodes.put(1, client.createConnection("host1", 123)); nodes.put(2, client.createConnection("host2", 123)); RpcResourcePool rpcResourcePool = new RpcResourcePool(client, nodes); - RpcInvokerFactory factory = new RpcInvokerFactory(rpcResourcePool, null); + RpcInvokerFactory factory = new RpcInvokerFactory(rpcResourcePool, null, true); Query query = new Query(); Result result = new Result(query); @@ -116,7 +116,7 @@ public class FillTestCase { Map<Integer, Client.NodeConnection> nodes = new HashMap<>(); nodes.put(0, client.createConnection("host0", 123)); RpcResourcePool rpcResourcePool = new RpcResourcePool(client, nodes); - RpcInvokerFactory factory = new RpcInvokerFactory(rpcResourcePool, null); + RpcInvokerFactory factory = new RpcInvokerFactory(rpcResourcePool, null, true); Query query = new Query(); Result result = new Result(query); @@ -134,7 +134,7 @@ public class FillTestCase { Map<Integer, Client.NodeConnection> nodes = new HashMap<>(); nodes.put(0, client.createConnection("host0", 123)); RpcResourcePool rpcResourcePool = new RpcResourcePool(client, nodes); - RpcInvokerFactory factory = new RpcInvokerFactory(rpcResourcePool, null); + RpcInvokerFactory factory = new RpcInvokerFactory(rpcResourcePool, null, true); Query query = new Query(); Result result = new Result(query); diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/resource/package-info.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/resource/package-info.java new file mode 100644 index 00000000000..bfb44eb94ba --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/resource/package-info.java @@ -0,0 +1,5 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.vespa.hosted.controller.api.integration.resource; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java index da0eedbdd36..506231f086b 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java @@ -86,7 +86,7 @@ public class ControllerMaintenance extends AbstractComponent { contactInformationMaintainer = new ContactInformationMaintainer(controller, Duration.ofHours(12), jobControl, contactRetriever); costReportMaintainer = new CostReportMaintainer(controller, Duration.ofHours(2), reportConsumer, jobControl, nodeRepositoryClient, Clock.systemUTC(), selfHostedCostConfig); routingPolicyMaintainer = new RoutingPolicyMaintainer(controller, Duration.ofMinutes(5), jobControl, nameService, curator); - resourceMeterMaintainer = new ResourceMeterMaintainer(controller, Duration.ofMinutes(5), jobControl, nodeRepositoryClient, Clock.systemUTC(), resourceSnapshotConsumer); + resourceMeterMaintainer = new ResourceMeterMaintainer(controller, Duration.ofMinutes(5), jobControl, nodeRepositoryClient, Clock.systemUTC(), metric, resourceSnapshotConsumer); } public Upgrader upgrader() { return upgrader; } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CostReportMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CostReportMaintainer.java index a76d472cc89..2b26e93aeb8 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CostReportMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CostReportMaintainer.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.hosted.controller.maintenance; import com.google.inject.Inject; +import com.yahoo.config.provision.CloudName; import com.yahoo.config.provision.SystemName; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeRepositoryClientInterface; @@ -46,6 +47,6 @@ public class CostReportMaintainer extends Maintainer { @Override protected void maintain() { - consumer.Consume(CostCalculator.resourceShareByPropertyToCsv(nodeRepository, controller(), clock, selfHostedCostConfig)); + consumer.Consume(CostCalculator.resourceShareByPropertyToCsv(nodeRepository, controller(), clock, selfHostedCostConfig, CloudName.from("yahoo"))); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainer.java index d9bd3e8131b..5ed7a14836e 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainer.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.config.provision.SystemName; +import com.yahoo.jdisc.Metric; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.config.provision.ApplicationId; import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeOwner; @@ -14,11 +15,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.resource.ResourceAlloca import java.time.Clock; import java.time.Duration; import java.time.Instant; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Set; +import java.util.*; import java.util.stream.Collectors; import static com.yahoo.yolean.Exceptions.uncheck; @@ -26,23 +23,31 @@ import static com.yahoo.yolean.Exceptions.uncheck; /** * Creates a ResourceSnapshot per application, which is then passed on to a ResourceSnapshotConsumer * TODO: Write JSON blob of node repo somewhere + * * @author olaa */ public class ResourceMeterMaintainer extends Maintainer { private final Clock clock; + private final Metric metric; private final NodeRepositoryClientInterface nodeRepository; private final ResourceSnapshotConsumer resourceSnapshotConsumer; + private static final String metering_last_reported = "metering_last_reported"; + private static final String metering_total_reported = "metering_total_reported"; + + @SuppressWarnings("WeakerAccess") public ResourceMeterMaintainer(Controller controller, Duration interval, JobControl jobControl, NodeRepositoryClientInterface nodeRepository, Clock clock, + Metric metric, ResourceSnapshotConsumer resourceSnapshotConsumer) { super(controller, interval, jobControl, ResourceMeterMaintainer.class.getSimpleName(), Set.of(SystemName.cd)); this.clock = clock; this.nodeRepository = nodeRepository; + this.metric = metric; this.resourceSnapshotConsumer = resourceSnapshotConsumer; } @@ -61,6 +66,13 @@ public class ResourceMeterMaintainer extends Maintainer { resourceSnapshotConsumer.consume(resourceSnapshots); + + metric.set(metering_last_reported, clock.millis() / 1000, metric.createContext(Collections.emptyMap())); + metric.set(metering_total_reported, resourceSnapshots.values().stream() + .map(ResourceSnapshot::getResourceAllocation) + .mapToDouble(r -> r.getCpuCores() + r.getMemoryGb() + r.getDiskGb()) // total metered resource usage, for alerting on drastic changes + .sum() + , metric.createContext(Collections.emptyMap())); } private List<NodeRepositoryNode> getNodes() { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java index 69a4216d221..402f91f1a14 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java @@ -110,6 +110,8 @@ import static java.util.stream.Collectors.joining; @SuppressWarnings("unused") // created by injection public class ApplicationApiHandler extends LoggingRequestHandler { + private static final String OPTIONAL_PREFIX = "/api"; + private final Controller controller; private final AccessControlRequests accessControlRequests; @@ -162,7 +164,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler { } private HttpResponse handleGET(HttpRequest request) { - Path path = new Path(request.getUri().getPath()); + Path path = new Path(request.getUri().getPath(), OPTIONAL_PREFIX); if (path.matches("/application/v4/")) return root(request); if (path.matches("/application/v4/user")) return authenticatedUser(request); if (path.matches("/application/v4/tenant")) return tenants(request); @@ -185,7 +187,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler { } private HttpResponse handlePUT(HttpRequest request) { - Path path = new Path(request.getUri().getPath()); + Path path = new Path(request.getUri().getPath(), OPTIONAL_PREFIX); if (path.matches("/application/v4/user")) return createUser(request); if (path.matches("/application/v4/tenant/{tenant}")) return updateTenant(path.get("tenant"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/global-rotation/override")) @@ -194,7 +196,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler { } private HttpResponse handlePOST(HttpRequest request) { - Path path = new Path(request.getUri().getPath()); + Path path = new Path(request.getUri().getPath(), OPTIONAL_PREFIX); if (path.matches("/application/v4/tenant/{tenant}")) return createTenant(path.get("tenant"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}")) return createApplication(path.get("tenant"), path.get("application"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/promote")) return promoteApplication(path.get("tenant"), path.get("application"), request); @@ -213,14 +215,14 @@ public class ApplicationApiHandler extends LoggingRequestHandler { } private HttpResponse handlePATCH(HttpRequest request) { - Path path = new Path(request.getUri().getPath()); + Path path = new Path(request.getUri().getPath(), OPTIONAL_PREFIX); if (path.matches("/application/v4/tenant/{tenant}/application/{application}")) return setMajorVersion(path.get("tenant"), path.get("application"), request); return ErrorResponse.notFoundError("Nothing at " + path); } private HttpResponse handleDELETE(HttpRequest request) { - Path path = new Path(request.getUri().getPath()); + Path path = new Path(request.getUri().getPath(), OPTIONAL_PREFIX); if (path.matches("/application/v4/tenant/{tenant}")) return deleteTenant(path.get("tenant"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}")) return deleteApplication(path.get("tenant"), path.get("application"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/deploying")) return cancelDeploy(path.get("tenant"), path.get("application"), "all"); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/cost/CostApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/cost/CostApiHandler.java index 444153089da..a82d2f22e74 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/cost/CostApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/cost/CostApiHandler.java @@ -1,5 +1,6 @@ package com.yahoo.vespa.hosted.controller.restapi.cost; +import com.yahoo.config.provision.CloudName; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.container.jdisc.LoggingRequestHandler; @@ -11,6 +12,7 @@ import com.yahoo.vespa.hosted.controller.restapi.StringResponse; import com.yahoo.vespa.hosted.controller.restapi.cost.config.SelfHostedCostConfig; import java.time.Clock; +import java.util.Optional; import static com.yahoo.jdisc.http.HttpRequest.Method.GET; @@ -36,7 +38,9 @@ public class CostApiHandler extends LoggingRequestHandler { Path path = new Path(request.getUri().getPath()); if (path.matches("/cost/v1/csv")) { - return new StringResponse(CostCalculator.resourceShareByPropertyToCsv(nodeRepository, controller, Clock.systemUTC(), selfHostedCostConfig)); + Optional<String> cloudProperty = Optional.ofNullable(request.getProperty("cloud")); + CloudName cloud = cloudProperty.map(CloudName::from).orElse(CloudName.defaultName()); + return new StringResponse(CostCalculator.resourceShareByPropertyToCsv(nodeRepository, controller, Clock.systemUTC(), selfHostedCostConfig, cloud)); } return ErrorResponse.notFoundError("Nothing at " + path); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/cost/CostCalculator.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/cost/CostCalculator.java index fc30ecc97bb..18c00d69b62 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/cost/CostCalculator.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/cost/CostCalculator.java @@ -3,7 +3,6 @@ package com.yahoo.vespa.hosted.controller.restapi.cost; import com.yahoo.config.provision.CloudName; import com.yahoo.config.provision.Environment; import com.yahoo.vespa.hosted.controller.Controller; -import com.yahoo.vespa.hosted.controller.api.identifiers.ApplicationId; import com.yahoo.vespa.hosted.controller.api.identifiers.Property; import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeOwner; import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeRepositoryClientInterface; @@ -14,9 +13,7 @@ import com.yahoo.vespa.hosted.controller.tenant.AthenzTenant; import java.time.Clock; import java.time.LocalDate; -import java.util.ArrayList; import java.util.Comparator; -import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.stream.Collectors; @@ -29,30 +26,32 @@ public class CostCalculator { private static final double SELF_HOSTED_DISCOUNT = .5; public static String resourceShareByPropertyToCsv(NodeRepositoryClientInterface nodeRepository, - Controller controller, - Clock clock, - SelfHostedCostConfig selfHostedCostConfig) { + Controller controller, + Clock clock, + SelfHostedCostConfig selfHostedCostConfig, + CloudName cloudName) { String date = LocalDate.now(clock).toString(); List<NodeRepositoryNode> nodes = controller.zoneRegistry().zones() - .reachable().in(Environment.prod).ofCloud(CloudName.from("yahoo")).ids().stream() + .reachable().in(Environment.prod).ofCloud(cloudName).ids().stream() .flatMap(zoneId -> uncheck(() -> nodeRepository.listNodes(zoneId, true).nodes().stream())) .filter(node -> node.getOwner() != null && !node.getOwner().getTenant().equals("hosted-vespa")) .collect(Collectors.toList()); - selfHostedCostConfig.properties().stream().map(property -> { - NodeRepositoryNode selfHostedNode = new NodeRepositoryNode(); + if (cloudName.equals(CloudName.from("yahoo"))) + selfHostedCostConfig.properties().stream().map(property -> { + NodeRepositoryNode selfHostedNode = new NodeRepositoryNode(); - NodeOwner owner = new NodeOwner(); - owner.tenant = property.name(); - selfHostedNode.setOwner(owner); - selfHostedNode.setMinCpuCores(property.cpuCores() * SELF_HOSTED_DISCOUNT); - selfHostedNode.setMinMainMemoryAvailableGb(property.memoryGb() * SELF_HOSTED_DISCOUNT); - selfHostedNode.setMinDiskAvailableGb(property.diskGb() * SELF_HOSTED_DISCOUNT); + NodeOwner owner = new NodeOwner(); + owner.tenant = property.name(); + selfHostedNode.setOwner(owner); + selfHostedNode.setMinCpuCores(property.cpuCores() * SELF_HOSTED_DISCOUNT); + selfHostedNode.setMinMainMemoryAvailableGb(property.memoryGb() * SELF_HOSTED_DISCOUNT); + selfHostedNode.setMinDiskAvailableGb(property.diskGb() * SELF_HOSTED_DISCOUNT); - return selfHostedNode; - }).forEach(nodes::add); + return selfHostedNode; + }).forEach(nodes::add); ResourceAllocation totalResourceAllocation = ResourceAllocation.from(nodes); @@ -73,7 +72,7 @@ public class CostCalculator { node -> propertyByTenantName.get(node.getOwner().tenant), Collectors.collectingAndThen( Collectors.toList(), - (tenantNodes) -> ResourceAllocation.from(tenantNodes) + ResourceAllocation::from ) )); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainerTest.java index 14c75e791a5..df2a4b5ca7f 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainerTest.java @@ -6,6 +6,7 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.vespa.hosted.controller.api.integration.resource.ResourceSnapshot; import com.yahoo.vespa.hosted.controller.api.integration.stubs.MockResourceSnapshotConsumer; import com.yahoo.vespa.hosted.controller.integration.NodeRepositoryClientMock; +import com.yahoo.vespa.hosted.controller.integration.MetricsMock; import org.junit.Test; import java.time.Duration; @@ -19,13 +20,14 @@ import static org.junit.Assert.*; public class ResourceMeterMaintainerTest { private final double DELTA = Double.MIN_VALUE; - NodeRepositoryClientMock nodeRepository = new NodeRepositoryClientMock(); - MockResourceSnapshotConsumer snapshotConsumer = new MockResourceSnapshotConsumer(); + private NodeRepositoryClientMock nodeRepository = new NodeRepositoryClientMock(); + private MockResourceSnapshotConsumer snapshotConsumer = new MockResourceSnapshotConsumer(); + private MetricsMock metrics = new MetricsMock(); @Test public void testMaintainer() { ControllerTester tester = new ControllerTester(); - ResourceMeterMaintainer resourceMeterMaintainer = new ResourceMeterMaintainer(tester.controller(), Duration.ofMinutes(5), new JobControl(tester.curator()), nodeRepository, tester.clock(), snapshotConsumer); + ResourceMeterMaintainer resourceMeterMaintainer = new ResourceMeterMaintainer(tester.controller(), Duration.ofMinutes(5), new JobControl(tester.curator()), nodeRepository, tester.clock(), metrics, snapshotConsumer); resourceMeterMaintainer.maintain(); Map<ApplicationId, ResourceSnapshot> consumedResources = snapshotConsumer.consumedResources(); @@ -42,5 +44,7 @@ public class ResourceMeterMaintainerTest { assertEquals(96, app2.getResourceAllocation().getMemoryGb(), DELTA); assertEquals(2000, app2.getResourceAllocation().getDiskGb(), DELTA); + assertEquals(tester.clock().millis()/1000, metrics.getMetric("metering_last_reported")); + assertEquals(4448.0d, (Double) metrics.getMetric("metering_total_reported"), DELTA); } }
\ No newline at end of file diff --git a/document/src/main/java/com/yahoo/document/json/DocumentUpdateJsonSerializer.java b/document/src/main/java/com/yahoo/document/json/DocumentUpdateJsonSerializer.java index 8d50f4a14dd..7b78f90bc56 100644 --- a/document/src/main/java/com/yahoo/document/json/DocumentUpdateJsonSerializer.java +++ b/document/src/main/java/com/yahoo/document/json/DocumentUpdateJsonSerializer.java @@ -65,8 +65,8 @@ import static com.yahoo.document.json.JsonSerializationHelper.*; * @see #serialize(com.yahoo.document.DocumentUpdate) * @author Vegard Sjonfjell */ -public class DocumentUpdateJsonSerializer -{ +public class DocumentUpdateJsonSerializer { + private final JsonFactory jsonFactory = new JsonFactory(); private final JsonDocumentUpdateWriter writer = new JsonDocumentUpdateWriter(); private JsonGenerator generator; diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index 9de9f8f2ba4..c4c3021d607 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -95,6 +95,12 @@ public class Flags { "Takes effect at redeployment", APPLICATION_ID); + public static final UnboundBooleanFlag DISPATCH_WITH_PROTOBUF = defineFeatureFlag( + "dispatch-with-protobuf", false, + "Should the java dispatcher use protobuf/jrt as the default", + "Takes effect at redeployment", + APPLICATION_ID); + public static final UnboundBooleanFlag ENABLE_DYNAMIC_PROVISIONING = defineFeatureFlag( "enable-dynamic-provisioning", false, "Provision a new docker host when we otherwise can't allocate a docker node", @@ -107,12 +113,6 @@ public class Flags { "Takes effect on next node agent tick (but does not clear existing failure reports)", HOSTNAME); - public static final UnboundBooleanFlag USE_SEPARATE_SERVICE_TYPE_FOR_LOGSERVER_CONTAINER = defineFeatureFlag( - "use-separate-service-type-for-logserver-container", true, - "Use separate service type for Logserver container, resulting in logserver container not being an application endpoint", - "Takes effect at redeployment", - APPLICATION_ID); - public static final UnboundBooleanFlag ENABLE_METRICS_PROXY_CONTAINER = defineFeatureFlag( "enable-metrics-proxy-container", false, "Start a container for metrics-proxy on every vespa node", @@ -125,6 +125,12 @@ public class Flags { "Takes effect on restart of Docker container", APPLICATION_ID, HOSTNAME); + public static final UnboundBooleanFlag USE_NEW_LOGROTATE_CONFIGURATION = defineFeatureFlag( + "use-new-logrotate-configuration", false, + "Set up logrotate configuration without using any proprietary software", + "Takes effect on restart of Docker container", + APPLICATION_ID, HOSTNAME); + public static final UnboundBooleanFlag USE_HTTPS_LOAD_BALANCER_UPSTREAM = defineFeatureFlag( "use-https-load-balancer-upstream", false, "Use https between load balancer and upstream containers", diff --git a/logd/CMakeLists.txt b/logd/CMakeLists.txt index e16d3bd5179..55afaf9d126 100644 --- a/logd/CMakeLists.txt +++ b/logd/CMakeLists.txt @@ -18,6 +18,7 @@ vespa_define_module( src/tests/proto_converter src/tests/rotate src/tests/rpc_forwarder + src/tests/watcher ) vespa_install_script(src/apps/retention/retention-enforcer.sh vespa-retention-enforcer sbin) diff --git a/logd/src/tests/watcher/CMakeLists.txt b/logd/src/tests/watcher/CMakeLists.txt new file mode 100644 index 00000000000..0bf0a574fa9 --- /dev/null +++ b/logd/src/tests/watcher/CMakeLists.txt @@ -0,0 +1,9 @@ +# Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(logd_watcher_test_app TEST + SOURCES + watcher_test.cpp + DEPENDS + logd + gtest +) +vespa_add_test(NAME logd_watcher_test_app COMMAND logd_watcher_test_app) diff --git a/logd/src/tests/watcher/watcher_test.cpp b/logd/src/tests/watcher/watcher_test.cpp new file mode 100644 index 00000000000..ae0fbd63b90 --- /dev/null +++ b/logd/src/tests/watcher/watcher_test.cpp @@ -0,0 +1,287 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/vespalib/gtest/gtest.h> +#include <logd/config_subscriber.h> +#include <logd/legacy_forwarder.h> +#include <logd/metrics.h> +#include <logd/watcher.h> +#include <vespa/vespalib/io/fileutil.h> +#include <vespa/vespalib/metrics/dummy_metrics_manager.h> +#include <vespa/vespalib/util/threadstackexecutor.h> +#include <vespa/vespalib/util/lambdatask.h> +#include <fstream> +#include <regex> +#include <thread> +#include <mutex> +#include <condition_variable> + +using cloud::config::log::LogdConfigBuilder; +using cloud::config::log::LogdConfig; +using config::ConfigSet; +using config::ConfigUri; +using config::IConfigContext; +using config::ConfigContext; +using vespalib::metrics::DummyMetricsManager; +using vespalib::ThreadStackExecutor; +using vespalib::makeLambdaTask; +using namespace std::chrono_literals; + +std::basic_regex<char> rotated_log(R"(vespa.log-[0-9]*-[0-9]*-[0-9]*\.[0-9]*-[0-9]*-[0-9]*)"); + +namespace logdemon { + +namespace { + +struct ConfigFixture { + const std::string configId; + LogdConfigBuilder logdBuilder; + ConfigSet set; + IConfigContext::SP context; + int idcounter; + + ConfigFixture(const std::string & id); + ~ConfigFixture(); + void reload() { context->reload(); } +}; + +ConfigFixture::ConfigFixture(const std::string & id) + : configId(id), + logdBuilder(), + set(), + context(std::make_shared<ConfigContext>(set)), + idcounter(-1) +{ + logdBuilder.logserver.use = false; + logdBuilder.rotate.size = 1024; + set.addBuilder(configId, &logdBuilder); +} + +ConfigFixture::~ConfigFixture() = default; + +struct DummyForwarder : public Forwarder { + std::mutex lock; + std::condition_variable cond; + std::atomic<int> sendModeCount; + std::vector<std::string> lines; + DummyForwarder() + : Forwarder(), + lock(), + cond(), + sendModeCount(0), + lines() + { + } + ~DummyForwarder() override = default; + void sendMode() override { ++sendModeCount; } + void forwardLine(std::string_view log_line) override { + std::lock_guard guard(lock); + assert(log_line.size() > 0u); + lines.emplace_back(log_line.substr(0, log_line.size() - 1)); + cond.notify_all(); + } + void flush() override { } + int badLines() const override { return 0; } + void resetBadLines() override { } + std::vector<std::string> getLines() { + std::lock_guard guard(lock); + return lines; + } + void waitLineCount(size_t lineCount) { + std::unique_lock guard(lock); + cond.wait_for(guard, 10s, [this, lineCount]() { return lineCount <= lines.size(); }); + } +}; + +struct WatcherFixture +{ + Metrics metrics; + LegacyForwarder legacy_fwd; + DummyForwarder fwd; + ConfigSubscriber subscriber; + Watcher watcher; + + WatcherFixture(ConfigFixture &cfg); + ~WatcherFixture(); +}; + +WatcherFixture::WatcherFixture(ConfigFixture &cfg) + : metrics(DummyMetricsManager::create()), + legacy_fwd(metrics), + fwd(), + subscriber(legacy_fwd, config::ConfigUri(cfg.configId, cfg.context)), + watcher(subscriber, fwd) +{ + subscriber.latch(); +} + +WatcherFixture::~WatcherFixture() = default; + +} + +class WatcherTest : public ::testing::Test { +protected: + std::unique_ptr<ConfigFixture> _cfg; + std::unique_ptr<WatcherFixture> _watcher; + ThreadStackExecutor _executor; + + void setup_watcher(); + void run_watcher(); + void stop_watcher(); + void log_line(const std::string &line); + void assert_lines(const std::vector<std::string> &lines); + void remove_files(); + void remove_rotated(); + int count_rotated(); +public: + WatcherTest(); + ~WatcherTest(); +}; + +WatcherTest::WatcherTest() + : _executor(1, 256 * 1024) +{ + remove_files(); + setenv("VESPA_LOG_TARGET", "file:vespa.log", true); + vespalib::mkdir("var/db/vespa", true); // for logd.donestate + _cfg = std::make_unique<ConfigFixture>("testconfigid"); +} + +WatcherTest::~WatcherTest() +{ + remove_files(); +} + +void +WatcherTest::setup_watcher() +{ + _watcher = std::make_unique<WatcherFixture>(*_cfg); +} + +void +WatcherTest::run_watcher() +{ + // Spin off watcher task + _executor.execute(makeLambdaTask([this]() { _watcher->watcher.watchfile(); })); +} + +void +WatcherTest::stop_watcher() +{ + _cfg->reload(); + _executor.sync(); +} + +void +WatcherTest::log_line(const std::string &line) +{ + std::ofstream log_file("vespa.log", std::ios::out | std::ios::app); + log_file << line << std::endl; +} + +void +WatcherTest::assert_lines(const std::vector<std::string> &lines) +{ + EXPECT_EQ(lines, _watcher->fwd.getLines()); +} + +void +WatcherTest::remove_files() +{ + vespalib::rmdir("var", true); + remove_rotated(); + vespalib::unlink("vespa.log"); +} + +void +WatcherTest::remove_rotated() +{ + auto dirlist = vespalib::listDirectory("."); + for (const auto &entry : dirlist) { + if (std::regex_match(entry.data(), entry.data() + entry.size(), rotated_log)) { + vespalib::unlink(entry); + } + } +} + +int +WatcherTest::count_rotated() +{ + int result = 0; + auto dirlist = vespalib::listDirectory("."); + for (const auto &entry : dirlist) { + if (std::regex_match(entry.data(), entry.data() + entry.size(), rotated_log)) { + ++result; + } + } + return result; +} + +TEST_F(WatcherTest, require_that_watching_no_logging_works) +{ + setup_watcher(); + run_watcher(); + stop_watcher(); + assert_lines({}); + EXPECT_EQ(0, count_rotated()); +} + +TEST_F(WatcherTest, require_that_watching_simple_logging_works) +{ + setup_watcher(); + run_watcher(); + log_line("foo"); + _watcher->fwd.waitLineCount(1); + stop_watcher(); + EXPECT_EQ(0, count_rotated()); + assert_lines({"foo"}); +} + +TEST_F(WatcherTest, require_that_watching_can_resume) +{ + setup_watcher(); + run_watcher(); + log_line("foo"); + _watcher->fwd.waitLineCount(1); + stop_watcher(); + assert_lines({"foo"}); + setup_watcher(); + run_watcher(); + log_line("bar"); + log_line("baz"); + _watcher->fwd.waitLineCount(2); + stop_watcher(); + assert_lines({"bar", "baz"}); + // remove state file. Old entry will resurface + vespalib::unlink("var/db/vespa/logd.donestate"); + setup_watcher(); + run_watcher(); + _watcher->fwd.waitLineCount(3); + stop_watcher(); + assert_lines({"foo", "bar", "baz"}); +} + +TEST_F(WatcherTest, require_that_watching_can_rotate_log_files) +{ + setup_watcher(); + run_watcher(); + std::vector<std::string> exp_lines; + for (int i = 0; i < 100; ++i) { + std::ostringstream os; + os << "this is a malformatted " << std::setw(3) << i << + std::setw(0) << " line but who cares ?"; + log_line(os.str()); + exp_lines.push_back(os.str()); + std::this_thread::sleep_for(100ms); + if (count_rotated() > 0) { + break; + } + } + _watcher->fwd.waitLineCount(exp_lines.size()); + stop_watcher(); + assert_lines(exp_lines); + EXPECT_LT(0, count_rotated()); +} + +} + +GTEST_MAIN_RUN_ALL_TESTS() diff --git a/logserver/src/test/java/com/yahoo/logserver/handlers/archive/ArchiverHandlerTestCase.java b/logserver/src/test/java/com/yahoo/logserver/handlers/archive/ArchiverHandlerTestCase.java index 525d02c4298..578caab366c 100644 --- a/logserver/src/test/java/com/yahoo/logserver/handlers/archive/ArchiverHandlerTestCase.java +++ b/logserver/src/test/java/com/yahoo/logserver/handlers/archive/ArchiverHandlerTestCase.java @@ -28,10 +28,10 @@ import static org.junit.Assert.fail; public class ArchiverHandlerTestCase { private static final String[] mStrings = { - "1095159244.095\thost\t1/2\tservice\tcomponent\tinfo\tpayload1", - "1095206399.000\thost\t1/2\tservice\tcomponent\tinfo\tpayload2", - "1095206400.000\thost\t1/2\tservice\tcomponent\tinfo\tpayload3", - "1095206401.000\thost\t1/2\tservice\tcomponent\tinfo\tpayload4", + "1095159244.095000\thost\t1/2\tservice\tcomponent\tinfo\tpayload1", + "1095206399.000000\thost\t1/2\tservice\tcomponent\tinfo\tpayload2", + "1095206400.000000\thost\t1/2\tservice\tcomponent\tinfo\tpayload3", + "1095206401.000000\thost\t1/2\tservice\tcomponent\tinfo\tpayload4", }; private static final LogMessage[] msg = new LogMessage[mStrings.length]; diff --git a/searchcore/src/vespa/searchcore/proton/matching/document_scorer.cpp b/searchcore/src/vespa/searchcore/proton/matching/document_scorer.cpp index 81ac1f923b9..c7721b428b9 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/document_scorer.cpp +++ b/searchcore/src/vespa/searchcore/proton/matching/document_scorer.cpp @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "document_scorer.h" +#include <cassert> using search::feature_t; using search::fef::FeatureResolver; @@ -8,8 +9,7 @@ using search::fef::RankProgram; using search::fef::LazyValue; using search::queryeval::SearchIterator; -namespace proton { -namespace matching { +namespace proton::matching { namespace { @@ -36,5 +36,4 @@ DocumentScorer::score(uint32_t docId) return doScore(docId); } -} // namespace proton::matching -} // namespace proton +} diff --git a/searchlib/src/vespa/searchlib/attribute/attribute_blueprint_factory.cpp b/searchlib/src/vespa/searchlib/attribute/attribute_blueprint_factory.cpp index f09f48363db..192d498125c 100644 --- a/searchlib/src/vespa/searchlib/attribute/attribute_blueprint_factory.cpp +++ b/searchlib/src/vespa/searchlib/attribute/attribute_blueprint_factory.cpp @@ -83,20 +83,28 @@ private: ISearchContext::UP _search_context; AttributeFieldBlueprint(const FieldSpec &field, const IAttributeVector &attribute, - const string &query_stack, const attribute::SearchContextParams ¶ms) + QueryTermSimple::UP term, const attribute::SearchContextParams ¶ms) : SimpleLeafBlueprint(field), - _search_context(attribute.createSearchContext(QueryTermDecoder::decodeTerm(query_stack), params)) + _search_context(attribute.createSearchContext(std::move(term), params)) { uint32_t estHits = _search_context->approximateHits(); HitEstimate estimate(estHits, estHits == 0); setEstimate(estimate); } + AttributeFieldBlueprint(const FieldSpec &field, const IAttributeVector &attribute, + const string &query_stack, const attribute::SearchContextParams ¶ms) + : AttributeFieldBlueprint(field, attribute, QueryTermDecoder::decodeTerm(query_stack), params) + { } + public: - AttributeFieldBlueprint(const FieldSpec &field, const IAttributeVector &attribute, const string &query_stack) - : AttributeFieldBlueprint(field, attribute, query_stack, + AttributeFieldBlueprint(const FieldSpec &field, const IAttributeVector &attribute, QueryTermSimple::UP term) + : AttributeFieldBlueprint(field, attribute, std::move(term), attribute::SearchContextParams().useBitVector(field.isFilter())) {} + AttributeFieldBlueprint(const FieldSpec &field, const IAttributeVector &attribute, const string &query_stack) + : AttributeFieldBlueprint(field, attribute, QueryTermDecoder::decodeTerm(query_stack)) + {} AttributeFieldBlueprint(const FieldSpec &field, const IAttributeVector &attribute, const IAttributeVector &diversity, const string &query_stack, @@ -115,6 +123,13 @@ public: return _search_context->createIterator(tfmda[0], strict); } + SearchIterator::UP + createSearch(fef::MatchData &md, bool strict) const override { + const State &state = getState(); + assert(state.numFields() == 1); + return _search_context->createIterator(state.field(0).resolve(md), strict); + } + void fetchPostings(bool strict) override { _search_context->fetchPostings(strict); @@ -505,15 +520,23 @@ public: setResult(std::move(result)); } + static QueryTermSimple::UP + extractTerm(const query::Node &node, bool isInteger) { + vespalib::string term = queryeval::termAsString(node); + if (isInteger) { + return std::make_unique<QueryTermSimple>(term, QueryTermSimple::WORD); + } + return std::make_unique<QueryTermBase>(term, QueryTermSimple::WORD); + } + template <typename WS, typename NODE> - void createShallowWeightedSet(WS *bp, NODE &n, const FieldSpec &fs) { + void createShallowWeightedSet(WS *bp, NODE &n, const FieldSpec &fs, bool isInteger) { Blueprint::UP result(bp); for (size_t i = 0; i < n.getChildren().size(); ++i) { const query::Node &node = *n.getChildren()[i]; uint32_t weight = queryeval::getWeightFromNode(node).percent(); - const string stack = StackDumpCreator::create(node); FieldSpec childfs = bp->getNextChildField(fs); - bp->addTerm(std::make_unique<AttributeFieldBlueprint>(childfs, _attr, stack), weight); + bp->addTerm(std::make_unique<AttributeFieldBlueprint>(childfs, _attr, extractTerm(node, isInteger)), weight); } setResult(std::move(result)); } @@ -527,14 +550,7 @@ public: for (size_t i = 0; i < n.getChildren().size(); ++i) { const query::Node &node = *n.getChildren()[i]; uint32_t weight = queryeval::getWeightFromNode(node).percent(); - vespalib::string term = queryeval::termAsString(node); - QueryTermSimple::UP qt; - if (isInteger) { - qt = std::make_unique<QueryTermSimple>(term, QueryTermSimple::WORD); - } else { - qt = std::make_unique<QueryTermBase>(term, QueryTermSimple::WORD); - } - ws->addToken(_attr.createSearchContext(std::move(qt), attribute::SearchContextParams()), weight); + ws->addToken(_attr.createSearchContext(extractTerm(node, isInteger), attribute::SearchContextParams()), weight); } setResult(std::move(ws)); } else { @@ -543,7 +559,7 @@ public: createDirectWeightedSet(bp, n); } else { auto *bp = new WeightedSetTermBlueprint(_field); - createShallowWeightedSet(bp, n, _field); + createShallowWeightedSet(bp, n, _field, _attr.isIntegerType()); } } } @@ -554,7 +570,7 @@ public: createDirectWeightedSet(bp, n); } else { auto *bp = new DotProductBlueprint(_field); - createShallowWeightedSet(bp, n, _field); + createShallowWeightedSet(bp, n, _field, _attr.isIntegerType()); } } @@ -569,7 +585,7 @@ public: n.getTargetNumHits(), n.getScoreThreshold(), n.getThresholdBoostFactor()); - createShallowWeightedSet(bp, n, _field); + createShallowWeightedSet(bp, n, _field, _attr.isIntegerType()); } } }; diff --git a/searchlib/src/vespa/searchlib/attribute/attributeiterators.cpp b/searchlib/src/vespa/searchlib/attribute/attributeiterators.cpp index 3f86a7f57db..ea3b4c2f32d 100644 --- a/searchlib/src/vespa/searchlib/attribute/attributeiterators.cpp +++ b/searchlib/src/vespa/searchlib/attribute/attributeiterators.cpp @@ -8,13 +8,6 @@ namespace search { using queryeval::MinMaxPostingInfo; using fef::TermFieldMatchData; -AttributeIteratorBase::AttributeIteratorBase(const attribute::ISearchContext &baseSearchCtx, - TermFieldMatchData *matchData) - : _baseSearchCtx(baseSearchCtx), - _matchData(matchData), - _matchPosition(_matchData->populate_fixed()) -{ } - void AttributeIteratorBase::visitMembers(vespalib::ObjectVisitor &visitor) const { @@ -23,13 +16,6 @@ AttributeIteratorBase::visitMembers(vespalib::ObjectVisitor &visitor) const visit(visitor, "tfmd.docId", _matchData->getDocId()); } -FilterAttributeIterator::FilterAttributeIterator(const attribute::ISearchContext &baseSearchCtx, - fef::TermFieldMatchData *matchData) - : AttributeIteratorBase(baseSearchCtx, matchData) -{ - _matchPosition->setElementWeight(1); -} - void AttributeIterator::visitMembers(vespalib::ObjectVisitor &visitor) const { @@ -37,27 +23,12 @@ AttributeIterator::visitMembers(vespalib::ObjectVisitor &visitor) const visit(visitor, "weight", _weight); } - void FlagAttributeIterator::doUnpack(uint32_t docId) { _matchData->resetOnlyDocId(docId); } -AttributePostingListIterator::AttributePostingListIterator(const attribute::ISearchContext &baseSearchCtx, - bool hasWeight, - TermFieldMatchData *matchData) - : AttributeIteratorBase(baseSearchCtx, matchData), - _hasWeight(hasWeight) -{ -} - -FilterAttributePostingListIterator:: -FilterAttributePostingListIterator(const attribute::ISearchContext &baseSearchCtx, TermFieldMatchData *matchData) - : AttributeIteratorBase(baseSearchCtx, matchData) -{ -} - void AttributeIterator::doUnpack(uint32_t docId) { @@ -65,7 +36,6 @@ AttributeIterator::doUnpack(uint32_t docId) _matchPosition->setElementWeight(_weight); } - void FilterAttributeIterator::doUnpack(uint32_t docId) { @@ -81,7 +51,6 @@ doUnpack(uint32_t docId) _matchPosition->setElementWeight(getWeight()); } - template <> void AttributePostingListIteratorT<WeightedInnerAttributePostingListIterator>:: @@ -91,7 +60,6 @@ doUnpack(uint32_t docId) _matchPosition->setElementWeight(getWeight()); } - template <> void FilterAttributePostingListIteratorT<InnerAttributePostingListIterator>:: @@ -100,7 +68,6 @@ doUnpack(uint32_t docId) _matchData->resetOnlyDocId(docId); } - template <> void FilterAttributePostingListIteratorT<WeightedInnerAttributePostingListIterator>:: @@ -109,7 +76,6 @@ doUnpack(uint32_t docId) _matchData->resetOnlyDocId(docId); } - template <> void AttributePostingListIteratorT<InnerAttributePostingListIterator>:: @@ -121,7 +87,6 @@ setupPostingInfo() } } - template <> void AttributePostingListIteratorT<WeightedInnerAttributePostingListIterator>:: @@ -134,7 +99,6 @@ setupPostingInfo() } } - template <> void AttributePostingListIteratorT<DocIdMinMaxIterator<AttributePosting> >:: @@ -146,7 +110,6 @@ setupPostingInfo() } } - template <> void AttributePostingListIteratorT<DocIdMinMaxIterator<AttributeWeightPosting> >:: @@ -170,7 +133,6 @@ setupPostingInfo() } } - template <> void FilterAttributePostingListIteratorT<WeightedInnerAttributePostingListIterator>:: @@ -182,7 +144,6 @@ setupPostingInfo() } } - template <> void FilterAttributePostingListIteratorT<DocIdMinMaxIterator<AttributePosting> >:: @@ -194,7 +155,6 @@ setupPostingInfo() } } - template <> void FilterAttributePostingListIteratorT<DocIdMinMaxIterator<AttributeWeightPosting> >:: @@ -206,5 +166,4 @@ setupPostingInfo() } } - } // namespace search diff --git a/searchlib/src/vespa/searchlib/attribute/attributeiterators.h b/searchlib/src/vespa/searchlib/attribute/attributeiterators.h index 0fbcab03ec0..3f85c2a20fe 100644 --- a/searchlib/src/vespa/searchlib/attribute/attributeiterators.h +++ b/searchlib/src/vespa/searchlib/attribute/attributeiterators.h @@ -5,6 +5,7 @@ #include "dociditerator.h" #include "postinglisttraits.h" #include <vespa/searchlib/queryeval/searchiterator.h> +#include <vespa/searchlib/fef/termfieldmatchdata.h> namespace search { @@ -29,12 +30,16 @@ protected: template <typename SC> std::unique_ptr<BitVector> get_hits(const SC & sc, uint32_t begin_id) const; void visitMembers(vespalib::ObjectVisitor &visitor) const override; - const attribute::ISearchContext &_baseSearchCtx; - fef::TermFieldMatchData * _matchData; + const attribute::ISearchContext & _baseSearchCtx; + fef::TermFieldMatchData * _matchData; fef::TermFieldMatchDataPosition * _matchPosition; public: - AttributeIteratorBase(const attribute::ISearchContext &baseSearchCtx, fef::TermFieldMatchData *matchData); + AttributeIteratorBase(const attribute::ISearchContext &baseSearchCtx, fef::TermFieldMatchData *matchData) + : _baseSearchCtx(baseSearchCtx), + _matchData(matchData), + _matchPosition(_matchData->populate_fixed()) + { } Trinary is_strict() const override { return Trinary::False; } const attribute::ISearchContext *getAttributeSearchContext() const override { return &_baseSearchCtx; } }; @@ -67,7 +72,11 @@ class FilterAttributeIterator : public AttributeIteratorBase { public: FilterAttributeIterator(const attribute::ISearchContext &baseSearchCtx, - fef::TermFieldMatchData *matchData); + fef::TermFieldMatchData *matchData) + : AttributeIteratorBase(baseSearchCtx, matchData) + { + _matchPosition->setElementWeight(1); + } protected: void doUnpack(uint32_t docId) override; }; @@ -167,7 +176,10 @@ class AttributePostingListIterator : public AttributeIteratorBase { public: AttributePostingListIterator(const attribute::ISearchContext &baseSearchCtx, - bool hasWeight, fef::TermFieldMatchData *matchData); + bool hasWeight, fef::TermFieldMatchData *matchData) + : AttributeIteratorBase(baseSearchCtx, matchData), + _hasWeight(hasWeight) + {} Trinary is_strict() const override { return Trinary::True; } protected: bool _hasWeight; @@ -177,7 +189,9 @@ protected: class FilterAttributePostingListIterator : public AttributeIteratorBase { public: - FilterAttributePostingListIterator(const attribute::ISearchContext &baseSearchCtx, fef::TermFieldMatchData *matchData); + FilterAttributePostingListIterator(const attribute::ISearchContext &baseSearchCtx, fef::TermFieldMatchData *matchData) + : AttributeIteratorBase(baseSearchCtx, matchData) + {} Trinary is_strict() const override { return Trinary::True; } }; diff --git a/searchlib/src/vespa/searchlib/attribute/attributeiterators.hpp b/searchlib/src/vespa/searchlib/attribute/attributeiterators.hpp index d1165ed20e1..caf6c028c4d 100644 --- a/searchlib/src/vespa/searchlib/attribute/attributeiterators.hpp +++ b/searchlib/src/vespa/searchlib/attribute/attributeiterators.hpp @@ -5,7 +5,6 @@ #include "attributeiterators.h" #include <vespa/searchlib/btree/btreenode.hpp> #include <vespa/searchlib/btree/btreeiterator.hpp> -#include <vespa/searchlib/fef/termfieldmatchdata.h> #include <vespa/searchlib/fef/termfieldmatchdataposition.h> #include <vespa/searchlib/query/queryterm.h> #include <vespa/searchlib/common/bitvector.h> diff --git a/searchlib/src/vespa/searchlib/features/dotproductfeature.cpp b/searchlib/src/vespa/searchlib/features/dotproductfeature.cpp index 153d44ea720..38842b1da53 100644 --- a/searchlib/src/vespa/searchlib/features/dotproductfeature.cpp +++ b/searchlib/src/vespa/searchlib/features/dotproductfeature.cpp @@ -27,7 +27,7 @@ template <typename DimensionVType, typename DimensionHType, typename ComponentTy VectorBase<DimensionVType, DimensionHType, ComponentType, HashMapComparator>::VectorBase() { } template <typename DimensionVType, typename DimensionHType, typename ComponentType, typename HashMapComparator> -VectorBase<DimensionVType, DimensionHType, ComponentType, HashMapComparator>::~VectorBase() { } +VectorBase<DimensionVType, DimensionHType, ComponentType, HashMapComparator>::~VectorBase() = default; template <typename V> V copyAndSync(const V & v) { @@ -101,7 +101,7 @@ DotProductExecutor<A>::DotProductExecutor(const A * attribute, const V & queryVe } template <typename A> -DotProductExecutor<A>::~DotProductExecutor() { } +DotProductExecutor<A>::~DotProductExecutor() = default; template <typename A> size_t @@ -119,7 +119,7 @@ SparseDotProductExecutor<A>::SparseDotProductExecutor(const A * attribute, const } template <typename A> -SparseDotProductExecutor<A>::~SparseDotProductExecutor() { } +SparseDotProductExecutor<A>::~SparseDotProductExecutor() = default; template <typename A> size_t @@ -143,7 +143,7 @@ DotProductByCopyExecutor<A>::DotProductByCopyExecutor(const A * attribute, const } template <typename A> -DotProductByCopyExecutor<A>::~DotProductByCopyExecutor() { } +DotProductByCopyExecutor<A>::~DotProductByCopyExecutor() = default; template <typename A> size_t @@ -166,7 +166,7 @@ SparseDotProductByCopyExecutor<A>::SparseDotProductByCopyExecutor(const A * attr } template <typename A> -SparseDotProductByCopyExecutor<A>::~SparseDotProductByCopyExecutor() { } +SparseDotProductByCopyExecutor<A>::~SparseDotProductByCopyExecutor() = default; template <typename A> size_t @@ -197,8 +197,7 @@ DotProductByContentFillExecutor<BaseType>::DotProductByContentFillExecutor( } template <typename BaseType> -DotProductByContentFillExecutor<BaseType>::~DotProductByContentFillExecutor() { -} +DotProductByContentFillExecutor<BaseType>::~DotProductByContentFillExecutor() = default; namespace { @@ -239,8 +238,7 @@ SparseDotProductByContentFillExecutor<BaseType>::SparseDotProductByContentFillEx } template <typename BaseType> -SparseDotProductByContentFillExecutor<BaseType>::~SparseDotProductByContentFillExecutor() { -} +SparseDotProductByContentFillExecutor<BaseType>::~SparseDotProductByContentFillExecutor() = default; template <typename BaseType> size_t SparseDotProductByContentFillExecutor<BaseType>::getAttributeValues(uint32_t docid, const AT * & values) { @@ -266,10 +264,11 @@ size_t SparseDotProductByContentFillExecutor<BaseType>::getAttributeValues(uint3 DotProductBlueprint::DotProductBlueprint() : Blueprint("dotProduct"), _defaultAttribute(), - _queryVector() + _queryVector(), + _attribute(nullptr) { } -DotProductBlueprint::~DotProductBlueprint() {} +DotProductBlueprint::~DotProductBlueprint() = default; vespalib::string DotProductBlueprint::getAttribute(const IQueryEnvironment & env) const @@ -305,7 +304,7 @@ DotProductBlueprint::getDescriptions() const { Blueprint::UP DotProductBlueprint::createInstance() const { - return Blueprint::UP(new DotProductBlueprint()); + return std::make_unique<DotProductBlueprint>(); } namespace { @@ -616,7 +615,8 @@ fef::Anything::UP attemptParseArrayQueryVector(const IAttributeVector & attribut void DotProductBlueprint::prepareSharedState(const IQueryEnvironment & env, IObjectStore & store) const { - const IAttributeVector * attribute = env.getAttributeContext().getAttribute(getAttribute(env)); + _attribute = env.getAttributeContext().getAttribute(getAttribute(env)); + const IAttributeVector * attribute = _attribute; if (attribute != nullptr) { if ((attribute->getCollectionType() == attribute::CollectionType::WSET) && attribute->hasEnum() && @@ -654,7 +654,7 @@ DotProductBlueprint::prepareSharedState(const IQueryEnvironment & env, IObjectSt FeatureExecutor & DotProductBlueprint::createExecutor(const IQueryEnvironment & env, vespalib::Stash &stash) const { - const IAttributeVector * attribute = env.getAttributeContext().getAttribute(getAttribute(env)); + const IAttributeVector * attribute = (_attribute != nullptr) ? _attribute : env.getAttributeContext().getAttribute(getAttribute(env)); if (attribute == nullptr) { LOG(warning, "The attribute vector '%s' was not found in the attribute manager, returning executor with default value.", getAttribute(env).c_str()); diff --git a/searchlib/src/vespa/searchlib/features/dotproductfeature.h b/searchlib/src/vespa/searchlib/features/dotproductfeature.h index 9356fe9c7f2..b6107a1a271 100644 --- a/searchlib/src/vespa/searchlib/features/dotproductfeature.h +++ b/searchlib/src/vespa/searchlib/features/dotproductfeature.h @@ -91,9 +91,9 @@ public: **/ class EnumVector : public VectorBase<search::attribute::EnumHandle, search::attribute::EnumHandle, feature_t> { private: - const search::attribute::IAttributeVector * _attribute; + const attribute::IAttributeVector * _attribute; public: - EnumVector(const search::attribute::IAttributeVector * attribute) : _attribute(attribute) {} + EnumVector(const attribute::IAttributeVector * attribute) : _attribute(attribute) {} void insert(vespalib::stringref label, vespalib::stringref value) { search::attribute::EnumHandle e; if (_attribute->findEnum(label.data(), e)) { @@ -115,7 +115,7 @@ private: Buffer _buffer; public: - DotProductExecutor(const search::attribute::IAttributeVector * attribute, const Vector & queryVector); + DotProductExecutor(const attribute::IAttributeVector * attribute, const Vector & queryVector); void execute(uint32_t docId) override; }; @@ -263,6 +263,8 @@ private: vespalib::string _defaultAttribute; vespalib::string _queryVector; + mutable const attribute::IAttributeVector * _attribute; + vespalib::string getAttribute(const fef::IQueryEnvironment & env) const; public: diff --git a/searchlib/src/vespa/searchlib/features/element_completeness_feature.cpp b/searchlib/src/vespa/searchlib/features/element_completeness_feature.cpp index 1d71fa9d47c..1622a87e733 100644 --- a/searchlib/src/vespa/searchlib/features/element_completeness_feature.cpp +++ b/searchlib/src/vespa/searchlib/features/element_completeness_feature.cpp @@ -5,6 +5,7 @@ #include <vespa/searchlib/fef/featurenamebuilder.h> #include <vespa/searchlib/fef/properties.h> #include <vespa/vespalib/locale/c.h> +#include <cassert> namespace search::features { diff --git a/searchlib/src/vespa/searchlib/features/flow_completeness_feature.cpp b/searchlib/src/vespa/searchlib/features/flow_completeness_feature.cpp index 871c53221c8..eda83b991bf 100644 --- a/searchlib/src/vespa/searchlib/features/flow_completeness_feature.cpp +++ b/searchlib/src/vespa/searchlib/features/flow_completeness_feature.cpp @@ -7,6 +7,8 @@ #include <vespa/searchlib/fef/indexproperties.h> #include <vespa/vespalib/stllike/hash_map.h> #include <vespa/vespalib/locale/c.h> +#include <cassert> + #include <vespa/log/log.h> LOG_SETUP(".features.flowcompleteness"); diff --git a/searchlib/src/vespa/searchlib/features/nativerankfeature.h b/searchlib/src/vespa/searchlib/features/nativerankfeature.h index 3b8ddccaa1d..06088a06fc8 100644 --- a/searchlib/src/vespa/searchlib/features/nativerankfeature.h +++ b/searchlib/src/vespa/searchlib/features/nativerankfeature.h @@ -5,6 +5,7 @@ #include <vespa/searchlib/fef/blueprint.h> #include <vespa/searchlib/fef/fieldtype.h> #include <vespa/searchlib/fef/table.h> +#include <cassert> namespace search::features { diff --git a/searchlib/src/vespa/searchlib/features/term_field_md_feature.cpp b/searchlib/src/vespa/searchlib/features/term_field_md_feature.cpp index 8b0eecd9cff..a4ca8524140 100644 --- a/searchlib/src/vespa/searchlib/features/term_field_md_feature.cpp +++ b/searchlib/src/vespa/searchlib/features/term_field_md_feature.cpp @@ -6,24 +6,22 @@ #include <vespa/searchlib/fef/indexproperties.h> #include <vespa/searchlib/fef/itablemanager.h> #include <vespa/searchlib/fef/properties.h> +#include <cassert> using namespace search::fef; -namespace search { -namespace features { +namespace search::features { - -TermFieldMdExecutor::TermFieldMdExecutor(const search::fef::IQueryEnvironment &env, - uint32_t fieldId) +TermFieldMdExecutor::TermFieldMdExecutor(const fef::IQueryEnvironment &env, uint32_t fieldId) : _terms(), _md(nullptr) { for (uint32_t i = 0; i < env.getNumTerms(); ++i) { - const search::fef::ITermData *td = env.getTerm(i); + const fef::ITermData *td = env.getTerm(i); assert(td != 0); - const search::fef::ITermFieldData *tfd = td->lookupField(fieldId); + const fef::ITermFieldData *tfd = td->lookupField(fieldId); if (tfd != 0) { - assert(tfd->getHandle() != search::fef::IllegalHandle); + assert(tfd->getHandle() != fef::IllegalHandle); _terms.push_back(std::make_pair(tfd->getHandle(), td->getWeight())); } } @@ -113,6 +111,4 @@ TermFieldMdBlueprint::createExecutor(const IQueryEnvironment & env, vespalib::St return stash.create<TermFieldMdExecutor>(env, _field->id()); } - -} // namespace features -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/fef/blueprint.cpp b/searchlib/src/vespa/searchlib/fef/blueprint.cpp index 6e04ede5253..b4def052379 100644 --- a/searchlib/src/vespa/searchlib/fef/blueprint.cpp +++ b/searchlib/src/vespa/searchlib/fef/blueprint.cpp @@ -2,12 +2,12 @@ #include "blueprint.h" #include "parametervalidator.h" +#include <cassert> #include <vespa/log/log.h> LOG_SETUP(".fef.blueprint"); -namespace search { -namespace fef { +namespace search::fef { const FeatureType & Blueprint::defineInput(vespalib::stringref inName, @@ -34,9 +34,7 @@ Blueprint::Blueprint(vespalib::stringref baseName) { } -Blueprint::~Blueprint() -{ -} +Blueprint::~Blueprint() = default; ParameterDescriptions Blueprint::getDescriptions() const @@ -71,6 +69,4 @@ Blueprint::setup(const IIndexEnvironment &indexEnv, return false; } - -} // namespace fef -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/fef/blueprintresolver.cpp b/searchlib/src/vespa/searchlib/fef/blueprintresolver.cpp index a8d2e4d7f6b..a9e500fc802 100644 --- a/searchlib/src/vespa/searchlib/fef/blueprintresolver.cpp +++ b/searchlib/src/vespa/searchlib/fef/blueprintresolver.cpp @@ -3,14 +3,14 @@ #include "blueprintresolver.h" #include "blueprintfactory.h" #include "featurenameparser.h" -#include <stack> #include <vespa/vespalib/util/stringfmt.h> +#include <stack> +#include <cassert> #include <vespa/log/log.h> LOG_SETUP(".fef.blueprintresolver"); -namespace search { -namespace fef { +namespace search::fef { namespace { @@ -230,6 +230,4 @@ BlueprintResolver::getSeedMap() const return _seedMap; } - -} // namespace fef -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/fef/matchdatalayout.cpp b/searchlib/src/vespa/searchlib/fef/matchdatalayout.cpp index 169dfd45863..bcecbf6d773 100644 --- a/searchlib/src/vespa/searchlib/fef/matchdatalayout.cpp +++ b/searchlib/src/vespa/searchlib/fef/matchdatalayout.cpp @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "matchdatalayout.h" +#include <cassert> namespace search::fef { @@ -10,15 +11,14 @@ MatchDataLayout::MatchDataLayout() { } -MatchDataLayout::~MatchDataLayout() { } +MatchDataLayout::~MatchDataLayout() = default; MatchData::UP MatchDataLayout::createMatchData() const { - MatchData::UP md(new MatchData(MatchData::params() - .numTermFields(_numTermFields))); assert(_numTermFields == _fieldIds.size()); + auto md = std::make_unique<MatchData>(MatchData::params().numTermFields(_numTermFields)); for (size_t i = 0; i < _numTermFields; ++i) { md->resolveTermField(i)->setFieldId(_fieldIds[i]); } diff --git a/searchlib/src/vespa/searchlib/fef/rank_program.cpp b/searchlib/src/vespa/searchlib/fef/rank_program.cpp index 33a65b48a1c..80729d90bf5 100644 --- a/searchlib/src/vespa/searchlib/fef/rank_program.cpp +++ b/searchlib/src/vespa/searchlib/fef/rank_program.cpp @@ -4,6 +4,7 @@ #include "featureoverrider.h" #include <vespa/vespalib/locale/c.h> #include <algorithm> +#include <cassert> using vespalib::Stash; diff --git a/searchlib/src/vespa/searchlib/fef/termfieldmatchdata.cpp b/searchlib/src/vespa/searchlib/fef/termfieldmatchdata.cpp index c5c7d2dc574..69baa772639 100644 --- a/searchlib/src/vespa/searchlib/fef/termfieldmatchdata.cpp +++ b/searchlib/src/vespa/searchlib/fef/termfieldmatchdata.cpp @@ -2,6 +2,7 @@ #include "termfieldmatchdata.h" #include <limits> +#include <cassert> namespace search::fef { @@ -28,7 +29,8 @@ TermFieldMatchData::TermFieldMatchData(const TermFieldMatchData & rhs) : } } -TermFieldMatchData & TermFieldMatchData::operator = (const TermFieldMatchData & rhs) +TermFieldMatchData & +TermFieldMatchData::operator = (const TermFieldMatchData & rhs) { if (this != & rhs) { TermFieldMatchData tmp(rhs); @@ -47,6 +49,27 @@ TermFieldMatchData::~TermFieldMatchData() } } +TermFieldMatchData::MutablePositionsIterator +TermFieldMatchData::populate_fixed() { + assert(!allocated()); + if (_sz == 0) { + new (_data._position) TermFieldMatchDataPosition(); + _sz = 1; + } + return getFixed(); +} + +TermFieldMatchData & +TermFieldMatchData::setFieldId(uint32_t fieldId) { + if (fieldId == IllegalFieldId) { + fieldId = FIELDID_MASK; + } else { + assert(fieldId < FIELDID_MASK); + } + _fieldId = (_fieldId & ~FIELDID_MASK) | fieldId; + return *this; +} + namespace { template <typename T> diff --git a/searchlib/src/vespa/searchlib/fef/termfieldmatchdata.h b/searchlib/src/vespa/searchlib/fef/termfieldmatchdata.h index a0a37d9d66c..01851a7c2fb 100644 --- a/searchlib/src/vespa/searchlib/fef/termfieldmatchdata.h +++ b/searchlib/src/vespa/searchlib/fef/termfieldmatchdata.h @@ -6,7 +6,6 @@ #include "fieldinfo.h" #include <vespa/searchlib/common/feature.h> #include <cstring> -#include <cassert> class MatchDataHeapTest; @@ -92,14 +91,7 @@ public: **/ void swap(TermFieldMatchData &rhs); - MutablePositionsIterator populate_fixed() { - assert(!allocated()); - if (_sz == 0) { - new (_data._position) TermFieldMatchDataPosition(); - _sz = 1; - } - return getFixed(); - } + MutablePositionsIterator populate_fixed(); /** * Set which field this object has match information for. @@ -107,15 +99,7 @@ public: * @return this object (for chaining) * @param fieldId field id **/ - TermFieldMatchData &setFieldId(uint32_t fieldId) { - if (fieldId == IllegalFieldId) { - fieldId = FIELDID_MASK; - } else { - assert(fieldId < FIELDID_MASK); - } - _fieldId = (_fieldId & ~FIELDID_MASK) | fieldId; - return *this; - } + TermFieldMatchData &setFieldId(uint32_t fieldId); /** * Obtain the field id diff --git a/searchlib/src/vespa/searchlib/fef/test/plugin/double.cpp b/searchlib/src/vespa/searchlib/fef/test/plugin/double.cpp index 62b7e4dc775..d9ec8b13e57 100644 --- a/searchlib/src/vespa/searchlib/fef/test/plugin/double.cpp +++ b/searchlib/src/vespa/searchlib/fef/test/plugin/double.cpp @@ -3,10 +3,9 @@ #include "double.h" #include <vespa/searchlib/fef/featurenamebuilder.h> #include <vespa/vespalib/stllike/asciistream.h> +#include <cassert> -namespace search { -namespace fef { -namespace test { +namespace search::fef::test { void DoubleExecutor::execute(uint32_t) @@ -57,6 +56,4 @@ DoubleBlueprint::createExecutor(const IQueryEnvironment &queryEnv, vespalib::Sta return stash.create<DoubleExecutor>(_cnt); } -} // namespace test -} // namespace fef -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/fef/utils.cpp b/searchlib/src/vespa/searchlib/fef/utils.cpp index eb628e5f8a0..456060118d7 100644 --- a/searchlib/src/vespa/searchlib/fef/utils.cpp +++ b/searchlib/src/vespa/searchlib/fef/utils.cpp @@ -2,9 +2,9 @@ #include "utils.h" #include <vector> +#include <cassert> -namespace search { -namespace fef { +namespace search::fef { feature_t Utils::getScoreFeature(const RankProgram &rankProgram, uint32_t docid) @@ -55,5 +55,4 @@ Utils::getAllFeatures(const RankProgram &rankProgram, uint32_t docid) return resolveFeatures(resolver, docid); } -} // namespace fef -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/queryeval/field_spec.cpp b/searchlib/src/vespa/searchlib/queryeval/field_spec.cpp index 6e64264facb..8c864c91ef8 100644 --- a/searchlib/src/vespa/searchlib/queryeval/field_spec.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/field_spec.cpp @@ -1,9 +1,9 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "field_spec.h" +#include <cassert> -namespace search { -namespace queryeval { +namespace search::queryeval { FieldSpecBase::FieldSpecBase(uint32_t fieldId, fef::TermFieldHandle handle, bool isFilter_) : _fieldId(fieldId | (isFilter_ ? 0x1000000u : 0)), @@ -12,5 +12,4 @@ FieldSpecBase::FieldSpecBase(uint32_t fieldId, fef::TermFieldHandle handle, bool assert(fieldId < 0x1000000); // Can be represented by 24 bits } -} // namespace queryeval -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/queryeval/searchiterator.cpp b/searchlib/src/vespa/searchlib/queryeval/searchiterator.cpp index c4a308a17f6..9450aceb2be 100644 --- a/searchlib/src/vespa/searchlib/queryeval/searchiterator.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/searchiterator.cpp @@ -11,18 +11,6 @@ namespace search::queryeval { -SearchIterator::SearchIterator() : - _docid(0), - _endid(0) -{ } - -void -SearchIterator::initRange(uint32_t beginid, uint32_t endid) -{ - _docid = beginid - 1; - _endid = endid; -} - BitVector::UP SearchIterator::get_hits(uint32_t begin_id) { diff --git a/searchlib/src/vespa/searchlib/queryeval/searchiterator.h b/searchlib/src/vespa/searchlib/queryeval/searchiterator.h index d2a5ed1e5f9..27494e08f90 100644 --- a/searchlib/src/vespa/searchlib/queryeval/searchiterator.h +++ b/searchlib/src/vespa/searchlib/queryeval/searchiterator.h @@ -4,6 +4,7 @@ #include "posting_info.h" #include "begin_and_end_id.h" +#include <vespa/searchlib/fef/termfieldmatchdataposition.h> #include <vespa/vespalib/stllike/string.h> #include <vespa/vespalib/util/trinary.h> #include <memory> @@ -118,7 +119,10 @@ public: * @param beginId This is the first valid docId and the lowest that will be given to doSeek. * @param endId This is the first docid after the valid range. */ - virtual void initRange(uint32_t begin_id, uint32_t end_id); + virtual void initRange(uint32_t begin_id, uint32_t end_id) { + _docid = begin_id - 1; + _endid = end_id; + } /** * Will initialize the full range. @@ -177,7 +181,7 @@ public: /** * The constructor sets the current document id to @ref beginId. **/ - SearchIterator(); + SearchIterator() : _docid(0), _endid(0) { } SearchIterator(const SearchIterator &) = delete; SearchIterator &operator=(const SearchIterator &) = delete; diff --git a/searchlib/src/vespa/searchlib/queryeval/weighted_set_term_blueprint.cpp b/searchlib/src/vespa/searchlib/queryeval/weighted_set_term_blueprint.cpp index fc68c48a247..cec72129475 100644 --- a/searchlib/src/vespa/searchlib/queryeval/weighted_set_term_blueprint.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/weighted_set_term_blueprint.cpp @@ -44,8 +44,7 @@ WeightedSetTermBlueprint::addTerm(Blueprint::UP term, int32_t weight) SearchIterator::UP -WeightedSetTermBlueprint::createLeafSearch(const search::fef::TermFieldMatchDataArray &tfmda, - bool) const +WeightedSetTermBlueprint::createLeafSearch(const fef::TermFieldMatchDataArray &tfmda, bool) const { assert(tfmda.size() == 1); fef::MatchData::UP md = _layout.createMatchData(); diff --git a/vespa-hadoop/pom.xml b/vespa-hadoop/pom.xml index 2f8dbb46a78..999bb7bcc01 100644 --- a/vespa-hadoop/pom.xml +++ b/vespa-hadoop/pom.xml @@ -21,6 +21,28 @@ <pig.version>0.14.0</pig.version> </properties> + <!-- This is a client jar and should be compilable with jdk8 --> + <profiles> + <profile> + <id>jdk11</id> + <activation> + <jdk>11</jdk> + </activation> + <properties> + <java.version>11</java.version> + </properties> + </profile> + <profile> + <id>jdk1.8</id> + <activation> + <jdk>1.8</jdk> + </activation> + <properties> + <java.version>8</java.version> + </properties> + </profile> + </profiles> + <dependencies> <!-- Hadoop dependencies --> <dependency> @@ -95,18 +117,6 @@ <plugins> <plugin> <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-compiler-plugin</artifactId> - <configuration> - <showDeprecation>true</showDeprecation> - <compilerArgs> - <arg>-Xlint:all</arg> - <arg>-Werror</arg> - </compilerArgs> - </configuration> - </plugin> - - <plugin> - <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-shade-plugin</artifactId> <executions> <execution> @@ -145,8 +155,16 @@ </plugin> <plugin> - <groupId>com.yahoo.vespa</groupId> - <artifactId>abi-check-plugin</artifactId> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-compiler-plugin</artifactId> + <version>3.8.0</version> + <configuration> + <jdkToolchain> + <version>${java.version}</version> + </jdkToolchain> + <source>${java.version}</source> + <target>${java.version}</target> + </configuration> </plugin> </plugins> diff --git a/vespa-http-client/pom.xml b/vespa-http-client/pom.xml index b42c4c32a08..6ee3d56673d 100644 --- a/vespa-http-client/pom.xml +++ b/vespa-http-client/pom.xml @@ -14,6 +14,29 @@ <name>${project.artifactId}</name> <description>Independent external feeding API towards Vespa.</description> + <!-- This is a client jar and should be compilable with jdk8 --> + <profiles> + <profile> + <id>jdk11</id> + <activation> + <jdk>11</jdk> + </activation> + <properties> + <java.version>11</java.version> + </properties> + </profile> + <profile> + <id>jdk1.8</id> + <activation> + <jdk>1.8</jdk> + </activation> + <properties> + <java.version>8</java.version> + </properties> + </profile> + </profiles> + + <dependencies> <!-- NOTE: Adding dependencies here may break clients because this is used outside an OSGi container with @@ -135,8 +158,16 @@ </executions> </plugin> <plugin> - <groupId>com.yahoo.vespa</groupId> - <artifactId>abi-check-plugin</artifactId> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-compiler-plugin</artifactId> + <version>3.8.0</version> + <configuration> + <jdkToolchain> + <version>${java.version}</version> + </jdkToolchain> + <source>${java.version}</source> + <target>${java.version}</target> + </configuration> </plugin> </plugins> </build> diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/ApacheGatewayConnection.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/ApacheGatewayConnection.java index 4468355698f..f9357ab16d8 100644 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/ApacheGatewayConnection.java +++ b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/ApacheGatewayConnection.java @@ -412,7 +412,7 @@ class ApacheGatewayConnection implements GatewayConnection { } clientBuilder.setUserAgent(String.format("vespa-http-client (%s)", Vtag.currentVersion)); - clientBuilder.setDefaultHeaders(List.of(new BasicHeader(Headers.CLIENT_VERSION, Vtag.currentVersion))); + clientBuilder.setDefaultHeaders(Collections.singletonList(new BasicHeader(Headers.CLIENT_VERSION, Vtag.currentVersion))); clientBuilder.setMaxConnPerRoute(1); clientBuilder.setMaxConnTotal(1); clientBuilder.disableContentCompression(); diff --git a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/runner/CommandLineArgumentsTest.java b/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/runner/CommandLineArgumentsTest.java index ec80dd53a4b..53715259a0c 100644 --- a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/runner/CommandLineArgumentsTest.java +++ b/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/runner/CommandLineArgumentsTest.java @@ -7,11 +7,8 @@ import com.yahoo.vespa.http.client.config.SessionParams; import org.junit.Test; import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; import java.util.Comparator; import java.util.HashSet; -import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Set; diff --git a/vespalog/abi-spec.json b/vespalog/abi-spec.json index 6f9f631c6a5..10a7a83c835 100644 --- a/vespalog/abi-spec.json +++ b/vespalog/abi-spec.json @@ -264,6 +264,7 @@ "public void <init>()", "public static java.lang.String escape(java.lang.String)", "public static void formatTime(long, java.lang.StringBuilder)", + "public static java.lang.String formatTime(java.time.Instant)", "public static java.lang.String format(java.lang.String, java.lang.String, java.lang.String, long, java.lang.String, java.lang.String, java.lang.String, java.lang.Throwable)", "public static void formatException(java.lang.Throwable, java.lang.StringBuilder)" ], diff --git a/vespalog/src/main/java/com/yahoo/log/VespaFormat.java b/vespalog/src/main/java/com/yahoo/log/VespaFormat.java index 3d1a2ae0dc8..d0dc5607af5 100644 --- a/vespalog/src/main/java/com/yahoo/log/VespaFormat.java +++ b/vespalog/src/main/java/com/yahoo/log/VespaFormat.java @@ -11,6 +11,7 @@ import java.util.regex.Pattern; * written by Bjørn Borud, licensed under the Apache 2.0 license. * * @author arnej27959 + * @author bjorncs */ public class VespaFormat { @@ -87,31 +88,18 @@ public class VespaFormat { /** - * It is easier to slice and dice strings in Java than formatting - * numbers... + * @deprecated Use {@link #formatTime(Instant)} ()} */ + @Deprecated(since = "7", forRemoval = true) public static void formatTime (long time, StringBuilder sbuffer) { - String timeString = Long.toString(time); - int len = timeString.length(); - - // something wrong. handle it by just returning the input - // long as a string. we prefer this to just crashing in - // the substring handling. - if (len < 3) { - sbuffer.append(timeString); - return; - } - sbuffer.append(timeString.substring(0, len - 3)); - sbuffer.append('.'); - sbuffer.append(timeString.substring(len - 3)); + sbuffer.append(formatTime(Instant.ofEpochMilli(time))); } - static String formatTime(Instant instant) { - StringBuilder builder = new StringBuilder(); - VespaFormat.formatTime(instant.toEpochMilli(), builder); - return builder.toString(); + public static String formatTime(Instant instant) { + return String.format("%d.%06d", instant.getEpochSecond(), instant.getNano() / 1000); } + @Deprecated(since = "7", forRemoval = true) // Unused - this is not the format used by the Vespa log handler public static String format(String levelName, String component, String componentPrefix, @@ -165,6 +153,7 @@ public class VespaFormat { * @param sbuf The stringbuffer into which we wish to * format the Throwable */ + @Deprecated(since = "7", forRemoval = true) // Unused - this is not the format used by the Vespa log handler public static void formatException (Throwable t, StringBuilder sbuf) { Throwable last = t; int depth = 0; diff --git a/vespalog/src/main/java/com/yahoo/log/VespaFormatter.java b/vespalog/src/main/java/com/yahoo/log/VespaFormatter.java index 9db9fd09b26..72a80227138 100644 --- a/vespalog/src/main/java/com/yahoo/log/VespaFormatter.java +++ b/vespalog/src/main/java/com/yahoo/log/VespaFormatter.java @@ -2,6 +2,8 @@ // $Id$ package com.yahoo.log; +import com.yahoo.log.event.Event; + import java.io.PrintWriter; import java.io.StringWriter; import java.util.logging.LogRecord; @@ -9,8 +11,6 @@ import java.util.logging.SimpleFormatter; import java.util.regex.Matcher; import java.util.regex.Pattern; -import com.yahoo.log.event.Event; - /** * This class implements a log formatter which takes care of * formatting messages according to the VESPA common log format. @@ -98,7 +98,7 @@ public class VespaFormatter extends SimpleFormatter { String component = r.getLoggerName(); // format the time - VespaFormat.formatTime(r.getMillis(), sbuf); + sbuf.append(VespaFormat.formatTime(r.getInstant())); sbuf.append("\t"); sbuf.append(hostname).append("\t") diff --git a/vespalog/src/test/java/com/yahoo/log/LogSetupTestCase.java b/vespalog/src/test/java/com/yahoo/log/LogSetupTestCase.java index d0e2baf47c5..2cea88b1c36 100644 --- a/vespalog/src/test/java/com/yahoo/log/LogSetupTestCase.java +++ b/vespalog/src/test/java/com/yahoo/log/LogSetupTestCase.java @@ -1,20 +1,24 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.log; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +import java.io.File; import java.io.IOException; import java.time.Instant; +import java.util.logging.Level; import java.util.logging.LogRecord; import java.util.logging.Logger; -import java.util.logging.Level; -import java.io.File; - -import org.junit.Rule; -import org.junit.Test; -import org.junit.Before; -import org.junit.rules.TemporaryFolder; -import static org.junit.Assert.*; import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; /** * Make sure we can install the logging stuff properly. @@ -52,7 +56,7 @@ public class LogSetupTestCase { hostname = Util.getHostName(); pid = Util.getPID(); - zookeeperLogRecordString = "1107011348.029\t" + zookeeperLogRecordString = "1107011348.029000\t" + hostname + "\t" + pid diff --git a/vespalog/src/test/java/com/yahoo/log/VespaFormatterTestCase.java b/vespalog/src/test/java/com/yahoo/log/VespaFormatterTestCase.java index cbfbd609f61..d3d1d52c1d5 100644 --- a/vespalog/src/test/java/com/yahoo/log/VespaFormatterTestCase.java +++ b/vespalog/src/test/java/com/yahoo/log/VespaFormatterTestCase.java @@ -40,7 +40,7 @@ public class VespaFormatterTestCase { testRecord1.setInstant(Instant.ofEpochMilli(1098709021843L)); testRecord1.setThreadID(123); - expected1 = "1098709021.843\t" + expected1 = "1098709021.843000\t" + hostname + "\t" + pid + "/123" + "\t" @@ -49,7 +49,7 @@ public class VespaFormatterTestCase { + "info\t" + "this is a test\n"; - expected2 = "1098709021.843\t" + expected2 = "1098709021.843000\t" + hostname + "\t" + pid + "/123" + "\t" @@ -64,7 +64,7 @@ public class VespaFormatterTestCase { testRecord2.setThreadID(123); testRecord2.setLoggerName("org.foo"); - expected3 = "1098709021.843\t" + expected3 = "1098709021.843000\t" + hostname + "\t" + pid + "/123" + "\t" @@ -73,7 +73,7 @@ public class VespaFormatterTestCase { + "info\t" + "this is a test\n"; - expected4 = "1098709021.843\t" + expected4 = "1098709021.843000\t" + hostname + "\t" + pid + "/123" + "\t" @@ -112,7 +112,7 @@ public class VespaFormatterTestCase { Object[] params = { "a small", "message" }; testRecord.setParameters(params); - String expected = "1098709021.843\t" + String expected = "1098709021.843000\t" + hostname + "\t" + pid + "/123" + "\t" @@ -232,6 +232,7 @@ public class VespaFormatterTestCase { * of runtime errors. -bb */ @Test + @SuppressWarnings("removal") // for VespaFormat.formatException public void testExceptionFormatting () { StringBuilder sb = new StringBuilder(128); Exception e = new Exception("testing", new Exception("nested")); @@ -240,9 +241,10 @@ public class VespaFormatterTestCase { @Test - public void testGeneralFormat() { + @SuppressWarnings("removal") + public void testGeneralFormat() { // for VespaFormat.format String[] expected = new String[] { - "54.321", + "54.321000", "hostname", "26019/UnitTest-Thread-37", "UnitTestRunner", diff --git a/vespalog/src/test/java/com/yahoo/log/VespaLogHandlerTestCase.java b/vespalog/src/test/java/com/yahoo/log/VespaLogHandlerTestCase.java index d18bce2f4ec..c0dd856b634 100644 --- a/vespalog/src/test/java/com/yahoo/log/VespaLogHandlerTestCase.java +++ b/vespalog/src/test/java/com/yahoo/log/VespaLogHandlerTestCase.java @@ -5,19 +5,28 @@ import org.junit.After; import org.junit.Before; import org.junit.Test; -import java.io.*; -import java.time.Instant; +import java.io.BufferedReader; +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.InputStreamReader; +import java.io.OutputStream; import java.util.LinkedList; import java.util.List; import java.util.concurrent.BrokenBarrierException; import java.util.concurrent.CyclicBarrier; import java.util.logging.Level; import java.util.logging.LogRecord; -import java.util.logging.Logger; -import static java.time.Instant.ofEpochMilli; +import static java.time.Instant.ofEpochSecond; import static org.hamcrest.core.Is.is; -import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; /** * @author Bjorn Borud @@ -43,8 +52,8 @@ public class VespaLogHandlerTestCase { pid = Util.getPID(); record1 = new LogRecord(Level.INFO, "This is a test"); - record1.setInstant(ofEpochMilli(1100011348029L)); - record1String = "1100011348.029\t" + record1.setInstant(ofEpochSecond(1100011348L, 29_123_543)); + record1String = "1100011348.029123\t" + hostname + "\t" + pid @@ -53,9 +62,9 @@ public class VespaLogHandlerTestCase { + "\tmy-test-config-id\tTST\tinfo\tThis is a test"; record2 = new LogRecord(Level.FINE, "This is a test too"); - record2.setInstant(ofEpochMilli(1100021348029L)); + record2.setInstant(ofEpochSecond(1100021348L, 29_987_654)); record2.setLoggerName("com.yahoo.log.test"); - record2String = "1100021348.029\t" + record2String = "1100021348.029987\t" + hostname + "\t" + pid @@ -63,8 +72,8 @@ public class VespaLogHandlerTestCase { record3 = new LogRecord(Level.WARNING, "another test"); record3.setLoggerName("com.yahoo.log.test"); - record3.setInstant(ofEpochMilli(1107011348029L)); - record3String = "1107011348.029\t" + record3.setInstant(ofEpochSecond(1107011348L, 29_000_000L)); + record3String = "1107011348.029000\t" + hostname + "\t" + pid @@ -73,8 +82,8 @@ public class VespaLogHandlerTestCase { record4 = new LogRecord(Level.WARNING, "unicode \u00E6\u00F8\u00E5 test \u7881 unicode"); record4.setLoggerName("com.yahoo.log.test"); - record4.setInstant(ofEpochMilli(1107011348029L)); - record4String = "1107011348.029\t" + record4.setInstant(ofEpochSecond(1107011348, 29_000_001L)); + record4String = "1107011348.029000\t" + hostname + "\t" + pid diff --git a/vespalog/src/vespa-logfmt/vespa-logfmt.pl b/vespalog/src/vespa-logfmt/vespa-logfmt.pl index eed53af8923..f15a44d4419 100755 --- a/vespalog/src/vespa-logfmt/vespa-logfmt.pl +++ b/vespalog/src/vespa-logfmt/vespa-logfmt.pl @@ -104,7 +104,6 @@ if ( $opthelp || $bad ) { " -c REGEX\t--component=REGEX\tselect components matching REGEX\n", " -m REGEX\t--message=REGEX\t\tselect message text matching REGEX\n", " -f\t\t--follow\t\tinvoke tail -F to follow input file\n", - " -L\t\t--livestream\t\tfollow log stream from logserver\n", " -N\t\t--nldequote\t\tdequote newlines in message text field\n", " -t\t--tc\t--truncatecomponent\tchop component to 15 chars\n", " --ts\t\t--truncateservice\tchop service to 9 chars\n", |