diff options
64 files changed, 521 insertions, 289 deletions
diff --git a/bootstrap.sh b/bootstrap.sh index e31e0cabda9..126359b3951 100755 --- a/bootstrap.sh +++ b/bootstrap.sh @@ -45,9 +45,11 @@ $top/dist/getversion.pl -M $top > $top/dist/vtag.map # The 'full' mode also builds modules needed by C++ tests. # must install parent pom first: +echo "Downloading all dependencies. This make take a few of minutes with an empty Maven cache." mvn_install -N # and build plugins first: +echo "Building Vespa Maven plugins." mvn_install -f maven-plugins/pom.xml # now everything else should just work with normal maven dependency resolution: @@ -56,9 +58,11 @@ case "$MODE" in java) ;; full) + echo "Building full set of dependencies." mvn_install -am -pl filedistributionmanager,jrt,linguistics,messagebus ;; default) + echo "Building default set of dependencies." mvn_install -am -pl filedistributionmanager ;; esac diff --git a/config-model/src/main/java/com/yahoo/vespa/model/InstanceResolver.java b/config-model/src/main/java/com/yahoo/vespa/model/InstanceResolver.java index 39d07257e1a..108b016e067 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/InstanceResolver.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/InstanceResolver.java @@ -5,7 +5,6 @@ import com.yahoo.config.ConfigBuilder; import com.yahoo.config.ConfigInstance; import com.yahoo.config.ConfigurationRuntimeException; import com.yahoo.config.codegen.CNode; -import com.yahoo.config.codegen.ConfigGenerator; import com.yahoo.config.codegen.InnerCNode; import com.yahoo.config.codegen.LeafCNode; import com.yahoo.vespa.config.ConfigDefinitionKey; @@ -18,6 +17,8 @@ import java.lang.reflect.Method; import java.util.List; import java.util.Map; +import static com.yahoo.config.codegen.ConfiggenUtil.createClassName; + /** * <p> * This class is capable of resolving config from a config model for a given request. It will handle @@ -152,7 +153,7 @@ class InstanceResolver { */ @SuppressWarnings("unchecked") private static Class<? extends ConfigInstance> getConfigClass(ConfigDefinitionKey cKey, ClassLoader instanceLoader) { - String className = ConfigGenerator.createClassName(cKey.getName()); + String className = createClassName(cKey.getName()); String fullClassName = packageName(cKey) + "." + className; Class<?> clazz; try { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java b/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java index 6b5ee4d2f6f..eda8b564ffd 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java @@ -9,7 +9,6 @@ import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.application.api.ValidationId; import com.yahoo.config.application.api.ValidationOverrides; -import com.yahoo.config.codegen.ConfigGenerator; import com.yahoo.config.codegen.InnerCNode; import com.yahoo.config.model.ApplicationConfigProducerRoot; import com.yahoo.config.model.ConfigModelRegistry; @@ -63,6 +62,7 @@ import java.util.Set; import java.util.logging.Logger; import java.util.stream.Collectors; +import static com.yahoo.config.codegen.ConfiggenUtil.createClassName; import static com.yahoo.text.StringUtilities.quote; /** @@ -358,7 +358,7 @@ public final class VespaModel extends AbstractConfigProducerRoot implements Seri } public ConfigInstance.Builder createBuilder(ConfigDefinitionKey key, ConfigDefinition targetDef) { - String className = ConfigGenerator.createClassName(key.getName()); + String className = createClassName(key.getName()); Class<?> clazz; final String fullClassName = InstanceResolver.packageName(key) + "." + className; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/builder/xml/MetricsBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/builder/xml/MetricsBuilder.java index 98578fdd475..fab1e90cc03 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/builder/xml/MetricsBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/builder/xml/MetricsBuilder.java @@ -25,6 +25,7 @@ import static com.yahoo.vespa.model.admin.monitoring.SystemMetrics.systemMetricS public class MetricsBuilder { private static final String ID_ATTRIBUTE = "id"; + private static final String DISPLAY_NAME_ATTRIBUTE = "display-name"; private final ApplicationType applicationType; private final Map<String, MetricSet> availableMetricSets; @@ -46,9 +47,18 @@ public class MetricsBuilder { return metrics; } + private static Metric metricFromElement(Element elem) { + String m_id = elem.getAttribute(ID_ATTRIBUTE); + String m_dn = elem.getAttribute(DISPLAY_NAME_ATTRIBUTE); + if (m_dn == null || "".equals(m_dn)) { + return new Metric(m_id); + } + return new Metric(m_id, m_dn); + } + private MetricSet buildMetricSet(String consumerId, Element consumerElement) { List<Metric> metrics = XML.getChildren(consumerElement, "metric").stream() - .map(metricElement -> new Metric(metricElement.getAttribute(ID_ATTRIBUTE))) + .map(metricElement -> metricFromElement(metricElement)) .collect(Collectors.toCollection(LinkedList::new)); List<MetricSet> metricSets = XML.getChildren(consumerElement, "metric-set").stream() diff --git a/config-model/src/main/resources/schema/admin.rnc b/config-model/src/main/resources/schema/admin.rnc index 3db16f74e77..26705784a34 100644 --- a/config-model/src/main/resources/schema/admin.rnc +++ b/config-model/src/main/resources/schema/admin.rnc @@ -88,7 +88,10 @@ Metrics = element metrics { element consumer { attribute id { xsd:Name } & element metric-set { attribute id { xsd:Name } }* & - element metric { attribute id { xsd:Name } }* + element metric { + attribute id { xsd:Name } & + attribute display-name { xsd:Name }? + }* }+ } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/admin/DedicatedAdminV4Test.java b/config-model/src/test/java/com/yahoo/vespa/model/admin/DedicatedAdminV4Test.java index f6e7d50f643..5b5094a9c43 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/admin/DedicatedAdminV4Test.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/admin/DedicatedAdminV4Test.java @@ -49,6 +49,11 @@ public class DedicatedAdminV4Test { " <slobroks><nodes count='2' dedicated='true'/></slobroks>" + " <logservers><nodes count='1' dedicated='true'/></logservers>" + " <yamas systemname='vespa.routing' interval='60' />" + + " <metrics>" + + " <consumer id='slingstone'>" + + " <metric id='foobar.count' display-name='foobar'/>" + + " </consumer>" + + " </metrics>" + " <metric-consumers>" + " <consumer name='yamas'>" + " <metric name='upstreams_generated' />" + @@ -73,10 +78,16 @@ public class DedicatedAdminV4Test { assertEquals("vespa.routing", monitoring.getClustername()); assertEquals(60L, (long) monitoring.getIntervalSeconds()); - MetricsConsumer consumer = model.getAdmin().getLegacyUserMetricsConsumers().get(VESPA_CONSUMER_ID); + MetricsConsumer consumer = model.getAdmin().getUserMetrics().getConsumers().get("slingstone"); + assertNotNull(consumer); + Metric metric = consumer.getMetrics().get("foobar.count"); + assertNotNull(metric); + assertEquals("foobar", metric.outputName); + + consumer = model.getAdmin().getLegacyUserMetricsConsumers().get(VESPA_CONSUMER_ID); assertNotNull(consumer); assertEquals(3, consumer.getMetrics().size()); - Metric metric = consumer.getMetrics().get("nginx.upstreams.down.last"); + metric = consumer.getMetrics().get("nginx.upstreams.down.last"); assertNotNull(metric); assertEquals("nginx.upstreams.down", metric.outputName); } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ConstantTensorJsonValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ConstantTensorJsonValidatorTest.java index 9d196287d57..1f8dcc2da64 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ConstantTensorJsonValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ConstantTensorJsonValidatorTest.java @@ -271,7 +271,7 @@ public class ConstantTensorJsonValidatorTest { inputJsonToReader( "{", " 'stats': {", - " 'パープルゴム製のアヒルは私を殺すために望んでいます': true,", + " '\u30d1\u30fc\u30d7\u30eb\u30b4\u30e0\u88fd\u306e\u30a2\u30d2\u30eb\u306f\u79c1\u3092\u6bba\u3059\u305f\u3081\u306b\u671b\u3093\u3067\u3044\u307e\u3059': true,", " 'points': 47", " }", "}")); diff --git a/config-model/src/test/schema-test-files/services.xml b/config-model/src/test/schema-test-files/services.xml index 9143709d855..322f4ed8356 100644 --- a/config-model/src/test/schema-test-files/services.xml +++ b/config-model/src/test/schema-test-files/services.xml @@ -19,7 +19,8 @@ <consumer id="my-consumer"> <metric-set id="my-set" /> <metric id="my-metric"/> - <metric id="my-metric2"/> + <metric id="my-metric2" display-name="my-metric3"/> + <metric display-name="my-metric4" id="my-metric4.avg"/> </consumer> <consumer id="my-consumer2"> <metric-set id="my-set2" /> diff --git a/config/src/test/java/com/yahoo/vespa/config/buildergen/ConfigBuilderGeneratorTest.java b/config/src/test/java/com/yahoo/vespa/config/buildergen/ConfigBuilderGeneratorTest.java index 4d122744388..525921d3052 100644 --- a/config/src/test/java/com/yahoo/vespa/config/buildergen/ConfigBuilderGeneratorTest.java +++ b/config/src/test/java/com/yahoo/vespa/config/buildergen/ConfigBuilderGeneratorTest.java @@ -3,7 +3,6 @@ package com.yahoo.vespa.config.buildergen; import com.google.common.io.Files; import com.yahoo.config.ConfigInstance; -import com.yahoo.config.codegen.ConfigGenerator; import com.yahoo.slime.Cursor; import com.yahoo.slime.Slime; import com.yahoo.vespa.config.ConfigDefinitionKey; @@ -16,6 +15,7 @@ import java.io.IOException; import java.lang.reflect.InvocationTargetException; import java.net.URISyntaxException; +import static com.yahoo.config.codegen.ConfiggenUtil.createClassName; import static junit.framework.TestCase.assertNotNull; import static org.hamcrest.CoreMatchers.is; import static org.junit.Assert.assertThat; @@ -43,7 +43,7 @@ public class ConfigBuilderGeneratorTest { root.setString("intval", "3"); root.setString("stringval", "Hello, world"); payloadApplier.applyPayload(new ConfigPayload(slime)); - String className = ConfigGenerator.createClassName(key.getName()); + String className = createClassName(key.getName()); ConfigInstance instance = (ConfigInstance) builder.getClass().getClassLoader().loadClass("com.yahoo." + key.getNamespace() + "." + className).getConstructor(new Class<?>[]{builder.getClass()}).newInstance(builder); assertNotNull(instance); assertThat(instance.toString(), is("intval 3\nstringval \"Hello, world\"")); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ActivationConflictException.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ActivationConflictException.java new file mode 100644 index 00000000000..48023293cc4 --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ActivationConflictException.java @@ -0,0 +1,15 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server; + +/** + * Exception used when activation cannot be done because activation is for + * an older session than the one that is active now or because current active + * session has changed since the session to be activated was created + * + * @author hmusum + */ +public class ActivationConflictException extends RuntimeException { + public ActivationConflictException(String s) { + super(s); + } +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java index 51995eb98cf..14c130ab463 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java @@ -9,6 +9,7 @@ import com.yahoo.log.LogLevel; import com.yahoo.path.Path; import com.yahoo.transaction.NestedTransaction; import com.yahoo.transaction.Transaction; +import com.yahoo.vespa.config.server.ActivationConflictException; import com.yahoo.vespa.config.server.tenant.ActivateLock; import com.yahoo.vespa.config.server.ApplicationRepository; import com.yahoo.vespa.config.server.TimeoutBudget; @@ -202,25 +203,25 @@ public class Deployment implements com.yahoo.config.provision.Deployment { ", current active session=" + currentActiveSessionSessionId); if (currentActiveSession.isNewerThan(activeSessionAtCreate) && currentActiveSessionSessionId != sessionId) { - String errMsg = currentActiveSession.logPre()+"Cannot activate session " + + String errMsg = currentActiveSession.logPre() + "Cannot activate session " + sessionId + " because the currently active session (" + currentActiveSessionSessionId + ") has changed since session " + sessionId + " was created (was " + activeSessionAtCreate + " at creation time)"; if (ignoreStaleSessionFailure) { log.warning(errMsg + " (Continuing because of force.)"); } else { - throw new IllegalStateException(errMsg); + throw new ActivationConflictException(errMsg); } } } - // As of now, config generation is based on session id, and config generation must be an monotonically + // As of now, config generation is based on session id, and config generation must be a monotonically // increasing number private void checkIfActiveIsNewerThanSessionToBeActivated(long sessionId, long currentActiveSessionId) { if (sessionId < currentActiveSessionId) { - throw new IllegalArgumentException("It is not possible to activate session " + sessionId + - ", because it is older than current active session (" + - currentActiveSessionId + ")"); + throw new ActivationConflictException("It is not possible to activate session " + sessionId + + ", because it is older than current active session (" + + currentActiveSessionId + ")"); } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpErrorResponse.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpErrorResponse.java index 7e0c57f2e5f..7a86357cd95 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpErrorResponse.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpErrorResponse.java @@ -35,6 +35,7 @@ public class HttpErrorResponse extends HttpResponse { public enum errorCodes { APPLICATION_LOCK_FAILURE, BAD_REQUEST, + ACTIVATION_FAILED, INTERNAL_SERVER_ERROR, INVALID_APPLICATION_PACKAGE, METHOD_NOT_ALLOWED, @@ -64,6 +65,10 @@ public class HttpErrorResponse extends HttpResponse { return new HttpErrorResponse(BAD_REQUEST, errorCodes.BAD_REQUEST.name(), msg); } + public static HttpErrorResponse conflictWhenActivating(String msg) { + return new HttpErrorResponse(CONFLICT, errorCodes.ACTIVATION_FAILED.name(), msg); + } + public static HttpErrorResponse methodNotAllowed(String msg) { return new HttpErrorResponse(METHOD_NOT_ALLOWED, errorCodes.METHOD_NOT_ALLOWED.name(), msg); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpHandler.java index 53bfbf5135f..c13d1b3fcfa 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpHandler.java @@ -8,6 +8,7 @@ import com.yahoo.container.jdisc.LoggingRequestHandler; import com.yahoo.container.logging.AccessLog; import com.yahoo.log.LogLevel; import com.yahoo.config.provision.OutOfCapacityException; +import com.yahoo.vespa.config.server.ActivationConflictException; import com.yahoo.yolean.Exceptions; import java.io.PrintWriter; @@ -47,6 +48,8 @@ public class HttpHandler extends LoggingRequestHandler { } } catch (NotFoundException | com.yahoo.vespa.config.server.NotFoundException e) { return HttpErrorResponse.notFoundError(getMessage(e, request)); + } catch (ActivationConflictException e) { + return HttpErrorResponse.conflictWhenActivating(getMessage(e, request)); } catch (BadRequestException | IllegalArgumentException | IllegalStateException e) { return HttpErrorResponse.badRequest(getMessage(e, request)); } catch (OutOfCapacityException e) { diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionActiveHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionActiveHandlerTest.java index 7fca78b087b..f6108611f73 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionActiveHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionActiveHandlerTest.java @@ -67,8 +67,12 @@ import java.time.Clock; import java.util.Collections; import java.util.Optional; -import static com.yahoo.jdisc.Response.Status.*; +import static com.yahoo.jdisc.Response.Status.BAD_REQUEST; +import static com.yahoo.jdisc.Response.Status.CONFLICT; +import static com.yahoo.jdisc.Response.Status.INTERNAL_SERVER_ERROR; import static com.yahoo.jdisc.Response.Status.METHOD_NOT_ALLOWED; +import static com.yahoo.jdisc.Response.Status.NOT_FOUND; +import static com.yahoo.jdisc.Response.Status.OK; import static org.hamcrest.CoreMatchers.containsString; import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.CoreMatchers.not; @@ -154,7 +158,7 @@ public class SessionActiveHandlerTest extends SessionHandlerTest { ActivateRequest activateRequest = new ActivateRequest(sessionId, 1, "", Clock.systemUTC()).invoke(); HttpResponse actResponse = activateRequest.getActResponse(); String message = getRenderedString(actResponse); - assertThat(message, actResponse.getStatus(), Is.is(BAD_REQUEST)); + assertThat(message, actResponse.getStatus(), Is.is(CONFLICT)); assertThat(message, containsString("Cannot activate session 3 because the currently active session (2) has changed since session 3 was created (was 1 at creation time)")); } @@ -178,8 +182,8 @@ public class SessionActiveHandlerTest extends SessionHandlerTest { Clock clock = Clock.systemUTC(); activateAndAssertOK(90l, 0l, clock); activateAndAssertError(92l, 89l, clock, - HttpErrorResponse.errorCodes.BAD_REQUEST, - "tenant:"+tenant+" app:default:default Cannot activate session 92 because the currently active session (90) has changed since session 92 was created (was 89 at creation time)"); + Response.Status.CONFLICT, HttpErrorResponse.errorCodes.ACTIVATION_FAILED, + "tenant:" + tenant + " app:default:default Cannot activate session 92 because the currently active session (90) has changed since session 92 was created (was 89 at creation time)"); } @Test @@ -209,7 +213,7 @@ public class SessionActiveHandlerTest extends SessionHandlerTest { public void require_that_handler_gives_error_when_provisioner_activated_fails() throws Exception { hostProvisioner = new FailingMockProvisioner(); hostProvisioner.activated = false; - activateAndAssertError(1, 0, Clock.systemUTC(), HttpErrorResponse.errorCodes.BAD_REQUEST, "Cannot activate application"); + activateAndAssertError(1, 0, Clock.systemUTC(), BAD_REQUEST, HttpErrorResponse.errorCodes.BAD_REQUEST, "Cannot activate application"); assertFalse(hostProvisioner.activated); } @@ -249,12 +253,13 @@ public class SessionActiveHandlerTest extends SessionHandlerTest { return activateRequest; } - private ActivateRequest activateAndAssertErrorPut(long sessionId, long previousSessionId, Clock clock, HttpErrorResponse.errorCodes errorCode, String expectedError) throws Exception { + private ActivateRequest activateAndAssertErrorPut(long sessionId, long previousSessionId, Clock clock, + int statusCode, HttpErrorResponse.errorCodes errorCode, String expectedError) throws Exception { ActivateRequest activateRequest = new ActivateRequest(sessionId, previousSessionId, "", clock); activateRequest.invoke(); HttpResponse actResponse = activateRequest.getActResponse(); RemoteSession session = activateRequest.getSession(); - assertThat(actResponse.getStatus(), Is.is(BAD_REQUEST)); + assertThat(actResponse.getStatus(), Is.is(statusCode)); String message = getRenderedString(actResponse); assertThat(message, Is.is("{\"error-code\":\"" + errorCode.name() + "\",\"message\":\"" + expectedError + "\"}")); assertThat(session.getStatus(), Is.is(Session.Status.PREPARE)); @@ -353,9 +358,9 @@ public class SessionActiveHandlerTest extends SessionHandlerTest { assertThat(hostProvisioner.lastHosts.size(), is(1)); } - private void activateAndAssertError(long sessionId, long previousSessionId, Clock clock, HttpErrorResponse.errorCodes errorCode, String expectedError) throws Exception { + private void activateAndAssertError(long sessionId, long previousSessionId, Clock clock, int statusCode, HttpErrorResponse.errorCodes errorCode, String expectedError) throws Exception { hostProvisioner.activated = false; - activateAndAssertErrorPut(sessionId, previousSessionId, clock, errorCode, expectedError); + activateAndAssertErrorPut(sessionId, previousSessionId, clock, statusCode, errorCode, expectedError); assertFalse(hostProvisioner.activated); } diff --git a/container-dev/pom.xml b/container-dev/pom.xml index e858ba1c50b..f435a223e9a 100644 --- a/container-dev/pom.xml +++ b/container-dev/pom.xml @@ -86,6 +86,12 @@ <groupId>com.yahoo.vespa</groupId> <artifactId>container-disc</artifactId> <version>${project.version}</version> + <exclusions> + <exclusion> + <groupId>commons-lang</groupId> + <artifactId>commons-lang</artifactId> + </exclusion> + </exclusions> </dependency> <dependency> <groupId>com.yahoo.vespa</groupId> @@ -102,6 +108,12 @@ <groupId>com.yahoo.vespa</groupId> <artifactId>vespajlib</artifactId> <version>${project.version}</version> + <exclusions> + <exclusion> + <groupId>commons-lang</groupId> + <artifactId>commons-lang</artifactId> + </exclusion> + </exclusions> </dependency> <dependency> <groupId>com.yahoo.vespa</groupId> @@ -114,6 +126,10 @@ <version>${project.version}</version> <exclusions> <exclusion> + <groupId>commons-lang</groupId> + <artifactId>commons-lang</artifactId> + </exclusion> + <exclusion> <groupId>org.antlr</groupId> <artifactId>antlr4-runtime</artifactId> </exclusion> diff --git a/container-search/src/main/java/com/yahoo/search/searchers/InputCheckingSearcher.java b/container-search/src/main/java/com/yahoo/search/searchers/InputCheckingSearcher.java index f0dde55b285..ed1af514e88 100644 --- a/container-search/src/main/java/com/yahoo/search/searchers/InputCheckingSearcher.java +++ b/container-search/src/main/java/com/yahoo/search/searchers/InputCheckingSearcher.java @@ -29,7 +29,7 @@ import com.yahoo.search.searchchain.Execution; * Check whether the query tree seems to be "well formed". In other words, run heurestics against * the input data to see whether the query should sent to the search backend. * - * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + * @author Steinar Knutsen */ public class InputCheckingSearcher extends Searcher { diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/MemoryNameService.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/MemoryNameService.java index f70afb3a0a0..10f9e18fa41 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/MemoryNameService.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/MemoryNameService.java @@ -2,9 +2,10 @@ package com.yahoo.vespa.hosted.controller.api.integration.dns; -import java.util.HashSet; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; import java.util.Optional; -import java.util.Set; import java.util.UUID; /** @@ -14,11 +15,15 @@ import java.util.UUID; */ public class MemoryNameService implements NameService { - private final Set<Record> records = new HashSet<>(); + private final List<Record> records = new ArrayList<>(); + + public List<Record> records() { + return Collections.unmodifiableList(records); + } @Override public RecordId createCname(String alias, String canonicalName) { - records.add(new Record("CNAME", alias, canonicalName)); + records.add(new Record(Record.Type.CNAME.name(), alias, canonicalName)); return new RecordId(UUID.randomUUID().toString()); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java index 63711b55d27..1b3821a12bf 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java @@ -248,7 +248,7 @@ public class ApplicationController { DeploymentJobs.JobType jobType = DeploymentJobs.JobType.from(controller.zoneRegistry().system(), zone); ApplicationRevision revision = toApplicationPackageRevision(applicationPackage, options.screwdriverBuildJob); - if( ! options.deployCurrentVersion) { + if ( ! options.deployCurrentVersion) { // Add missing information to application (unless we're deploying the previous version (initial staging step) application = application.with(applicationPackage.deploymentSpec()); application = application.with(applicationPackage.validationOverrides()); @@ -264,14 +264,14 @@ public class ApplicationController { application = application.with(application.deploymentJobs().withTriggering(jobType, version, Optional.of(revision), clock.instant())); } - store(application, lock); // store missing information even if we fail deployment below - // Delete zones not listed in DeploymentSpec, if allowed // We do this at deployment time to be able to return a validation failure message when necessary application = deleteRemovedDeployments(application); // Clean up deployment jobs that are no longer referenced by deployment spec application = deleteUnreferencedDeploymentJobs(application); + + store(application, lock); // store missing information even if we fail deployment below } // Carry out deployment @@ -280,7 +280,8 @@ public class ApplicationController { applicationPackage)); options = withVersion(version, options); ConfigServerClient.PreparedApplication preparedApplication = - configserverClient.prepare(deploymentId, options, rotationInDns.cnames(), rotationInDns.rotations(), applicationPackage.zippedContent()); + configserverClient.prepare(deploymentId, options, rotationInDns.cnames(), rotationInDns.rotations(), + applicationPackage.zippedContent()); preparedApplication.activate(); application = application.with(new Deployment(zone, revision, version, clock.instant())); store(application, lock); @@ -373,7 +374,7 @@ public class ApplicationController { Rotation rotation = applicationRotation.rotations().iterator().next(); // at this time there should be only one rotation assigned String endpointName = alias.toString(); try { - Optional<Record> record = nameService.findRecord(Record.Type.CNAME, rotation.rotationName); + Optional<Record> record = nameService.findRecord(Record.Type.CNAME, endpointName); if (!record.isPresent()) { RecordId recordId = nameService.createCname(endpointName, rotation.rotationName); log.info("Registered mapping with record ID " + recordId.id() + ": " + diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java index 95f4400b2b5..3d756034b71 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java @@ -33,6 +33,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.BuildService.BuildJob; import com.yahoo.vespa.hosted.controller.api.integration.athens.NToken; import com.yahoo.vespa.hosted.controller.api.integration.athens.mock.AthensDbMock; import com.yahoo.vespa.hosted.controller.api.integration.athens.mock.NTokenMock; +import com.yahoo.vespa.hosted.controller.api.integration.dns.Record; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; import com.yahoo.vespa.hosted.controller.application.ApplicationRevision; import com.yahoo.vespa.hosted.controller.application.Change; @@ -607,7 +608,6 @@ public class ControllerTest { // Current system version, matches version in test data Version version = Version.fromString("6.141.117"); - Version oldVersion = Version.fromString("6.98.12"); tester.configServerClientMock().setDefaultConfigServerVersion(version); tester.updateVersionStatus(version); assertEquals(version, tester.controller().versionStatus().systemVersion().get().versionNumber()); @@ -661,4 +661,23 @@ public class ControllerTest { assertEquals("Relevant (cd) data is not removed.", cdJobsCount, newCdJobsCount); } + @Test + public void testDnsAliasRegistration() { + DeploymentTester tester = new DeploymentTester(); + Application application = tester.createApplication("app1", "tenant1", 1, 1L); + + ApplicationPackage applicationPackage = new ApplicationPackageBuilder() + .environment(Environment.prod) + .region("us-west-1") + .region("us-central-1") // Two deployments should result in DNS alias being registered once + .build(); + + tester.deployCompletely(application, applicationPackage); + assertEquals(1, tester.controllerTester().nameService().records().size()); + Optional<Record> record = tester.controllerTester().nameService().findRecord(Record.Type.CNAME, "app1.tenant1.global.vespa.yahooapis.com"); + assertTrue(record.isPresent()); + assertEquals("app1.tenant1.global.vespa.yahooapis.com", record.get().name()); + assertEquals("fake-global-rotation-tenant1.app1", record.get().value()); + } + } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java index 41d9bbea5b2..63feb01f1f2 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java @@ -63,12 +63,14 @@ public final class ControllerTester { private final ZoneRegistryMock zoneRegistryMock = new ZoneRegistryMock(); private final GitHubMock gitHubMock = new GitHubMock(); private final CuratorDb curator = new MockCuratorDb(); - private Controller controller = createController(db, curator, configServerClientMock, clock, gitHubMock, zoneRegistryMock, athensDb); + private final MemoryNameService memoryNameService = new MemoryNameService(); + private Controller controller = createController(db, curator, configServerClientMock, clock, gitHubMock, + zoneRegistryMock, athensDb, memoryNameService); private static final Controller createController(ControllerDb db, CuratorDb curator, ConfigServerClientMock configServerClientMock, ManualClock clock, GitHubMock gitHubClientMock, ZoneRegistryMock zoneRegistryMock, - AthensDbMock athensDb) { + AthensDbMock athensDb, MemoryNameService nameService) { Controller controller = new Controller(db, curator, new MemoryRotationRepository(), @@ -80,7 +82,7 @@ public final class ControllerTester { new CostMock(new MockInsightBackend()), configServerClientMock, new MockMetricsService(), - new MemoryNameService(), + nameService, new MockRoutingGenerator(), new ChefMock(), clock, @@ -93,10 +95,12 @@ public final class ControllerTester { public CuratorDb curator() { return curator; } public ManualClock clock() { return clock; } public AthensDbMock athensDb() { return athensDb; } + public MemoryNameService nameService() { return memoryNameService; } /** Create a new controller instance. Useful to verify that controller state is rebuilt from persistence */ public final void createNewController() { - controller = createController(db, curator, configServerClientMock, clock, gitHubMock, zoneRegistryMock, athensDb); + controller = createController(db, curator, configServerClientMock, clock, gitHubMock, zoneRegistryMock, + athensDb, memoryNameService); } public ZoneRegistryMock getZoneRegistryMock() { return zoneRegistryMock; } diff --git a/jdisc_http_service/pom.xml b/jdisc_http_service/pom.xml index 0c19f2c3d15..9da06709533 100644 --- a/jdisc_http_service/pom.xml +++ b/jdisc_http_service/pom.xml @@ -24,6 +24,7 @@ <dependency> <groupId>org.apache.commons</groupId> <artifactId>commons-lang3</artifactId> + <scope>test</scope> </dependency> <dependency> <groupId>org.apache.httpcomponents</groupId> diff --git a/jrt/src/com/yahoo/jrt/InvocationClient.java b/jrt/src/com/yahoo/jrt/InvocationClient.java index ed95976f546..0b01ea0935b 100644 --- a/jrt/src/com/yahoo/jrt/InvocationClient.java +++ b/jrt/src/com/yahoo/jrt/InvocationClient.java @@ -89,7 +89,7 @@ class InvocationClient implements ReplyHandler, Runnable { if (!conn.cancelReply(this)) { return; } - req.setError(ErrorCode.TIMEOUT, "Request timed out"); + req.setError(ErrorCode.TIMEOUT, "Request timed out after " + timeout + " seconds."); reqWaiter.handleRequestDone(req); } } diff --git a/jrt/tests/com/yahoo/jrt/TimeoutTest.java b/jrt/tests/com/yahoo/jrt/TimeoutTest.java index 363ac8cde59..ce9bb954d74 100644 --- a/jrt/tests/com/yahoo/jrt/TimeoutTest.java +++ b/jrt/tests/com/yahoo/jrt/TimeoutTest.java @@ -56,6 +56,7 @@ public class TimeoutTest extends junit.framework.TestCase { assertTrue(req.isError()); assertEquals(ErrorCode.TIMEOUT, req.errorCode()); + assertEquals("Request timed out after 0.1 seconds.", req.errorMessage()); assertEquals(0, req.returnValues().size()); } diff --git a/orchestrator/pom.xml b/orchestrator/pom.xml index b5ac5233efe..55bf086dfc7 100644 --- a/orchestrator/pom.xml +++ b/orchestrator/pom.xml @@ -34,12 +34,6 @@ <scope>compile</scope> </dependency> <dependency> - <groupId>com.google.auto.value</groupId> - <artifactId>auto-value</artifactId> - <version>1.0-rc1</version> - <scope>provided</scope> - </dependency> - <dependency> <groupId>com.yahoo.vespa</groupId> <artifactId>jaxrs_client_utils</artifactId> <version>${project.version}</version> diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/InstanceStatusResponse.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/InstanceStatusResponse.java index 4ccb971e339..ce675de434d 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/InstanceStatusResponse.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/InstanceStatusResponse.java @@ -2,30 +2,61 @@ package com.yahoo.vespa.orchestrator.resources; import com.fasterxml.jackson.annotation.JsonProperty; -import com.google.auto.value.AutoValue; - import com.yahoo.vespa.applicationmodel.ApplicationInstance; import com.yahoo.vespa.applicationmodel.HostName; import com.yahoo.vespa.service.monitor.ServiceMonitorStatus; import java.util.Map; +import java.util.Objects; /* * @author andreer */ -@AutoValue -public abstract class InstanceStatusResponse { +public class InstanceStatusResponse { + + private final ApplicationInstance<ServiceMonitorStatus> applicationInstance; + private final Map<HostName, String> hostStates; + + private InstanceStatusResponse(ApplicationInstance<ServiceMonitorStatus> applicationInstance, Map<HostName, String> hostStates) { + this.applicationInstance = applicationInstance; + this.hostStates = hostStates; + } + + public static InstanceStatusResponse create( + ApplicationInstance<ServiceMonitorStatus> applicationInstance, + Map<HostName, String> hostStates) { + return new InstanceStatusResponse(applicationInstance, hostStates); + } @JsonProperty("applicationInstance") - public abstract ApplicationInstance<ServiceMonitorStatus> applicationInstance(); + public ApplicationInstance<ServiceMonitorStatus> applicationInstance() { + return applicationInstance; + } @JsonProperty("hostStates") - public abstract Map<HostName, String> hostStates(); + public Map<HostName, String> hostStates() { + return hostStates; + } - public static InstanceStatusResponse create( - ApplicationInstance<ServiceMonitorStatus> applicationInstance, - Map<HostName, String> hostStates) { - return new AutoValue_InstanceStatusResponse(applicationInstance, hostStates); + @Override + public String toString() { + return "InstanceStatusResponse{" + + "applicationInstance=" + applicationInstance + + ", hostStates=" + hostStates + + '}'; } + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + InstanceStatusResponse that = (InstanceStatusResponse) o; + return Objects.equals(applicationInstance, that.applicationInstance) && + Objects.equals(hostStates, that.hostStates); + } + + @Override + public int hashCode() { + return Objects.hash(applicationInstance, hostStates); + } } diff --git a/searchcore/src/apps/proton/proton.cpp b/searchcore/src/apps/proton/proton.cpp index 7515ddc4e23..5cdc8e04eb4 100644 --- a/searchcore/src/apps/proton/proton.cpp +++ b/searchcore/src/apps/proton/proton.cpp @@ -11,8 +11,6 @@ #include <vespa/vespalib/io/fileutil.h> #include <vespa/config/common/exceptions.h> #include <vespa/fastos/app.h> - -#include <string> #include <iostream> #include <vespa/log/log.h> @@ -224,9 +222,9 @@ App::Main() proton.getMetricManager().init(params.identity, proton.getThreadPool()); } EV_STARTED("proton"); - while (!(SIG::INT.check() || SIG::TERM.check())) { + while (!(SIG::INT.check() || SIG::TERM.check() || (spiProton && spiProton->getNode().attemptedStopped()))) { FastOS_Thread::Sleep(1000); - if (spiProton.get() && spiProton->configUpdated()) { + if (spiProton && spiProton->configUpdated()) { storage::ResumeGuard guard(spiProton->getNode().pause()); spiProton->updateConfig(); } @@ -247,7 +245,7 @@ App::Main() } } } - if (spiProton.get()) { + if (spiProton) { spiProton->getNode().requestShutdown("controlled shutdown"); spiProton->shutdown(); EV_STOPPING("servicelayer", "clean shutdown"); diff --git a/searchcore/src/tests/proton/documentdb/configurer/configurer_test.cpp b/searchcore/src/tests/proton/documentdb/configurer/configurer_test.cpp index f0a91a101eb..d76117df4c2 100644 --- a/searchcore/src/tests/proton/documentdb/configurer/configurer_test.cpp +++ b/searchcore/src/tests/proton/documentdb/configurer/configurer_test.cpp @@ -24,6 +24,7 @@ #include <vespa/searchcore/proton/test/documentdb_config_builder.h> #include <vespa/searchcore/proton/test/mock_summary_adapter.h> #include <vespa/searchcore/proton/test/mock_gid_to_lid_change_handler.h> +#include <vespa/searchcore/proton/reference/dummy_gid_to_lid_change_handler.h> #include <vespa/searchlib/index/dummyfileheadercontext.h> #include <vespa/searchlib/transactionlog/nosyncproxy.h> #include <vespa/vespalib/io/fileutil.h> @@ -95,6 +96,7 @@ struct ViewSet search::transactionlog::NoSyncProxy _noTlSyncer; ISummaryManager::SP _summaryMgr; IDocumentMetaStoreContext::SP _dmsc; + std::shared_ptr<IGidToLidChangeHandler> _gidToLidChangeHandler; std::unique_ptr<documentmetastore::ILidReuseDelayer> _lidReuseDelayer; CommitTimeTracker _commitTimeTracker; VarHolder<SearchView::SP> searchView; @@ -123,6 +125,7 @@ ViewSet::ViewSet() _noTlSyncer(), _summaryMgr(), _dmsc(), + _gidToLidChangeHandler(), _lidReuseDelayer(), _commitTimeTracker(TimeStamp()), searchView(), @@ -211,7 +214,7 @@ Fixture::initViewSet(ViewSet &views) IIndexWriter::SP indexWriter(new IndexWriter(indexMgr)); AttributeWriter::SP attrWriter(new AttributeWriter(attrMgr)); ISummaryAdapter::SP summaryAdapter(new SummaryAdapter(summaryMgr)); - std::shared_ptr<IGidToLidChangeHandler> gidToLidChangeHandler(std::make_shared<MockGidToLidChangeHandler>()); + views._gidToLidChangeHandler = std::make_shared<MockGidToLidChangeHandler>(); Schema::SP schema(new Schema()); views._summaryMgr = summaryMgr; views._dmsc = metaStore; @@ -236,6 +239,7 @@ Fixture::initViewSet(ViewSet &views) new SearchableFeedView(StoreOnlyFeedView::Context(summaryAdapter, schema, views.searchView.get()->getDocumentMetaStore(), + *views._gidToLidChangeHandler, views.repo, views._writeService, *views._lidReuseDelayer, @@ -248,7 +252,7 @@ Fixture::initViewSet(ViewSet &views) 0u /* subDbId */, SubDbType::READY), FastAccessFeedView::Context(attrWriter, views._docIdLimit), - SearchableFeedView::Context(indexWriter, gidToLidChangeHandler)))); + SearchableFeedView::Context(indexWriter)))); } @@ -263,6 +267,7 @@ struct MyFastAccessFeedView HwInfo _hwInfo; IDocumentMetaStoreContext::SP _dmsc; + std::shared_ptr<IGidToLidChangeHandler> _gidToLidChangeHandler; std::unique_ptr<documentmetastore::ILidReuseDelayer> _lidReuseDelayer; CommitTimeTracker _commitTimeTracker; VarHolder<FastAccessFeedView::SP> _feedView; @@ -274,6 +279,7 @@ struct MyFastAccessFeedView _writeService(writeService), _hwInfo(), _dmsc(), + _gidToLidChangeHandler(std::make_shared<DummyGidToLidChangeHandler>()), _lidReuseDelayer(), _commitTimeTracker(TimeStamp()), _feedView() @@ -293,7 +299,7 @@ struct MyFastAccessFeedView new documentmetastore::LidReuseDelayer(_writeService, docMetaCtx->get())); DocumentTypeRepo::SP repo = createRepo(); - StoreOnlyFeedView::Context storeOnlyCtx(summaryAdapter, schema, docMetaCtx, repo, _writeService, *_lidReuseDelayer, _commitTimeTracker); + StoreOnlyFeedView::Context storeOnlyCtx(summaryAdapter, schema, docMetaCtx, *_gidToLidChangeHandler, repo, _writeService, *_lidReuseDelayer, _commitTimeTracker); StoreOnlyFeedView::PersistentParams params(1, 1, DocTypeName(DOC_TYPE), _metrics, 0, SubDbType::NOTREADY); AttributeManager::SP mgr(new AttributeManager(BASE_DIR, "test.subdb", TuneFileAttributes(), diff --git a/searchcore/src/tests/proton/documentdb/feedview/feedview_test.cpp b/searchcore/src/tests/proton/documentdb/feedview/feedview_test.cpp index 2ed60f3c078..8af365d6327 100644 --- a/searchcore/src/tests/proton/documentdb/feedview/feedview_test.cpp +++ b/searchcore/src/tests/proton/documentdb/feedview/feedview_test.cpp @@ -755,13 +755,14 @@ struct SearchableFeedViewFixture : public FixtureBase fv(StoreOnlyFeedView::Context(sa, sc._schema, _dmsc, + *_gidToLidChangeHandler, sc.getRepo(), _writeService, _lidReuseDelayer, _commitTimeTracker), pc.getParams(), FastAccessFeedView::Context(aw, _docIdLimit), - SearchableFeedView::Context(iw, _gidToLidChangeHandler)) + SearchableFeedView::Context(iw)) { runInMaster([&]() { _lidReuseDelayer.setHasIndexedOrAttributeFields(true); }); } @@ -776,6 +777,7 @@ struct FastAccessFeedViewFixture : public FixtureBase fv(StoreOnlyFeedView::Context(sa, sc._schema, _dmsc, + *_gidToLidChangeHandler, sc.getRepo(), _writeService, _lidReuseDelayer, diff --git a/searchcore/src/tests/proton/documentdb/storeonlyfeedview/storeonlyfeedview_test.cpp b/searchcore/src/tests/proton/documentdb/storeonlyfeedview/storeonlyfeedview_test.cpp index 15bd55441d4..1ec201fdbd2 100644 --- a/searchcore/src/tests/proton/documentdb/storeonlyfeedview/storeonlyfeedview_test.cpp +++ b/searchcore/src/tests/proton/documentdb/storeonlyfeedview/storeonlyfeedview_test.cpp @@ -12,6 +12,7 @@ #include <vespa/searchcore/proton/server/putdonecontext.h> #include <vespa/searchcore/proton/server/removedonecontext.h> #include <vespa/searchcore/proton/server/storeonlyfeedview.h> +#include <vespa/searchcore/proton/reference/dummy_gid_to_lid_change_handler.h> #include <vespa/searchcore/proton/test/mock_summary_adapter.h> #include <vespa/searchcore/proton/test/thread_utils.h> #include <vespa/searchlib/common/idestructorcallback.h> @@ -67,7 +68,17 @@ DocumentTypeRepo::SP myGetDocumentTypeRepo() { return repo; } -struct MyMinimalFeedView : public StoreOnlyFeedView { +struct MyMinimalFeedViewBase +{ + std::shared_ptr<IGidToLidChangeHandler> gidToLidChangeHandler; + + MyMinimalFeedViewBase() + : gidToLidChangeHandler(std::make_shared<DummyGidToLidChangeHandler>()) + { + } +}; + +struct MyMinimalFeedView : public MyMinimalFeedViewBase, public StoreOnlyFeedView { using UP = std::unique_ptr<MyMinimalFeedView>; int removeMultiAttributesCount; @@ -83,10 +94,12 @@ struct MyMinimalFeedView : public StoreOnlyFeedView { CommitTimeTracker &commitTimeTracker, const PersistentParams ¶ms, int &outstandingMoveOps_) : + MyMinimalFeedViewBase(), StoreOnlyFeedView(StoreOnlyFeedView::Context(summaryAdapter, search::index::Schema::SP(), DocumentMetaStoreContext::SP( new DocumentMetaStoreContext(metaStore)), + *gidToLidChangeHandler, myGetDocumentTypeRepo(), writeService, lidReuseDelayer, diff --git a/searchcore/src/vespa/searchcore/proton/reference/CMakeLists.txt b/searchcore/src/vespa/searchcore/proton/reference/CMakeLists.txt index ca55b44c13d..fe2ca7a7a88 100644 --- a/searchcore/src/vespa/searchcore/proton/reference/CMakeLists.txt +++ b/searchcore/src/vespa/searchcore/proton/reference/CMakeLists.txt @@ -4,6 +4,7 @@ vespa_add_library(searchcore_reference STATIC document_db_reference.cpp document_db_reference_registry.cpp document_db_reference_resolver.cpp + dummy_gid_to_lid_change_handler.cpp gid_to_lid_change_handler.cpp gid_to_lid_change_listener.cpp gid_to_lid_change_registrator.cpp diff --git a/searchcore/src/vespa/searchcore/proton/reference/dummy_gid_to_lid_change_handler.cpp b/searchcore/src/vespa/searchcore/proton/reference/dummy_gid_to_lid_change_handler.cpp new file mode 100644 index 00000000000..c3c516a51dc --- /dev/null +++ b/searchcore/src/vespa/searchcore/proton/reference/dummy_gid_to_lid_change_handler.cpp @@ -0,0 +1,43 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "dummy_gid_to_lid_change_handler.h" + +namespace proton { + +DummyGidToLidChangeHandler::DummyGidToLidChangeHandler() + : IGidToLidChangeHandler() + +{ +} + +DummyGidToLidChangeHandler::~DummyGidToLidChangeHandler() +{ +} + +void +DummyGidToLidChangeHandler::notifyPut(GlobalId, uint32_t, SerialNum) +{ +} + +void +DummyGidToLidChangeHandler::notifyRemove(GlobalId, SerialNum) +{ +} + +void +DummyGidToLidChangeHandler::notifyRemoveDone(GlobalId, SerialNum) +{ +} + +void +DummyGidToLidChangeHandler::addListener(std::unique_ptr<IGidToLidChangeListener>) +{ +} + +void +DummyGidToLidChangeHandler::removeListeners(const vespalib::string &, + const std::set<vespalib::string> &) +{ +} + +} // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/reference/dummy_gid_to_lid_change_handler.h b/searchcore/src/vespa/searchcore/proton/reference/dummy_gid_to_lid_change_handler.h new file mode 100644 index 00000000000..24773924aaa --- /dev/null +++ b/searchcore/src/vespa/searchcore/proton/reference/dummy_gid_to_lid_change_handler.h @@ -0,0 +1,35 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "i_gid_to_lid_change_handler.h" +#include <vector> +#include <mutex> +#include <vespa/vespalib/stllike/hash_map.h> +#include <vespa/document/base/globalid.h> + +namespace searchcorespi { namespace index { class IThreadService; } } + +namespace proton { + +/* + * Dummy class for registering listeners that get notification when + * gid to lid mapping changes. + */ +class DummyGidToLidChangeHandler : public IGidToLidChangeHandler +{ + bool _closed; + +public: + DummyGidToLidChangeHandler(); + virtual ~DummyGidToLidChangeHandler(); + + virtual void notifyPut(GlobalId gid, uint32_t lid, SerialNum serialNum) override; + virtual void notifyRemove(GlobalId gid, SerialNum serialNum) override; + virtual void notifyRemoveDone(GlobalId gid, SerialNum serialNum) override; + virtual void addListener(std::unique_ptr<IGidToLidChangeListener> listener) override; + virtual void removeListeners(const vespalib::string &docTypeName, + const std::set<vespalib::string> &keepNames) override; +}; + +} // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/reference/gid_to_lid_change_handler.h b/searchcore/src/vespa/searchcore/proton/reference/gid_to_lid_change_handler.h index 736a34aba76..840cb61cc2a 100644 --- a/searchcore/src/vespa/searchcore/proton/reference/gid_to_lid_change_handler.h +++ b/searchcore/src/vespa/searchcore/proton/reference/gid_to_lid_change_handler.h @@ -33,9 +33,6 @@ public: GidToLidChangeHandler(); virtual ~GidToLidChangeHandler(); - /** - * Notify gid to lid mapping change. - */ virtual void notifyPut(GlobalId gid, uint32_t lid, SerialNum serialNum) override; virtual void notifyRemove(GlobalId gid, SerialNum serialNum) override; virtual void notifyRemoveDone(GlobalId gid, SerialNum serialNum) override; @@ -45,16 +42,7 @@ public: */ void close(); - /* - * Add listener unless a listener with matching docTypeName and - * name already exists. - */ virtual void addListener(std::unique_ptr<IGidToLidChangeListener> listener) override; - - /** - * Remove listeners with matching docTypeName unless name is present in - * keepNames. - */ virtual void removeListeners(const vespalib::string &docTypeName, const std::set<vespalib::string> &keepNames) override; }; diff --git a/searchcore/src/vespa/searchcore/proton/reference/i_gid_to_lid_change_handler.h b/searchcore/src/vespa/searchcore/proton/reference/i_gid_to_lid_change_handler.h index a3b1db59abd..53ade83ff32 100644 --- a/searchcore/src/vespa/searchcore/proton/reference/i_gid_to_lid_change_handler.h +++ b/searchcore/src/vespa/searchcore/proton/reference/i_gid_to_lid_change_handler.h @@ -25,10 +25,21 @@ public: virtual ~IGidToLidChangeHandler() { } + /* + * Add listener unless a listener with matching docTypeName and + * name already exists. + */ virtual void addListener(std::unique_ptr<IGidToLidChangeListener> listener) = 0; + /** + * Remove listeners with matching docTypeName unless name is present in + * keepNames. + */ virtual void removeListeners(const vespalib::string &docTypeName, const std::set<vespalib::string> &keepNames) = 0; + /** + * Notify gid to lid mapping change. + */ virtual void notifyPut(GlobalId gid, uint32_t lid, SerialNum serialNum) = 0; virtual void notifyRemove(GlobalId gid, SerialNum serialNum) = 0; virtual void notifyRemoveDone(GlobalId gid, SerialNum serialNum) = 0; diff --git a/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb_configurer.cpp b/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb_configurer.cpp index 94663273414..2b9a37fac0b 100644 --- a/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb_configurer.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb_configurer.cpp @@ -24,6 +24,7 @@ FastAccessDocSubDBConfigurer::reconfigureFeedView(const FastAccessFeedView::SP & StoreOnlyFeedView::Context(curr->getSummaryAdapter(), schema, curr->getDocumentMetaStore(), + curr->getGidToLidChangeHandler(), repo, curr->getWriteService(), curr->getLidReuseDelayer(), diff --git a/searchcore/src/vespa/searchcore/proton/server/fast_access_feed_view.cpp b/searchcore/src/vespa/searchcore/proton/server/fast_access_feed_view.cpp index a48c6373c00..5c304a1c7fc 100644 --- a/searchcore/src/vespa/searchcore/proton/server/fast_access_feed_view.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/fast_access_feed_view.cpp @@ -44,7 +44,7 @@ FastAccessFeedView::putAttributes(SerialNum serialNum, { _attributeWriter->put(serialNum, doc, lid, immediateCommit, onWriteDone); if (immediateCommit && onWriteDone) { - onWriteDone->registerPutLid(lid, &_docIdLimit); + onWriteDone->registerPutLid(&_docIdLimit); } } diff --git a/searchcore/src/vespa/searchcore/proton/server/putdonecontext.cpp b/searchcore/src/vespa/searchcore/proton/server/putdonecontext.cpp index ad50e3a92aa..8462e3d0a77 100644 --- a/searchcore/src/vespa/searchcore/proton/server/putdonecontext.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/putdonecontext.cpp @@ -3,15 +3,25 @@ #include "putdonecontext.h" #include <vespa/searchcore/proton/common/feedtoken.h> #include <vespa/searchcore/proton/common/docid_limit.h> +#include <vespa/searchcore/proton/reference/i_gid_to_lid_change_handler.h> namespace proton { PutDoneContext::PutDoneContext(std::unique_ptr<FeedToken> token, const FeedOperation::Type opType, - PerDocTypeFeedMetrics &metrics) + PerDocTypeFeedMetrics &metrics, + IGidToLidChangeHandler &gidToLidChangeHandler, + const document::GlobalId &gid, + uint32_t lid, + search::SerialNum serialNum, + bool changedDbdId) : OperationDoneContext(std::move(token), opType, metrics), - _lid(0), - _docIdLimit(nullptr) + _lid(lid), + _docIdLimit(nullptr), + _gidToLidChangeHandler(gidToLidChangeHandler), + _gid(gid), + _serialNum(serialNum), + _changedDbdId(changedDbdId) { } @@ -20,6 +30,9 @@ PutDoneContext::~PutDoneContext() if (_docIdLimit != nullptr) { _docIdLimit->bumpUpLimit(_lid + 1); } + if (_changedDbdId) { + _gidToLidChangeHandler.notifyPut(_gid, _lid, _serialNum); + } } } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/server/putdonecontext.h b/searchcore/src/vespa/searchcore/proton/server/putdonecontext.h index ba94891fd6e..5a5ff849ac7 100644 --- a/searchcore/src/vespa/searchcore/proton/server/putdonecontext.h +++ b/searchcore/src/vespa/searchcore/proton/server/putdonecontext.h @@ -3,12 +3,15 @@ #pragma once #include "operationdonecontext.h" +#include <vespa/document/base/globalid.h> +#include <vespa/searchlib/common/serialnum.h> namespace proton { class DocIdLimit; +class IGidToLidChangeHandler; /** * Context class for document put operations that acks operation when @@ -21,17 +24,25 @@ class PutDoneContext : public OperationDoneContext { uint32_t _lid; DocIdLimit *_docIdLimit; + IGidToLidChangeHandler &_gidToLidChangeHandler; + document::GlobalId _gid; + search::SerialNum _serialNum; + bool _changedDbdId; // lid or document subdb changed public: PutDoneContext(std::unique_ptr<FeedToken> token, const FeedOperation::Type opType, - PerDocTypeFeedMetrics &metrics); + PerDocTypeFeedMetrics &metrics, + IGidToLidChangeHandler &gidToLidChangeHandler, + const document::GlobalId &gid, + uint32_t lid, + search::SerialNum serialNum, + bool changedDbdId); virtual ~PutDoneContext(); - void registerPutLid(uint32_t lid, DocIdLimit *docIdLimit) + void registerPutLid(DocIdLimit *docIdLimit) { - _lid = lid; _docIdLimit = docIdLimit; } }; diff --git a/searchcore/src/vespa/searchcore/proton/server/removedonecontext.cpp b/searchcore/src/vespa/searchcore/proton/server/removedonecontext.cpp index 219d7482853..301224cf134 100644 --- a/searchcore/src/vespa/searchcore/proton/server/removedonecontext.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/removedonecontext.cpp @@ -3,6 +3,7 @@ #include "removedonecontext.h" #include "removedonetask.h" #include <vespa/searchcore/proton/common/feedtoken.h> +#include <vespa/searchcore/proton/reference/i_gid_to_lid_change_handler.h> namespace proton { @@ -11,10 +12,16 @@ RemoveDoneContext::RemoveDoneContext(std::unique_ptr<FeedToken> token, PerDocTypeFeedMetrics &metrics, vespalib::Executor &executor, IDocumentMetaStore &documentMetaStore, - uint32_t lid) + IGidToLidChangeHandler &gidToLidChangeHandler, + const document::GlobalId &gid, + uint32_t lid, + search::SerialNum serialNum) : OperationDoneContext(std::move(token), opType, metrics), _executor(executor), - _task() + _task(), + _gidToLidChangeHandler(gidToLidChangeHandler), + _gid(gid), + _serialNum(serialNum) { if (lid != 0) { _task = std::make_unique<RemoveDoneTask>(documentMetaStore, lid); @@ -23,6 +30,7 @@ RemoveDoneContext::RemoveDoneContext(std::unique_ptr<FeedToken> token, RemoveDoneContext::~RemoveDoneContext() { + _gidToLidChangeHandler.notifyRemoveDone(_gid, _serialNum); ack(); if (_task) { vespalib::Executor::Task::UP res = _executor.execute(std::move(_task)); diff --git a/searchcore/src/vespa/searchcore/proton/server/removedonecontext.h b/searchcore/src/vespa/searchcore/proton/server/removedonecontext.h index b59219fc8c9..ccd480a5a91 100644 --- a/searchcore/src/vespa/searchcore/proton/server/removedonecontext.h +++ b/searchcore/src/vespa/searchcore/proton/server/removedonecontext.h @@ -4,11 +4,14 @@ #include "operationdonecontext.h" #include <vespa/vespalib/util/executor.h> +#include <vespa/document/base/globalid.h> +#include <vespa/searchlib/common/serialnum.h> namespace proton { class IDocumentMetaStore; +class IGidToLidChangeHandler; /** @@ -23,6 +26,9 @@ class RemoveDoneContext : public OperationDoneContext { vespalib::Executor &_executor; std::unique_ptr<vespalib::Executor::Task> _task; + IGidToLidChangeHandler &_gidToLidChangeHandler; + document::GlobalId _gid; + search::SerialNum _serialNum; public: RemoveDoneContext(std::unique_ptr<FeedToken> token, @@ -30,7 +36,10 @@ public: PerDocTypeFeedMetrics &metrics, vespalib::Executor &executor, IDocumentMetaStore &documentMetaStore, - uint32_t lid); + IGidToLidChangeHandler &gidToLidChangeHandler, + const document::GlobalId &gid, + uint32_t lid, + search::SerialNum serialNum); virtual ~RemoveDoneContext(); }; diff --git a/searchcore/src/vespa/searchcore/proton/server/searchable_doc_subdb_configurer.cpp b/searchcore/src/vespa/searchcore/proton/server/searchable_doc_subdb_configurer.cpp index 1127f73a778..571e64e104c 100644 --- a/searchcore/src/vespa/searchcore/proton/server/searchable_doc_subdb_configurer.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/searchable_doc_subdb_configurer.cpp @@ -47,12 +47,13 @@ SearchableDocSubDBConfigurer::reconfigureFeedView(const IIndexWriter::SP &indexW StoreOnlyFeedView::Context(summaryAdapter, schema, searchView->getDocumentMetaStore(), + curr->getGidToLidChangeHandler(), repo, curr->getWriteService(), curr->getLidReuseDelayer(), curr->getCommitTimeTracker()), curr->getPersistentParams(), FastAccessFeedView::Context(attrWriter, curr->getDocIdLimit()), - SearchableFeedView::Context(indexWriter, curr->getGidToLidChangeHandler())))); + SearchableFeedView::Context(indexWriter)))); } void diff --git a/searchcore/src/vespa/searchcore/proton/server/searchable_feed_view.cpp b/searchcore/src/vespa/searchcore/proton/server/searchable_feed_view.cpp index 913956c509f..14556c86c18 100644 --- a/searchcore/src/vespa/searchcore/proton/server/searchable_feed_view.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/searchable_feed_view.cpp @@ -7,7 +7,6 @@ #include <vespa/searchcore/proton/common/feedtoken.h> #include <vespa/searchcore/proton/metrics/feed_metrics.h> #include <vespa/searchcore/proton/documentmetastore/ilidreusedelayer.h> -#include <vespa/searchcore/proton/reference/i_gid_to_lid_change_handler.h> #include <vespa/vespalib/text/stringtokenizer.h> #include <vespa/vespalib/util/closuretask.h> #include <vespa/vespalib/util/exceptions.h> @@ -37,10 +36,8 @@ bool shouldTrace(StoreOnlyFeedView::OnOperationDoneType onWriteDone, uint32_t tr } -SearchableFeedView::Context::Context(const IIndexWriter::SP &indexWriter, - const std::shared_ptr<IGidToLidChangeHandler> &gidToLidChangeHandler) - : _indexWriter(indexWriter), - _gidToLidChangeHandler(gidToLidChangeHandler) +SearchableFeedView::Context::Context(const IIndexWriter::SP &indexWriter) + : _indexWriter(indexWriter) {} @@ -52,8 +49,7 @@ SearchableFeedView::SearchableFeedView(const StoreOnlyFeedView::Context &storeOn Context ctx) : Parent(storeOnlyCtx, params, fastUpdateCtx), _indexWriter(ctx._indexWriter), - _hasIndexedFields(_schema->getNumIndexFields() > 0), - _gidToLidChangeHandler(ctx._gidToLidChangeHandler) + _hasIndexedFields(_schema->getNumIndexFields() > 0) { } SearchableFeedView::~SearchableFeedView() {} @@ -261,22 +257,4 @@ SearchableFeedView::forceCommit(SerialNum serialNum, OnForceCommitDoneType onCom _writeService.index().execute(makeLambdaTask([=]() { performIndexForceCommit(serialNum, onCommitDone); })); } -void -SearchableFeedView::notifyPutGidToLidChange(const document::GlobalId &gid, uint32_t lid, SerialNum serialNum) -{ - _gidToLidChangeHandler->notifyPut(gid, lid, serialNum); -} - -void -SearchableFeedView::notifyRemoveGidToLidChange(const document::GlobalId &gid, SerialNum serialNum) -{ - _gidToLidChangeHandler->notifyRemove(gid, serialNum); -} - -void -SearchableFeedView::notifyRemoveDoneGidToLidChange(const document::GlobalId &gid, SerialNum serialNum) -{ - _gidToLidChangeHandler->notifyRemoveDone(gid, serialNum); -} - } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/server/searchable_feed_view.h b/searchcore/src/vespa/searchcore/proton/server/searchable_feed_view.h index 4b536d0adde..81d81a6cc27 100644 --- a/searchcore/src/vespa/searchcore/proton/server/searchable_feed_view.h +++ b/searchcore/src/vespa/searchcore/proton/server/searchable_feed_view.h @@ -8,8 +8,6 @@ namespace proton { -class IGidToLidChangeHandler; - /** * The feed view used by the searchable sub database. * @@ -25,17 +23,14 @@ public: struct Context { const IIndexWriter::SP &_indexWriter; - const std::shared_ptr<IGidToLidChangeHandler> &_gidToLidChangeHandler; - Context(const IIndexWriter::SP &indexWriter, - const std::shared_ptr<IGidToLidChangeHandler> &gidToLidChangeHandler); + Context(const IIndexWriter::SP &indexWriter); ~Context(); }; private: const IIndexWriter::SP _indexWriter; const bool _hasIndexedFields; - const std::shared_ptr<IGidToLidChangeHandler> _gidToLidChangeHandler; bool hasIndexedFields() const { return _hasIndexedFields; } @@ -86,17 +81,12 @@ private: void performIndexForceCommit(SerialNum serialNum, OnForceCommitDoneType onCommitDone); void forceCommit(SerialNum serialNum, OnForceCommitDoneType onCommitDone) override; - virtual void notifyPutGidToLidChange(const document::GlobalId &gid, uint32_t lid, SerialNum serialNum) override; - virtual void notifyRemoveGidToLidChange(const document::GlobalId &gid, SerialNum serialNum) override; - virtual void notifyRemoveDoneGidToLidChange(const document::GlobalId &gid, SerialNum serialNum) override; - public: SearchableFeedView(const StoreOnlyFeedView::Context &storeOnlyCtx, const PersistentParams ¶ms, const FastAccessFeedView::Context &fastUpdateCtx, Context ctx); virtual ~SearchableFeedView(); const IIndexWriter::SP &getIndexWriter() const { return _indexWriter; } - const std::shared_ptr<IGidToLidChangeHandler> &getGidToLidChangeHandler() const { return _gidToLidChangeHandler; } void sync() override; }; diff --git a/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.cpp b/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.cpp index e9e0120dac1..f88fb07ab29 100644 --- a/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.cpp @@ -49,8 +49,10 @@ SearchableDocSubDB::SearchableDocSubDB(const Config &cfg, const Context &ctx) getSubDbName(), ctx._fastUpdCtx._storeOnlyCtx._owner.getDistributionKey()), _numSearcherThreads(cfg._numSearcherThreads), _warmupExecutor(ctx._warmupExecutor), - _gidToLidChangeHandler(std::make_shared<GidToLidChangeHandler>()) -{ } + _realGidToLidChangeHandler(std::make_shared<GidToLidChangeHandler>()) +{ + _gidToLidChangeHandler = _realGidToLidChangeHandler; +} SearchableDocSubDB::~SearchableDocSubDB() { @@ -244,7 +246,7 @@ SearchableDocSubDB::initFeedView(const IAttributeWriter::SP &attrWriter, SearchableFeedView::UP feedView(new SearchableFeedView(getStoreOnlyFeedViewContext(configSnapshot), getFeedViewPersistentParams(), FastAccessFeedView::Context(attrWriter, _docIdLimit), - SearchableFeedView::Context(getIndexWriter(), _gidToLidChangeHandler))); + SearchableFeedView::Context(getIndexWriter()))); // XXX: Not exception safe. _rFeedView.set(SearchableFeedView::SP(feedView.release())); @@ -351,7 +353,7 @@ SearchableDocSubDB::updateLidReuseDelayer(const LidReuseDelayerConfig &config) void SearchableDocSubDB::close() { - _gidToLidChangeHandler->close(); + _realGidToLidChangeHandler->close(); Parent::close(); } diff --git a/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.h b/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.h index fa3ded56c97..91ce868937b 100644 --- a/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.h +++ b/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.h @@ -83,7 +83,7 @@ private: SearchableDocSubDBConfigurer _configurer; const size_t _numSearcherThreads; vespalib::ThreadExecutor &_warmupExecutor; - std::shared_ptr<GidToLidChangeHandler> _gidToLidChangeHandler; + std::shared_ptr<GidToLidChangeHandler> _realGidToLidChangeHandler; // Note: lifetime of indexManager must be handled by caller. std::shared_ptr<initializer::InitializerTask> diff --git a/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.cpp b/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.cpp index b3f98b2a8ac..5fbae951536 100644 --- a/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.cpp @@ -19,6 +19,7 @@ #include <vespa/searchcore/proton/index/index_writer.h> #include <vespa/searchcore/proton/metrics/legacy_documentdb_metrics.h> #include <vespa/searchcore/proton/metrics/metricswireservice.h> +#include <vespa/searchcore/proton/reference/dummy_gid_to_lid_change_handler.h> #include <vespa/searchlib/attribute/configconverter.h> #include <vespa/searchlib/docstore/document_store_visitor_progress.h> #include <vespa/searchlib/util/fileheadertk.h> @@ -135,7 +136,8 @@ StoreOnlyDocSubDB::StoreOnlyDocSubDB(const Config &cfg, const Context &ctx) _subDbType(cfg._subDbType), _fileHeaderContext(*this, ctx._fileHeaderContext, _docTypeName, _baseDir), _lidReuseDelayer(), - _commitTimeTracker(TimeStamp::Seconds(3600.0)) + _commitTimeTracker(TimeStamp::Seconds(3600.0)), + _gidToLidChangeHandler(std::make_shared<DummyGidToLidChangeHandler>()) { vespalib::mkdir(_baseDir, false); // Assume parent is created. } @@ -387,6 +389,7 @@ StoreOnlyDocSubDB::getStoreOnlyFeedViewContext(const DocumentDBConfig &configSna return StoreOnlyFeedView::Context(getSummaryAdapter(), configSnapshot.getSchemaSP(), _metaStoreCtx, + *_gidToLidChangeHandler, configSnapshot.getDocumentTypeRepoSP(), _writeService, *_lidReuseDelayer, _commitTimeTracker); diff --git a/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.h b/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.h index b9e23e2436c..ed540053de3 100644 --- a/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.h +++ b/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.h @@ -176,6 +176,7 @@ protected: StoreOnlySubDBFileHeaderContext _fileHeaderContext; std::unique_ptr<documentmetastore::ILidReuseDelayer> _lidReuseDelayer; CommitTimeTracker _commitTimeTracker; + std::shared_ptr<IGidToLidChangeHandler> _gidToLidChangeHandler; std::shared_ptr<initializer::InitializerTask> createSummaryManagerInitializer(const ProtonConfig::Summary protonSummaryCfg, diff --git a/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.cpp b/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.cpp index 3e7e3d6ddfa..749078c56ba 100644 --- a/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.cpp @@ -12,6 +12,7 @@ #include <vespa/searchcore/proton/common/feedtoken.h> #include <vespa/searchcore/proton/documentmetastore/ilidreusedelayer.h> #include <vespa/searchcore/proton/metrics/feed_metrics.h> +#include <vespa/searchcore/proton/reference/i_gid_to_lid_change_handler.h> #include <vespa/searchlib/common/scheduletaskcallback.h> #include <vespa/vespalib/util/exceptions.h> @@ -26,6 +27,7 @@ using document::DocumentUpdate; using search::index::Schema; using search::makeLambdaTask; using search::IDestructorCallback; +using search::SerialNum; using storage::spi::BucketInfoResult; using storage::spi::Timestamp; using vespalib::IllegalStateException; @@ -56,32 +58,37 @@ private: public: PutDoneContextForMove(std::unique_ptr<FeedToken> token, const FeedOperation::Type opType, - PerDocTypeFeedMetrics &metrics, IDestructorCallback::SP moveDoneCtx) - : PutDoneContext(std::move(token), opType, metrics), + PerDocTypeFeedMetrics &metrics, + IGidToLidChangeHandler &gidToLidChangeHandler, + const document::GlobalId &gid, + uint32_t lid, search::SerialNum serialNum, bool changedDbdId, IDestructorCallback::SP moveDoneCtx) + : PutDoneContext(std::move(token), opType, metrics, gidToLidChangeHandler, gid, lid, serialNum, changedDbdId), _moveDoneCtx(std::move(moveDoneCtx)) {} virtual ~PutDoneContextForMove() {} }; std::shared_ptr<PutDoneContext> -createPutDoneContext(FeedToken::UP &token, FeedOperation::Type opType, PerDocTypeFeedMetrics &metrics, bool force, +createPutDoneContext(FeedToken::UP &token, FeedOperation::Type opType, PerDocTypeFeedMetrics &metrics, + IGidToLidChangeHandler &gidToLidChangeHandler, + const document::GlobalId &gid, uint32_t lid, + SerialNum serialNum, bool changedDbdId, IDestructorCallback::SP moveDoneCtx) { std::shared_ptr<PutDoneContext> result; - if (token || force) { - if (moveDoneCtx) { - result = std::make_shared<PutDoneContextForMove>(std::move(token), opType, metrics, std::move(moveDoneCtx)); - } else { - result = std::make_shared<PutDoneContext>(std::move(token), opType, metrics); - } + if (moveDoneCtx) { + result = std::make_shared<PutDoneContextForMove>(std::move(token), opType, metrics, gidToLidChangeHandler, gid, lid, serialNum, changedDbdId, std::move(moveDoneCtx)); + } else { + result = std::make_shared<PutDoneContext>(std::move(token), opType, metrics, gidToLidChangeHandler, gid, lid, serialNum, changedDbdId); } return result; } std::shared_ptr<PutDoneContext> -createPutDoneContext(FeedToken::UP &token, FeedOperation::Type opType, PerDocTypeFeedMetrics &metrics, bool force) +createPutDoneContext(FeedToken::UP &token, FeedOperation::Type opType, PerDocTypeFeedMetrics &metrics, IGidToLidChangeHandler &gidToLidChangeHandler, + const document::GlobalId &gid, uint32_t lid, SerialNum serialNum, bool changedDbdId) { - return createPutDoneContext(token, opType, metrics, force, IDestructorCallback::SP()); + return createPutDoneContext(token, opType, metrics, gidToLidChangeHandler, gid, lid, serialNum, changedDbdId, IDestructorCallback::SP()); } std::shared_ptr<UpdateDoneContext> @@ -108,9 +115,13 @@ private: public: RemoveDoneContextForMove(std::unique_ptr<FeedToken> token, const FeedOperation::Type opType, PerDocTypeFeedMetrics &metrics, vespalib::Executor &executor, - IDocumentMetaStore &documentMetaStore, uint32_t lid, + IDocumentMetaStore &documentMetaStore, + IGidToLidChangeHandler &gidToLidChangeHandler, + const document::GlobalId &gid, + uint32_t lid, + SerialNum serialNum, IDestructorCallback::SP moveDoneCtx) - : RemoveDoneContext(std::move(token), opType, metrics, executor, documentMetaStore, lid), + : RemoveDoneContext(std::move(token), opType, metrics, executor, documentMetaStore, gidToLidChangeHandler, gid, lid, serialNum), _moveDoneCtx(std::move(moveDoneCtx)) {} virtual ~RemoveDoneContextForMove() {} @@ -119,15 +130,19 @@ public: std::shared_ptr<RemoveDoneContext> createRemoveDoneContext(std::unique_ptr<FeedToken> token, const FeedOperation::Type opType, PerDocTypeFeedMetrics &metrics, vespalib::Executor &executor, - IDocumentMetaStore &documentMetaStore, uint32_t lid, + IDocumentMetaStore &documentMetaStore, + IGidToLidChangeHandler &gidToLidChangeHandler, + const document::GlobalId &gid, + uint32_t lid, + SerialNum serialNum, IDestructorCallback::SP moveDoneCtx) { if (moveDoneCtx) { return std::make_shared<RemoveDoneContextForMove> - (std::move(token), opType, metrics, executor, documentMetaStore, lid, std::move(moveDoneCtx)); + (std::move(token), opType, metrics, executor, documentMetaStore, gidToLidChangeHandler, gid, lid, serialNum, std::move(moveDoneCtx)); } else { return std::make_shared<RemoveDoneContext> - (std::move(token), opType, metrics, executor, documentMetaStore, lid); + (std::move(token), opType, metrics, executor, documentMetaStore, gidToLidChangeHandler, gid, lid, serialNum); } } @@ -202,7 +217,8 @@ StoreOnlyFeedView::StoreOnlyFeedView(const Context &ctx, const PersistentParams _schema(ctx._schema), _writeService(ctx._writeService), _params(params), - _metaStore(_documentMetaStoreContext->get()) + _metaStore(_documentMetaStoreContext->get()), + _gidToLidChangeHandler(ctx._gidToLidChangeHandler) { _docType = _repo->getDocumentType(_params._docTypeName.getName()); } @@ -279,7 +295,6 @@ StoreOnlyFeedView::internalPut(FeedToken::UP token, const PutOperation &putOp) putOp.getSubDbId(), putOp.getLid(), putOp.getPrevSubDbId(), putOp.getPrevLid(), _params._subDbId, doc->toString(true).size(), doc->toString(true).c_str()); - uint32_t oldDocIdLimit = _metaStore.getCommittedDocIdLimit(); adjustMetaStore(putOp, docId); considerEarlyAck(token, putOp.getType()); @@ -287,16 +302,18 @@ StoreOnlyFeedView::internalPut(FeedToken::UP token, const PutOperation &putOp) if (putOp.getValidDbdId(_params._subDbId)) { bool immediateCommit = _commitTimeTracker.needCommit(); + const document::GlobalId &gid = docId.getGlobalId(); std::shared_ptr<PutDoneContext> onWriteDone = createPutDoneContext(token, putOp.getType(), _params._metrics, - immediateCommit && putOp.getLid() >= oldDocIdLimit); + _gidToLidChangeHandler, gid, putOp.getLid(), serialNum, putOp.changedDbdId()); putSummary(serialNum, putOp.getLid(), doc, onWriteDone); putAttributes(serialNum, putOp.getLid(), *doc, immediateCommit, onWriteDone); putIndexedFields(serialNum, putOp.getLid(), doc, immediateCommit, onWriteDone); } if (docAlreadyExists && putOp.changedDbdId()) { assert(!putOp.getValidDbdId(_params._subDbId)); - internalRemove(std::move(token), serialNum, putOp.getPrevLid(), putOp.getType(), IDestructorCallback::SP()); + const document::GlobalId &gid = docId.getGlobalId(); + internalRemove(std::move(token), serialNum, gid, putOp.getPrevLid(), putOp.getType(), IDestructorCallback::SP()); } if (token.get() != NULL) { token->ack(putOp.getType(), _params._metrics); @@ -573,7 +590,8 @@ StoreOnlyFeedView::internalRemove(FeedToken::UP token, const RemoveOperation &rm if (rmOp.getValidPrevDbdId(_params._subDbId)) { if (rmOp.changedDbdId()) { assert(!rmOp.getValidDbdId(_params._subDbId)); - internalRemove(std::move(token), serialNum, rmOp.getPrevLid(), rmOp.getType(), IDestructorCallback::SP()); + const document::GlobalId &gid = docId.getGlobalId(); + internalRemove(std::move(token), serialNum, gid, rmOp.getPrevLid(), rmOp.getType(), IDestructorCallback::SP()); } } if (token.get() != NULL) { @@ -582,7 +600,7 @@ StoreOnlyFeedView::internalRemove(FeedToken::UP token, const RemoveOperation &rm } void -StoreOnlyFeedView::internalRemove(FeedToken::UP token, SerialNum serialNum, Lid lid, +StoreOnlyFeedView::internalRemove(FeedToken::UP token, SerialNum serialNum, const document::GlobalId &gid, Lid lid, FeedOperation::Type opType, IDestructorCallback::SP moveDoneCtx) { removeSummary(serialNum, lid); @@ -590,10 +608,10 @@ StoreOnlyFeedView::internalRemove(FeedToken::UP token, SerialNum serialNum, Lid std::shared_ptr<RemoveDoneContext> onWriteDone; if (explicitReuseLid || token) { onWriteDone = createRemoveDoneContext(std::move(token), opType, _params._metrics, _writeService.master(), - _metaStore, (explicitReuseLid ? lid : 0u), moveDoneCtx); + _metaStore, _gidToLidChangeHandler, gid, (explicitReuseLid ? lid : 0u), serialNum, moveDoneCtx); } else if (moveDoneCtx) { onWriteDone = createRemoveDoneContext(FeedToken::UP(), opType, _params._metrics, _writeService.master(), - _metaStore, 0u, moveDoneCtx); + _metaStore, _gidToLidChangeHandler, gid, 0u, serialNum, moveDoneCtx); } bool immediateCommit = _commitTimeTracker.needCommit(); removeAttributes(serialNum, lid, immediateCommit, onWriteDone); @@ -611,17 +629,12 @@ StoreOnlyFeedView::adjustMetaStore(const DocumentOperation &op, const DocumentId op.getLid() != op.getPrevLid()) { moveMetaData(_metaStore, docId, op); - notifyPutGidToLidChange(docId.getGlobalId(), op.getLid(), serialNum); } else { putMetaData(_metaStore, docId, op, _params._subDbType == SubDbType::REMOVED); - if (op.getDbDocumentId() != op.getPrevDbDocumentId()) { - notifyPutGidToLidChange(docId.getGlobalId(), op.getLid(), serialNum); - } } } else if (op.getValidPrevDbdId(_params._subDbId)) { + _gidToLidChangeHandler.notifyRemove(docId.getGlobalId(), serialNum); removeMetaData(_metaStore, docId, op, _params._subDbType == SubDbType::REMOVED); - notifyRemoveGidToLidChange(docId.getGlobalId(), serialNum); - notifyRemoveDoneGidToLidChange(docId.getGlobalId(), serialNum); } _metaStore.commit(serialNum, serialNum); } @@ -650,12 +663,14 @@ StoreOnlyFeedView::removeDocuments(const RemoveDocumentsOperation &op, bool remo bool explicitReuseLids = false; if (useDMS) { std::vector<document::GlobalId> gidsToRemove(getGidsToRemove(_metaStore, lidsToRemove)); - _metaStore.removeBatch(lidsToRemove, ctx->getDocIdLimit()); for (const auto &gid : gidsToRemove) { - notifyRemoveGidToLidChange(gid, serialNum); - notifyRemoveDoneGidToLidChange(gid, serialNum); + _gidToLidChangeHandler.notifyRemove(gid, serialNum); } + _metaStore.removeBatch(lidsToRemove, ctx->getDocIdLimit()); _metaStore.commit(serialNum, serialNum); + for (const auto &gid : gidsToRemove) { + _gidToLidChangeHandler.notifyRemoveDone(gid, serialNum); + } explicitReuseLids = _lidReuseDelayer.delayReuse(lidsToRemove); } std::shared_ptr<search::IDestructorCallback> onWriteDone; @@ -739,26 +754,28 @@ StoreOnlyFeedView::handleMove(const MoveOperation &moveOp, IDestructorCallback:: moveOp.getSubDbId(), moveOp.getLid(), moveOp.getPrevSubDbId(), moveOp.getPrevLid(), _params._subDbId, doc->toString(true).size(), doc->toString(true).c_str()); - uint32_t oldDocIdLimit = _metaStore.getCommittedDocIdLimit(); adjustMetaStore(moveOp, docId); bool docAlreadyExists = moveOp.getValidPrevDbdId(_params._subDbId); if (moveOp.getValidDbdId(_params._subDbId)) { bool immediateCommit = _commitTimeTracker.needCommit(); + const document::GlobalId &gid = docId.getGlobalId(); FeedToken::UP token; std::shared_ptr<PutDoneContext> onWriteDone = createPutDoneContext(token, moveOp.getType(), _params._metrics, - immediateCommit && (moveOp.getLid() >= oldDocIdLimit), doneCtx); + _gidToLidChangeHandler, gid, moveOp.getLid(), + serialNum, moveOp.changedDbdId(), doneCtx); putSummary(serialNum, moveOp.getLid(), doc, onWriteDone); putAttributes(serialNum, moveOp.getLid(), *doc, immediateCommit, onWriteDone); putIndexedFields(serialNum, moveOp.getLid(), doc, immediateCommit, onWriteDone); } if (docAlreadyExists && moveOp.changedDbdId()) { - internalRemove(FeedToken::UP(), serialNum, moveOp.getPrevLid(), moveOp.getType(), doneCtx); + const document::GlobalId &gid = docId.getGlobalId(); + internalRemove(FeedToken::UP(), serialNum, gid, moveOp.getPrevLid(), moveOp.getType(), doneCtx); } } void -StoreOnlyFeedView::heartBeat(search::SerialNum serialNum) +StoreOnlyFeedView::heartBeat(SerialNum serialNum) { assert(_writeService.master().isCurrentThread()); _metaStore.removeAllOldGenerations(); @@ -807,13 +824,4 @@ StoreOnlyFeedView::getDocumentMetaStorePtr() const return &_documentMetaStoreContext->get(); } -void -StoreOnlyFeedView::notifyPutGidToLidChange(const document::GlobalId &, uint32_t, SerialNum) {} - -void -StoreOnlyFeedView::notifyRemoveGidToLidChange(const document::GlobalId &, SerialNum) {} - -void -StoreOnlyFeedView::notifyRemoveDoneGidToLidChange(const document::GlobalId &, SerialNum) {} - } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.h b/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.h index ec3003fb5d6..3b3a5efbe24 100644 --- a/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.h +++ b/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.h @@ -22,6 +22,8 @@ namespace search { class IDestructorCallback; } +namespace document { class GLobalId; } + namespace proton { class IReplayConfig; @@ -31,6 +33,7 @@ class OperationDoneContext; class PutDoneContext; class RemoveDoneContext; class CommitTimeTracker; +class IGidToLidChangeHandler; namespace documentmetastore { class ILidReuseDelayer; } @@ -68,6 +71,7 @@ public: const ISummaryAdapter::SP &_summaryAdapter; const search::index::Schema::SP &_schema; const IDocumentMetaStoreContext::SP &_documentMetaStoreContext; + IGidToLidChangeHandler &_gidToLidChangeHandler; const document::DocumentTypeRepo::SP &_repo; searchcorespi::index::IThreadingService &_writeService; documentmetastore::ILidReuseDelayer &_lidReuseDelayer; @@ -76,6 +80,7 @@ public: Context(const ISummaryAdapter::SP &summaryAdapter, const search::index::Schema::SP &schema, const IDocumentMetaStoreContext::SP &documentMetaStoreContext, + IGidToLidChangeHandler &gidToLidChangeHandler, const document::DocumentTypeRepo::SP &repo, searchcorespi::index::IThreadingService &writeService, documentmetastore::ILidReuseDelayer &lidReuseDelayer, @@ -83,6 +88,7 @@ public: : _summaryAdapter(summaryAdapter), _schema(schema), _documentMetaStoreContext(documentMetaStoreContext), + _gidToLidChangeHandler(gidToLidChangeHandler), _repo(repo), _writeService(writeService), _lidReuseDelayer(lidReuseDelayer), @@ -143,6 +149,7 @@ protected: searchcorespi::index::IThreadingService &_writeService; PersistentParams _params; IDocumentMetaStore &_metaStore; + IGidToLidChangeHandler &_gidToLidChangeHandler; private: searchcorespi::index::IThreadService & summaryExecutor() { @@ -165,7 +172,7 @@ private: void internalPut(FeedTokenUP token, const PutOperation &putOp); void internalUpdate(FeedTokenUP token, const UpdateOperation &updOp); - bool lookupDocId(const document::DocumentId &gid, Lid & lid) const; + bool lookupDocId(const document::DocumentId &docId, Lid & lid) const; void internalRemove(FeedTokenUP token, const RemoveOperation &rmOp); // Removes documents from meta store and document store. @@ -173,16 +180,12 @@ private: size_t removeDocuments(const RemoveDocumentsOperation &op, bool remove_index_and_attribute_fields, bool immediateCommit); - void internalRemove(FeedTokenUP token, SerialNum serialNum, Lid lid, + void internalRemove(FeedTokenUP token, SerialNum serialNum, const document::GlobalId &gid, Lid lid, FeedOperation::Type opType, std::shared_ptr<search::IDestructorCallback> moveDoneCtx); // Ack token early if visibility delay is nonzero void considerEarlyAck(FeedTokenUP &token, FeedOperation::Type opType); - virtual void notifyPutGidToLidChange(const document::GlobalId &gid, uint32_t lid, SerialNum serialNum); - virtual void notifyRemoveGidToLidChange(const document::GlobalId &gid, SerialNum serialNum); - virtual void notifyRemoveDoneGidToLidChange(const document::GlobalId &gid, SerialNum serialNum); - void makeUpdatedDocument(SerialNum serialNum, Lid lid, DocumentUpdate::SP upd, OnOperationDoneType onWriteDone,PromisedDoc promisedDoc, PromisedStream promisedStream); @@ -229,6 +232,7 @@ public: searchcorespi::index::IThreadingService &getWriteService() { return _writeService; } documentmetastore::ILidReuseDelayer &getLidReuseDelayer() { return _lidReuseDelayer; } CommitTimeTracker &getCommitTimeTracker() { return _commitTimeTracker; } + IGidToLidChangeHandler &getGidToLidChangeHandler() const { return _gidToLidChangeHandler; } /** * Implements IFeedView. diff --git a/searchlib/src/vespa/searchlib/attribute/dociditerator.h b/searchlib/src/vespa/searchlib/attribute/dociditerator.h index 194d57a2b1a..a6366172269 100644 --- a/searchlib/src/vespa/searchlib/attribute/dociditerator.h +++ b/searchlib/src/vespa/searchlib/attribute/dociditerator.h @@ -42,12 +42,9 @@ public: } void lower_bound(uint32_t docId) { - if (valid() && (docId > getKey())) { - linearSeek(docId); - } else { - _cur = _begin; - linearSeek(docId); - } + P keyWrap; + keyWrap._key = docId; + _cur = std::lower_bound<const P *, P>(_begin, _end, keyWrap); } void swap(DocIdIterator &rhs) { diff --git a/searchlib/src/vespa/searchlib/attribute/imported_attribute_vector.cpp b/searchlib/src/vespa/searchlib/attribute/imported_attribute_vector.cpp index 270f4d51788..036f482b2e8 100644 --- a/searchlib/src/vespa/searchlib/attribute/imported_attribute_vector.cpp +++ b/searchlib/src/vespa/searchlib/attribute/imported_attribute_vector.cpp @@ -147,47 +147,5 @@ long ImportedAttributeVector::onSerializeForDescendingSort(DocId doc, _reference_attribute->getReferencedLid(doc), serTo, available, bc); } -namespace { - -class ImportedAttributeGuard : public AttributeGuard { -public: - ImportedAttributeGuard(const AttributeVectorSP& target_attr, - const AttributeVectorSP& reference_attr) - : AttributeGuard(), - _target_attr_guard(target_attr), - _reference_attr_guard(reference_attr) - { - } - -private: - AttributeGuard _target_attr_guard; - AttributeGuard _reference_attr_guard; -}; - -class ImportedAttributeEnumGuard : public AttributeEnumGuard { -public: - ImportedAttributeEnumGuard(const AttributeVectorSP& target_attr, - const AttributeVectorSP& reference_attr) - : AttributeEnumGuard(AttributeVectorSP()), - _target_attr_enum_guard(target_attr), - _reference_attr_guard(reference_attr) - { - } - -private: - AttributeEnumGuard _target_attr_enum_guard; - AttributeGuard _reference_attr_guard; -}; - -} - -std::unique_ptr<AttributeGuard> ImportedAttributeVector::acquireGuard() const { - return std::make_unique<ImportedAttributeGuard>(_target_attribute, _reference_attribute); -} - -std::unique_ptr<AttributeEnumGuard> ImportedAttributeVector::acquireEnumGuard() const { - return std::make_unique<ImportedAttributeEnumGuard>(_target_attribute, _reference_attribute); -} - } } diff --git a/searchlib/src/vespa/searchlib/attribute/imported_attribute_vector.h b/searchlib/src/vespa/searchlib/attribute/imported_attribute_vector.h index 686818c1093..bddd0fa9093 100644 --- a/searchlib/src/vespa/searchlib/attribute/imported_attribute_vector.h +++ b/searchlib/src/vespa/searchlib/attribute/imported_attribute_vector.h @@ -66,21 +66,6 @@ public: return _target_attribute; } - /** - * Acquire an opaque composite guard that covers both the target attribute and - * the reference attribute. Note that these are not directly accessible via the - * returned guard object itself; it does not expose a valid pointer (i.e. get() will - * return nullptr). - */ - std::unique_ptr<AttributeGuard> acquireGuard() const; - /** - * Acquires a composite guard similar to acquireGuard(), but the target attribute is - * covered by an AttributeEnumGuard instead of a regular AttributeGuard. - * - * The reference attribute is _not_ covered by an enum guard. - */ - std::unique_ptr<AttributeEnumGuard> acquireEnumGuard() const; - /* * Create an imported attribute with a snapshot of lid to lid mapping. */ diff --git a/storage/src/tests/storageserver/mergethrottlertest.cpp b/storage/src/tests/storageserver/mergethrottlertest.cpp index 29ff780807f..aa0c0667b0d 100644 --- a/storage/src/tests/storageserver/mergethrottlertest.cpp +++ b/storage/src/tests/storageserver/mergethrottlertest.cpp @@ -146,6 +146,7 @@ class MergeThrottlerTest : public CppUnit::TestFixture { CPPUNIT_TEST(testNewClusterStateAbortsAllOutdatedActiveMerges); CPPUNIT_TEST(backpressure_busy_bounces_merges_for_configured_duration); CPPUNIT_TEST(source_only_merges_are_not_affected_by_backpressure); + CPPUNIT_TEST(backpressure_evicts_all_queued_merges); CPPUNIT_TEST_SUITE_END(); public: void setUp() override; @@ -177,6 +178,7 @@ public: void testNewClusterStateAbortsAllOutdatedActiveMerges(); void backpressure_busy_bounces_merges_for_configured_duration(); void source_only_merges_are_not_affected_by_backpressure(); + void backpressure_evicts_all_queued_merges(); private: static const int _storageNodeCount = 3; static const int _messageWaitTime = 100; @@ -193,6 +195,8 @@ private: const std::shared_ptr<api::StorageMessage>& msg, const api::MessageType& expectedReplyType, api::ReturnCode::Result expectedResultCode); + + void fill_throttler_queue_with_n_commands(uint16_t throttler_index, size_t queued_count); }; const int MergeThrottlerTest::_storageNodeCount; @@ -1586,7 +1590,7 @@ void MergeThrottlerTest::backpressure_busy_bounces_merges_for_configured_duratio CPPUNIT_ASSERT_EQUAL(uint64_t(0), _throttlers[0]->getMetrics().bounced_due_to_back_pressure.getValue()); CPPUNIT_ASSERT_EQUAL(uint64_t(0), _throttlers[0]->getMetrics().local.failures.busy.getValue()); - sendAndExpectReply(MergeBuilder(bucket).clusterStateVersion(10).create(), + sendAndExpectReply(MergeBuilder(bucket).create(), api::MessageType::MERGEBUCKET_REPLY, api::ReturnCode::BUSY); @@ -1596,7 +1600,7 @@ void MergeThrottlerTest::backpressure_busy_bounces_merges_for_configured_duratio _servers[0]->getClock().addSecondsToTime(15); // Test-config has duration set to 15 seconds // Backpressure has now been lifted. New merges should be forwarded // to next node in chain as expected instead of being bounced with a reply. - sendMerge(MergeBuilder(bucket).clusterStateVersion(10)); + sendMerge(MergeBuilder(bucket)); _topLinks[0]->waitForMessage(MessageType::MERGEBUCKET, _messageWaitTime); CPPUNIT_ASSERT(!_throttlers[0]->backpressure_mode_active()); @@ -1608,12 +1612,35 @@ void MergeThrottlerTest::source_only_merges_are_not_affected_by_backpressure() { _throttlers[2]->apply_timed_backpressure(); document::BucketId bucket(16, 6789); - _topLinks[2]->sendDown(MergeBuilder(bucket).clusterStateVersion(10).chain(0, 1).source_only(2).create()); + _topLinks[2]->sendDown(MergeBuilder(bucket).chain(0, 1).source_only(2).create()); _topLinks[2]->waitForMessage(MessageType::MERGEBUCKET, _messageWaitTime); CPPUNIT_ASSERT_EQUAL(uint64_t(0), _throttlers[0]->getMetrics().bounced_due_to_back_pressure.getValue()); } +void MergeThrottlerTest::fill_throttler_queue_with_n_commands(uint16_t throttler_index, size_t queued_count) { + std::size_t max_pending = _throttlers[throttler_index]->getThrottlePolicy().getMaxPendingCount(); + for (std::size_t i = 0; i < max_pending + queued_count; ++i) { + _topLinks[throttler_index]->sendDown(MergeBuilder(document::BucketId(16, i)).create()); + } + + // Wait till we have max_pending merge forwards and queued_count enqueued. + _topLinks[throttler_index]->waitForMessages(max_pending, _messageWaitTime); + waitUntilMergeQueueIs(*_throttlers[throttler_index], queued_count, _messageWaitTime); +} + +void MergeThrottlerTest::backpressure_evicts_all_queued_merges() { + _servers[0]->getClock().setAbsoluteTimeInSeconds(1000); + + fill_throttler_queue_with_n_commands(0, 1); + _topLinks[0]->getRepliesOnce(); // Clear all forwarded merges + _throttlers[0]->apply_timed_backpressure(); + + _topLinks[0]->waitForMessage(MessageType::MERGEBUCKET_REPLY, _messageWaitTime); + auto reply = _topLinks[0]->getAndRemoveMessage(MessageType::MERGEBUCKET_REPLY); + CPPUNIT_ASSERT_EQUAL(ReturnCode::BUSY, dynamic_cast<const MergeBucketReply&>(*reply).getResult().getResult()); +} + // TODO test message queue aborting (use rendezvous functionality--make guard) } // namespace storage diff --git a/storage/src/vespa/storage/frameworkimpl/status/statuswebserver.cpp b/storage/src/vespa/storage/frameworkimpl/status/statuswebserver.cpp index 55de1d9ee87..483cffaf751 100644 --- a/storage/src/vespa/storage/frameworkimpl/status/statuswebserver.cpp +++ b/storage/src/vespa/storage/frameworkimpl/status/statuswebserver.cpp @@ -56,7 +56,7 @@ void StatusWebServer::configure(std::unique_ptr<vespa::config::content::core::St int newPort = config->httpport; // If server is already running, ignore config updates that doesn't // alter port, or suggests random port. - if (_httpServer.get() != 0) { + if (_httpServer) { if (newPort == 0 || newPort == _port) return; } // Try to create new server before destroying old. @@ -88,21 +88,23 @@ void StatusWebServer::configure(std::unique_ptr<vespa::config::content::core::St if (!started) { std::ostringstream ost; ost << "Failed to start status HTTP server using port " << newPort << "."; - if (_httpServer.get() != 0) { + if (_httpServer) { ost << " Status server still running on port " << _port << " instead of suggested port " << newPort; } LOG(fatal, "%s.", ost.str().c_str()); _component->requestShutdown(ost.str()); + _httpServer.reset(); + return; } // Now that we know config update went well, update internal state _port = server->getListenPort(); LOG(config, "Status pages now available on port %u", _port); - if (_httpServer.get() != 0) { + if (_httpServer) { LOG(debug, "Shutting down old status server."); - _httpServer.reset(0); + _httpServer.reset(); LOG(debug, "Done shutting down old status server."); } - } else if (_httpServer.get() != 0) { + } else if (_httpServer) { LOG(info, "No longer running status server as negative port was given " "in config, indicating not to run a server."); } diff --git a/storage/src/vespa/storage/storageserver/mergethrottler.cpp b/storage/src/vespa/storage/storageserver/mergethrottler.cpp index 483992559e7..943059c00fc 100644 --- a/storage/src/vespa/storage/storageserver/mergethrottler.cpp +++ b/storage/src/vespa/storage/storageserver/mergethrottler.cpp @@ -520,13 +520,11 @@ MergeThrottler::rejectOutdatedQueuedMerges( uint32_t rejectLessThanVersion) { // Flush all queued merges that have an outdated version - MergePriorityQueue::iterator queueEnd = _queue.end(); - for (MergePriorityQueue::iterator i = _queue.begin(); i != queueEnd;) { - MergePriorityQueue::iterator erase_iter = i; + auto queueEnd = _queue.end(); + for (auto i = _queue.begin(); i != queueEnd;) { + auto erase_iter = i; ++i; - if (rejectMergeIfOutdated( - erase_iter->_msg, rejectLessThanVersion, msgGuard)) - { + if (rejectMergeIfOutdated(erase_iter->_msg, rejectLessThanVersion, msgGuard)){ _queue.erase(erase_iter); } } @@ -687,8 +685,17 @@ void MergeThrottler::bounce_backpressure_throttled_merge(const api::MergeBucketC } void MergeThrottler::apply_timed_backpressure() { - vespalib::LockGuard lock(_stateLock); + MessageGuard msg_guard(_stateLock, *this); _throttle_until_time = _component.getClock().getMonotonicTime() + _backpressure_duration; + backpressure_bounce_all_queued_merges(msg_guard); +} + +void MergeThrottler::backpressure_bounce_all_queued_merges(MessageGuard& guard) { + for (auto& qm : _queue) { + auto& merge_cmd = dynamic_cast<api::MergeBucketCommand&>(*qm._msg); + bounce_backpressure_throttled_merge(merge_cmd, guard); + } + _queue.clear(); } bool MergeThrottler::backpressure_mode_active() const { diff --git a/storage/src/vespa/storage/storageserver/mergethrottler.h b/storage/src/vespa/storage/storageserver/mergethrottler.h index 070c2ef07c4..69fdfdc1b95 100644 --- a/storage/src/vespa/storage/storageserver/mergethrottler.h +++ b/storage/src/vespa/storage/storageserver/mergethrottler.h @@ -340,6 +340,7 @@ private: void bounce_backpressure_throttled_merge(const api::MergeBucketCommand& cmd, MessageGuard& guard); bool merge_has_this_node_as_source_only_node(const api::MergeBucketCommand& cmd) const; bool backpressure_mode_active_no_lock() const; + void backpressure_bounce_all_queued_merges(MessageGuard& guard); void sendReply(const api::MergeBucketCommand& cmd, const api::ReturnCode& result, diff --git a/storage/src/vespa/storage/storageserver/storagenode.cpp b/storage/src/vespa/storage/storageserver/storagenode.cpp index 27f98b478ce..85f54431cfb 100644 --- a/storage/src/vespa/storage/storageserver/storagenode.cpp +++ b/storage/src/vespa/storage/storageserver/storagenode.cpp @@ -103,13 +103,13 @@ StorageNode::subscribeToConfigs() void StorageNode::initialize() { + _context.getComponentRegister().registerShutdownListener(*this); + // Fetch configs needed first. These functions will just grab the config // and store them away, while having the config lock. subscribeToConfigs(); - _context.getMemoryManager().setMaximumMemoryUsage( - _serverConfig->memorytouse); - _context.getComponentRegister().registerShutdownListener(*this); + _context.getMemoryManager().setMaximumMemoryUsage(_serverConfig->memorytouse); updateUpgradeFlag(*_clusterConfig); // First update some basics that doesn't depend on anything else to be diff --git a/storage/src/vespa/storage/storageserver/storagenode.h b/storage/src/vespa/storage/storageserver/storagenode.h index 4c4918e6977..753f3c85330 100644 --- a/storage/src/vespa/storage/storageserver/storagenode.h +++ b/storage/src/vespa/storage/storageserver/storagenode.h @@ -111,7 +111,7 @@ private: // First components that doesn't depend on others std::unique_ptr<StatusWebServer> _statusWebServer; - std::shared_ptr<StorageMetricSet> _metrics; + std::shared_ptr<StorageMetricSet> _metrics; std::unique_ptr<metrics::MetricManager> _metricManager; // Depends on bucket databases and stop() functionality @@ -122,8 +122,7 @@ private: std::unique_ptr<StatusMetricConsumer> _statusMetrics; // Depends on metric manager std::unique_ptr<StateReporter> _stateReporter; - - std::unique_ptr<StateManager> _stateManager; + std::unique_ptr<StateManager> _stateManager; // The storage chain can depend on anything. std::unique_ptr<StorageLink> _chain; diff --git a/storageframework/src/vespa/storageframework/defaultimplementation/component/componentregisterimpl.cpp b/storageframework/src/vespa/storageframework/defaultimplementation/component/componentregisterimpl.cpp index cda2fee4628..a03f79b008d 100644 --- a/storageframework/src/vespa/storageframework/defaultimplementation/component/componentregisterimpl.cpp +++ b/storageframework/src/vespa/storageframework/defaultimplementation/component/componentregisterimpl.cpp @@ -5,9 +5,7 @@ #include <vespa/metrics/metricmanager.h> #include <vespa/vespalib/util/exceptions.h> -namespace storage { -namespace framework { -namespace defaultimplementation { +namespace storage::framework::defaultimplementation { ComponentRegisterImpl::ComponentRegisterImpl() : _componentLock(), @@ -184,6 +182,4 @@ ComponentRegisterImpl::registerShutdownListener(ShutdownListener& listener) _shutdownListener = &listener; } -} // defaultimplementation -} // framework -} // storage +} diff --git a/storageframework/src/vespa/storageframework/generic/component/component.cpp b/storageframework/src/vespa/storageframework/generic/component/component.cpp index 6457ed73795..a35cad68b00 100644 --- a/storageframework/src/vespa/storageframework/generic/component/component.cpp +++ b/storageframework/src/vespa/storageframework/generic/component/component.cpp @@ -4,10 +4,8 @@ #include "componentregister.h" #include <vespa/storageframework/generic/metric/metricregistrator.h> #include <vespa/storageframework/generic/thread/threadpool.h> -#include <vespa/vespalib/util/sync.h> -namespace storage { -namespace framework { +namespace storage::framework { void Component::open() @@ -137,6 +135,4 @@ Component::requestShutdown(vespalib::stringref reason) _componentRegister->requestShutdown(reason); } -} // framework -} // storage - +} diff --git a/storageframework/src/vespa/storageframework/generic/component/componentregister.h b/storageframework/src/vespa/storageframework/generic/component/componentregister.h index 521e23ba1aa..3babd8b29bc 100644 --- a/storageframework/src/vespa/storageframework/generic/component/componentregister.h +++ b/storageframework/src/vespa/storageframework/generic/component/componentregister.h @@ -13,8 +13,7 @@ #include <vespa/vespalib/stllike/string.h> -namespace storage { -namespace framework { +namespace storage::framework { class ManagedComponent; @@ -25,6 +24,4 @@ struct ComponentRegister { virtual void requestShutdown(vespalib::stringref reason) = 0; }; -} // framework -} // storage - +} diff --git a/storageserver/src/vespa/storageserver/app/process.h b/storageserver/src/vespa/storageserver/app/process.h index 030b51f929e..bbffca30593 100644 --- a/storageserver/src/vespa/storageserver/app/process.h +++ b/storageserver/src/vespa/storageserver/app/process.h @@ -19,7 +19,6 @@ #include <vespa/config/subscription/configuri.h> #include <vespa/config/subscription/configsubscriber.h> - namespace document { class DocumentTypeRepo; } namespace storage { diff --git a/vespaclient-java/pom.xml b/vespaclient-java/pom.xml index bb8ed9387fc..8cde41d53d7 100644 --- a/vespaclient-java/pom.xml +++ b/vespaclient-java/pom.xml @@ -33,6 +33,7 @@ <version>${project.version}</version> </dependency> <dependency> + <!-- TODO: remove dependency on container-dev, and instead depend directly on what this module needs! --> <groupId>com.yahoo.vespa</groupId> <artifactId>container-dev</artifactId> <version>${project.version}</version> @@ -44,10 +45,20 @@ </dependency> <dependency> <groupId>com.yahoo.vespa</groupId> + <artifactId>jdisc_http_service</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> <artifactId>predicate-search-core</artifactId> <version>${project.version}</version> </dependency> <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>vespajlib</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> <groupId>commons-cli</groupId> <artifactId>commons-cli</artifactId> </dependency> diff --git a/vespajlib/src/main/java/com/yahoo/net/HostName.java b/vespajlib/src/main/java/com/yahoo/net/HostName.java index 00e63516459..4c51a5e1ba1 100644 --- a/vespajlib/src/main/java/com/yahoo/net/HostName.java +++ b/vespajlib/src/main/java/com/yahoo/net/HostName.java @@ -19,7 +19,7 @@ import java.util.logging.Logger; /** * Utilities for getting the hostname of the system running the JVM. * - * @author lulf + * @author Ulf Lilleengen * @author bratseth * @author hakon */ |