diff options
32 files changed, 292 insertions, 147 deletions
diff --git a/.travis.yml b/.travis.yml index 20937176541..b44418606e1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -32,7 +32,5 @@ matrix: if: type = pull_request - env: BUILD_TYPE="cpp" if: type = pull_request - - env: BUILD_TYPE="install" - if: type = pull_request - env: BUILD_TYPE="full" if: NOT type = pull_request diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java index e338bc17788..c293e00ae48 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java @@ -241,6 +241,8 @@ public class Controller extends AbstractComponent { return chefClient; } + public CuratorDb curator() { return curator; } + private String printableVersion(Optional<VespaVersion> vespaVersion) { return vespaVersion.map(v -> v.versionNumber().toFullString()).orElse("Unknown"); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainer.java index d9ef451ffb7..7a771464957 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainer.java @@ -7,7 +7,9 @@ import com.yahoo.vespa.hosted.controller.api.integration.MetricsService; import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.vespa.hosted.controller.application.DeploymentMetrics; +import java.io.UncheckedIOException; import java.time.Duration; +import java.util.logging.Logger; /** * Retrieve deployment metrics like qps and document count from the metric service and @@ -17,6 +19,8 @@ import java.time.Duration; */ public class DeploymentMetricsMaintainer extends Maintainer { + private static final Logger log = Logger.getLogger(DeploymentMetricsMaintainer.class.getName()); + DeploymentMetricsMaintainer(Controller controller, Duration duration, JobControl jobControl) { super(controller, duration, jobControl); } @@ -25,19 +29,32 @@ public class DeploymentMetricsMaintainer extends Maintainer { protected void maintain() { for (Application application : controller().applications().asList()) { - try (Lock lock = controller().applications().lock(application.id())) { - for (Deployment deployment : application.deployments().values()) { - + for (Deployment deployment : application.deployments().values()) { + try { MetricsService.DeploymentMetrics metrics = controller().metricsService() .getDeploymentMetrics(application.id(), deployment.zone()); - DeploymentMetrics appMetrics = new DeploymentMetrics(metrics.queriesPerSecond(), metrics.writesPerSecond(), - metrics.documentCount(), metrics.queryLatencyMillis(), metrics.writeLatencyMillis()); - - Application app = application.with(deployment.withMetrics(appMetrics)); - controller().applications().store(app, lock); + metrics.documentCount(), metrics.queryLatencyMillis(), metrics.writeLatencyMillis()); + + // Avoid locking for a long time, due to slow YAMAS. + try (Lock lock = controller().applications().lock(application.id())) { + // Deployment (or application) may have changed (or be gone) now: + controller().applications().get(application.id()).ifPresent(freshApplication -> { + Deployment freshDeployment = freshApplication.deployments().get(deployment.zone()); + if (freshDeployment != null) + controller().applications().store(freshApplication.with(freshDeployment.withMetrics(appMetrics)), lock); + }); + } + } + catch (UncheckedIOException e) { + log.warning("Timed out talking to YAMAS; retrying in " + maintenanceInterval() + ":\n" + e); } + } + } + } + } + diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java index 0722a58e18d..44b15053a8c 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java @@ -7,13 +7,11 @@ import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.application.ApplicationList; import com.yahoo.vespa.hosted.controller.application.Change; -import com.yahoo.vespa.hosted.controller.deployment.BuildSystem; import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; import com.yahoo.vespa.hosted.controller.versions.VespaVersion; import com.yahoo.yolean.Exceptions; import java.time.Duration; -import java.time.Instant; import java.util.logging.Level; import java.util.logging.Logger; @@ -111,4 +109,18 @@ public class Upgrader extends Maintainer { curator.writeUpgradesPerMinute(n); } + /** + * Returns whether to ignore confidence calculations when upgrading + */ + public boolean ignoreConfidence() { + return curator.readIgnoreConfidence(); + } + + /** + * Controls whether to ignore confidence calculations or not + */ + public void ignoreConfidence(boolean value) { + curator.writeIgnoreConfidence(value); + } + } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java index 03b4dd6efe3..0a288955055 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java @@ -212,6 +212,20 @@ public class CuratorDb { transaction.commit(); } + public boolean readIgnoreConfidence() { + Optional<byte[]> value = curator.getData(ignoreConfidencePath()); + if (! value.isPresent() || value.get().length == 0) { + return false; // Default if value has never been written + } + return ByteBuffer.wrap(value.get()).getInt() == 1; + } + + public void writeIgnoreConfidence(boolean value) { + NestedTransaction transaction = new NestedTransaction(); + curator.set(ignoreConfidencePath(), ByteBuffer.allocate(Integer.BYTES).putInt(value ? 1 : 0).array()); + transaction.commit(); + } + public void writeVersionStatus(VersionStatus status) { VersionStatusSerializer serializer = new VersionStatusSerializer(); NestedTransaction transaction = new NestedTransaction(); @@ -305,6 +319,10 @@ public class CuratorDb { return root.append("upgrader").append("upgradesPerMinute"); } + private Path ignoreConfidencePath() { + return root.append("upgrader").append("ignoreConfidence"); + } + private Path versionStatusPath() { return root.append("versionStatus"); } private Path provisionStatePath() { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiHandler.java index 03ac073a34a..162827cdb99 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiHandler.java @@ -7,9 +7,10 @@ import com.yahoo.container.jdisc.LoggingRequestHandler; import com.yahoo.container.logging.AccessLog; import com.yahoo.io.IOUtils; import com.yahoo.slime.Inspector; -import com.yahoo.slime.Slime; +import com.yahoo.text.Utf8; import com.yahoo.vespa.config.SlimeUtils; import com.yahoo.vespa.hosted.controller.maintenance.ControllerMaintenance; +import com.yahoo.vespa.hosted.controller.maintenance.Upgrader; import com.yahoo.vespa.hosted.controller.restapi.ErrorResponse; import com.yahoo.vespa.hosted.controller.restapi.MessageResponse; import com.yahoo.vespa.hosted.controller.restapi.Path; @@ -62,7 +63,7 @@ public class ControllerApiHandler extends LoggingRequestHandler { Path path = new Path(request.getUri().getPath()); if (path.matches("/controller/v1/")) return root(request); if (path.matches("/controller/v1/maintenance/")) return new JobsResponse(maintenance.jobControl()); - if (path.matches("/controller/v1/jobs/upgrader")) return new UpgraderResponse(maintenance.upgrader().upgradesPerMinute()); + if (path.matches("/controller/v1/jobs/upgrader")) return new UpgraderResponse(maintenance.upgrader()); return notFound(path); } @@ -101,18 +102,26 @@ public class ControllerApiHandler extends LoggingRequestHandler { private HttpResponse configureUpgrader(HttpRequest request) { String upgradesPerMinuteField = "upgradesPerMinute"; - Slime slime = toSlime(request.getData()); - Inspector inspect = slime.get(); + String ignoreConfidenceField = "ignoreConfidence"; + + byte[] jsonBytes = toJsonBytes(request.getData()); + Inspector inspect = SlimeUtils.jsonToSlime(jsonBytes).get(); + Upgrader upgrader = maintenance.upgrader(); if (inspect.field(upgradesPerMinuteField).valid()) { - maintenance.upgrader().setUpgradesPerMinute(inspect.field(upgradesPerMinuteField).asDouble()); + upgrader.setUpgradesPerMinute(inspect.field(upgradesPerMinuteField).asDouble()); + } else if (inspect.field(ignoreConfidenceField).valid()) { + upgrader.ignoreConfidence(inspect.field(ignoreConfidenceField).asBool()); + } else { + return ErrorResponse.badRequest("Unable to configure upgrader with data in request: '" + + Utf8.toString(jsonBytes) + "'"); } - return new UpgraderResponse(maintenance.upgrader().upgradesPerMinute()); + + return new UpgraderResponse(maintenance.upgrader()); } - private Slime toSlime(InputStream jsonStream) { + private byte[] toJsonBytes(InputStream jsonStream) { try { - byte[] jsonBytes = IOUtils.readBytes(jsonStream, 1000 * 1000); - return SlimeUtils.jsonToSlime(jsonBytes); + return IOUtils.readBytes(jsonStream, 1000 * 1000); } catch (IOException e) { throw new UncheckedIOException(e); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/UpgraderResponse.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/UpgraderResponse.java index fe88a0f1f22..3444f710d4c 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/UpgraderResponse.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/UpgraderResponse.java @@ -4,6 +4,7 @@ import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.slime.Cursor; import com.yahoo.slime.JsonFormat; import com.yahoo.slime.Slime; +import com.yahoo.vespa.hosted.controller.maintenance.Upgrader; import java.io.IOException; import java.io.OutputStream; @@ -13,18 +14,19 @@ import java.io.OutputStream; */ public class UpgraderResponse extends HttpResponse { - private final double upgradesPerMinute; + private final Upgrader upgrader; - public UpgraderResponse(double upgradesPerMinute) { + public UpgraderResponse(Upgrader upgrader) { super(200); - this.upgradesPerMinute = upgradesPerMinute; + this.upgrader = upgrader; } @Override public void render(OutputStream outputStream) throws IOException { Slime slime = new Slime(); Cursor root = slime.setObject(); - root.setDouble("upgradesPerMinute", upgradesPerMinute); + root.setDouble("upgradesPerMinute", upgrader.upgradesPerMinute()); + root.setBool("ignoreConfidence", upgrader.ignoreConfidence()); new JsonFormat(true).encode(outputStream, slime); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VespaVersion.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VespaVersion.java index c1b9c045fbe..1541e4d35f8 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VespaVersion.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VespaVersion.java @@ -6,6 +6,7 @@ import com.yahoo.component.Version; import com.yahoo.component.Vtag; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.application.ApplicationList; +import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; import java.time.Instant; import java.util.Collection; @@ -57,8 +58,7 @@ public class VespaVersion implements Comparable<VespaVersion> { return Confidence.broken; // 'broken' if 4 non-canary was broken by this, and that is at least 10% of all - int brokenByThisVersion = failingOnThis.without(UpgradePolicy.canary).startedFailingAfter(releasedAt).size(); - if (brokenByThisVersion >= 4 && brokenByThisVersion >= productionOnThis.size() * 0.1) + if (nonCanaryApplicationsBroken(failingOnThis, productionOnThis, releasedAt, controller.curator())) return Confidence.broken; // 'low' unless all canary applications are upgraded @@ -136,4 +136,17 @@ public class VespaVersion implements Comparable<VespaVersion> { } + private static boolean nonCanaryApplicationsBroken(ApplicationList failingOnThis, + ApplicationList productionOnThis, + Instant releasedAt, + CuratorDb curator) { + ApplicationList failingNonCanaries = failingOnThis.without(UpgradePolicy.canary).startedFailingAfter(releasedAt); + ApplicationList productionNonCanaries = productionOnThis.without(UpgradePolicy.canary); + + if (productionNonCanaries.size() + failingNonCanaries.size() == 0 || curator.readIgnoreConfidence()) return false; + + // 'broken' if 4 non-canary was broken by this, and that is at least 10% of all + int brokenByThisVersion = failingNonCanaries.size(); + return brokenByThisVersion >= 4 && brokenByThisVersion >= productionOnThis.size() * 0.1; + } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiTest.java index e1c5cdb7742..8047b0d48c9 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiTest.java @@ -7,6 +7,7 @@ import com.yahoo.vespa.hosted.controller.restapi.ControllerContainerTest; import org.junit.Test; import java.io.File; +import java.io.IOException; /** * @author bratseth @@ -42,7 +43,7 @@ public class ControllerApiTest extends ControllerContainerTest { // Get current configuration tester.assertResponse(new Request("http://localhost:8080/controller/v1/jobs/upgrader"), - "{\"upgradesPerMinute\":0.5}", + "{\"upgradesPerMinute\":0.5,\"ignoreConfidence\":false}", 200); // Set invalid configuration @@ -51,16 +52,22 @@ public class ControllerApiTest extends ControllerContainerTest { "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Upgrades per minute must be >= 0\"}", 400); - // Unrecognized fields are ignored + // Unrecognized field tester.assertResponse(new Request("http://localhost:8080/controller/v1/jobs/upgrader", "{\"foo\":bar}", Request.Method.PATCH), - "{\"upgradesPerMinute\":0.5}", + "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Unable to configure upgrader with data in request: '{\\\"foo\\\":bar}'\"}", + + 400); + + // Patch configuration + tester.assertResponse(new Request("http://localhost:8080/controller/v1/jobs/upgrader", + "{\"upgradesPerMinute\":42.0}", Request.Method.PATCH), + "{\"upgradesPerMinute\":42.0,\"ignoreConfidence\":false}", 200); - // Set configuration + // Patch configuration tester.assertResponse(new Request("http://localhost:8080/controller/v1/jobs/upgrader", - "{\"upgradesPerMinute\":42}", Request.Method.PATCH), - "{\"upgradesPerMinute\":42.0}", + "{\"ignoreConfidence\":true}", Request.Method.PATCH), + "{\"upgradesPerMinute\":42.0,\"ignoreConfidence\":true}", 200); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/VersionStatusTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/VersionStatusTest.java index 17935906186..bc31079cfe0 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/VersionStatusTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/VersionStatusTest.java @@ -233,6 +233,49 @@ public class VersionStatusTest { VespaVersion.Confidence.broken, confidence(tester.controller(), version3)); } + + @Test + public void testIgnoreConfigdeince() { + DeploymentTester tester = new DeploymentTester(); + + Version version0 = new Version("5.0"); + tester.upgradeSystem(version0); + + // Setup applications - all running on version0 + Application canary0 = tester.createAndDeploy("canary0", 1, "canary"); + Application canary1 = tester.createAndDeploy("canary1", 2, "canary"); + Application default0 = tester.createAndDeploy("default0", 3, "default"); + Application default1 = tester.createAndDeploy("default1", 4, "default"); + Application default2 = tester.createAndDeploy("default2", 5, "default"); + Application default3 = tester.createAndDeploy("default3", 6, "default"); + Application default4 = tester.createAndDeploy("default4", 7, "default"); + + // New version is released + Version version1 = new Version("5.1"); + tester.upgradeSystem(version1); + + // All canaries upgrade successfully, 1 default apps ok, 3 default apps fail + tester.completeUpgrade(canary0, version1, "canary"); + tester.completeUpgrade(canary1, version1, "canary"); + tester.upgradeSystem(version1); + tester.completeUpgrade(default0, version1, "default"); + tester.completeUpgradeWithError(default1, version1, "default", stagingTest); + tester.completeUpgradeWithError(default2, version1, "default", stagingTest); + tester.completeUpgradeWithError(default3, version1, "default", stagingTest); + tester.completeUpgradeWithError(default4, version1, "default", stagingTest); + tester.updateVersionStatus(); + + assertEquals("Canaries have upgraded, 1 of 4 default apps failing: Broken", + Confidence.broken, confidence(tester.controller(), version1)); + + // Same as above, but ignore confidence calculations, will force normal confidence + tester.controllerTester().curator().writeIgnoreConfidence(true); + tester.updateVersionStatus(); + assertEquals("Canaries have upgraded, 1 of 4 default apps failing, but confidence ignored: Low", + Confidence.normal, confidence(tester.controller(), version1)); + tester.controllerTester().curator().writeIgnoreConfidence(false); + } + @Test public void testComputeIgnoresVersionWithUnknownGitMetadata() { ControllerTester tester = new ControllerTester(); diff --git a/searchcore/src/tests/proton/docsummary/docsummary.cpp b/searchcore/src/tests/proton/docsummary/docsummary.cpp index 3b199d266a8..bce3fb7267c 100644 --- a/searchcore/src/tests/proton/docsummary/docsummary.cpp +++ b/searchcore/src/tests/proton/docsummary/docsummary.cpp @@ -21,9 +21,8 @@ #include <vespa/searchcore/proton/server/memoryconfigstore.h> #include <vespa/searchcore/proton/server/searchview.h> #include <vespa/searchcore/proton/server/summaryadapter.h> -#include <vespa/searchlib/common/idestructorcallback.h> +#include <vespa/searchlib/common/gatecallback.h> #include <vespa/searchlib/common/transport.h> -#include <vespa/searchlib/docstore/logdocumentstore.h> #include <vespa/searchlib/engine/docsumapi.h> #include <vespa/searchlib/index/docbuilder.h> #include <vespa/searchlib/index/dummyfileheadercontext.h> @@ -254,7 +253,7 @@ public: op.setSerialNum(serialNum); op.setDbDocumentId(dbdId); op.setPrevDbDocumentId(prevDbdId); - _ddb->getFeedHandler().storeOperation(op); + _ddb->getFeedHandler().storeOperation(op, std::make_shared<search::IgnoreCallback>()); SearchView *sv(dynamic_cast<SearchView *>(_ddb->getReadySubDB()->getSearchView().get())); if (sv != NULL) { // cf. FeedView::putAttributes() diff --git a/searchcore/src/tests/proton/documentdb/feedhandler/feedhandler_test.cpp b/searchcore/src/tests/proton/documentdb/feedhandler/feedhandler_test.cpp index 823c31dd1c2..b8ffc41d3cd 100644 --- a/searchcore/src/tests/proton/documentdb/feedhandler/feedhandler_test.cpp +++ b/searchcore/src/tests/proton/documentdb/feedhandler/feedhandler_test.cpp @@ -370,7 +370,7 @@ struct MyTlsWriter : TlsWriter { bool erase_return; MyTlsWriter() : store_count(0), erase_count(0), erase_return(true) {} - void storeOperation(const FeedOperation &) override { ++store_count; } + void storeOperation(const FeedOperation &, DoneCallback) override { ++store_count; } bool erase(SerialNum) override { ++erase_count; return erase_return; } SerialNum sync(SerialNum syncTo) override { diff --git a/searchcore/src/tests/proton/documentdb/lid_space_compaction/lid_space_compaction_test.cpp b/searchcore/src/tests/proton/documentdb/lid_space_compaction/lid_space_compaction_test.cpp index 55f71da9687..56bd99c90f6 100644 --- a/searchcore/src/tests/proton/documentdb/lid_space_compaction/lid_space_compaction_test.cpp +++ b/searchcore/src/tests/proton/documentdb/lid_space_compaction/lid_space_compaction_test.cpp @@ -138,7 +138,7 @@ struct MyStorer : public IOperationStorer : _moveCnt(0), _compactCnt(0) {} - virtual void storeOperation(FeedOperation &op) override { + void storeOperation(const FeedOperation &op, DoneCallback) override { if (op.getType() == FeedOperation::MOVE) { ++ _moveCnt; } else if (op.getType() == FeedOperation::COMPACT_LID_SPACE) { diff --git a/searchcore/src/tests/proton/documentdb/maintenancecontroller/maintenancecontroller_test.cpp b/searchcore/src/tests/proton/documentdb/maintenancecontroller/maintenancecontroller_test.cpp index 559dbb240a8..f20ad01bcf6 100644 --- a/searchcore/src/tests/proton/documentdb/maintenancecontroller/maintenancecontroller_test.cpp +++ b/searchcore/src/tests/proton/documentdb/maintenancecontroller/maintenancecontroller_test.cpp @@ -26,7 +26,7 @@ #include <vespa/searchcore/proton/test/test.h> #include <vespa/searchlib/attribute/attributecontext.h> #include <vespa/searchlib/attribute/attributeguard.h> -#include <vespa/searchlib/common/idestructorcallback.h> +#include <vespa/searchlib/common/gatecallback.h> #include <vespa/searchlib/common/idocumentmetastore.h> #include <vespa/searchlib/index/docbuilder.h> #include <vespa/vespalib/data/slime/slime.h> @@ -232,7 +232,7 @@ public: } // Implements IOperationStorer - virtual void storeOperation(FeedOperation &op) override; + virtual void storeOperation(const FeedOperation &op, DoneCallback) override; uint32_t getHeartBeats() { return _heartBeats; @@ -781,7 +781,6 @@ MyFeedHandler::isExecutorThread() void MyFeedHandler::handleMove(MoveOperation &op, IDestructorCallback::SP moveDoneCtx) { - (void) moveDoneCtx; assert(isExecutorThread()); assert(op.getValidPrevDbdId()); _subDBs[op.getSubDbId()]->prepareMove(op); @@ -792,7 +791,7 @@ MyFeedHandler::handleMove(MoveOperation &op, IDestructorCallback::SP moveDoneCtx assert(op.getPrevSubDbId() != 1u); assert(op.getSubDbId() < _subDBs.size()); assert(op.getPrevSubDbId() < _subDBs.size()); - storeOperation(op); + storeOperation(op, std::move(moveDoneCtx)); _subDBs[op.getSubDbId()]->handleMove(op); _subDBs[op.getPrevSubDbId()]->handleMove(op); } @@ -803,7 +802,7 @@ MyFeedHandler::performPruneRemovedDocuments(PruneRemovedDocumentsOperation &op) { assert(isExecutorThread()); if (op.getLidsToRemove()->getNumLids() != 0u) { - storeOperation(op); + storeOperation(op, std::make_shared<search::IgnoreCallback>()); // magic number. _subDBs[1u]->handlePruneRemovedDocuments(op); } @@ -826,9 +825,9 @@ MyFeedHandler::setSubDBs(const std::vector<MyDocumentSubDB *> &subDBs) void -MyFeedHandler::storeOperation(FeedOperation &op) +MyFeedHandler::storeOperation(const FeedOperation &op, DoneCallback) { - op.setSerialNum(incSerialNum()); + const_cast<FeedOperation &>(op).setSerialNum(incSerialNum()); } @@ -1011,22 +1010,16 @@ MaintenanceControllerFixture::performForwardMaintenanceConfig() void -MaintenanceControllerFixture::insertDocs(const test::UserDocuments &docs, - MyDocumentSubDB &subDb) +MaintenanceControllerFixture::insertDocs(const test::UserDocuments &docs, MyDocumentSubDB &subDb) { - for (test::UserDocuments::Iterator itr = docs.begin(); - itr != docs.end(); - ++itr) { + for (auto itr = docs.begin(); itr != docs.end(); ++itr) { const test::BucketDocuments &bucketDocs = itr->second; for (size_t i = 0; i < bucketDocs.getDocs().size(); ++i) { const test::Document &testDoc = bucketDocs.getDocs()[i]; - PutOperation op(testDoc.getBucket(), - testDoc.getTimestamp(), - testDoc.getDoc()); - op.setDbDocumentId(DbDocumentId(subDb.getSubDBId(), - testDoc.getLid())); - _fh.storeOperation(op); + PutOperation op(testDoc.getBucket(), testDoc.getTimestamp(), testDoc.getDoc()); + op.setDbDocumentId(DbDocumentId(subDb.getSubDBId(), testDoc.getLid())); + _fh.storeOperation(op, std::make_shared<search::IgnoreCallback>()); subDb.handlePut(op); } } @@ -1038,18 +1031,13 @@ MaintenanceControllerFixture::removeDocs(const test::UserDocuments &docs, Timestamp timestamp) { - for (test::UserDocuments::Iterator itr = docs.begin(); - itr != docs.end(); - ++itr) { + for (auto itr = docs.begin(); itr != docs.end(); ++itr) { const test::BucketDocuments &bucketDocs = itr->second; for (size_t i = 0; i < bucketDocs.getDocs().size(); ++i) { const test::Document &testDoc = bucketDocs.getDocs()[i]; - RemoveOperation op(testDoc.getBucket(), - timestamp, - testDoc.getDoc()->getId()); - op.setDbDocumentId(DbDocumentId(_removed.getSubDBId(), - testDoc.getLid())); - _fh.storeOperation(op); + RemoveOperation op(testDoc.getBucket(), timestamp, testDoc.getDoc()->getId()); + op.setDbDocumentId(DbDocumentId(_removed.getSubDBId(), testDoc.getLid())); + _fh.storeOperation(op, std::make_shared<search::IgnoreCallback>()); _removed.handleRemove(op); } } diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp b/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp index 4198803d1fe..5babacfc4b6 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp @@ -307,7 +307,7 @@ DocumentDB::enterReprocessState() if (!runner.empty()) { runner.run(); NoopOperation op; - _feedHandler.storeOperation(op); + _feedHandler.storeOperationSync(op); sync(op.getSerialNum()); _subDBs.pruneRemovedFields(op.getSerialNum()); } @@ -397,15 +397,14 @@ DocumentDB::applyConfig(DocumentDBConfig::SP configSnapshot, SerialNum serialNum _config_store->saveConfig(*configSnapshot, serialNum); // save entry in transaction log NewConfigOperation op(serialNum, *_config_store); - _feedHandler.storeOperation(op); + _feedHandler.storeOperationSync(op); sync(op.getSerialNum()); } bool hasVisibilityDelayChanged = false; { bool elidedConfigSave = equalReplayConfig && tlsReplayDone; // Flush changes to attributes and memory index, cf. visibilityDelay - _feedView.get()->forceCommit(elidedConfigSave ? serialNum : - serialNum - 1); + _feedView.get()->forceCommit(elidedConfigSave ? serialNum : serialNum - 1); _writeService.sync(); fastos::TimeStamp visibilityDelay = configSnapshot->getMaintenanceConfigSP()->getVisibilityDelay(); hasVisibilityDelayChanged = (visibilityDelay != _visibility.getVisibilityDelay()); @@ -585,7 +584,7 @@ DocumentDB::saveInitialConfig(const DocumentDBConfig &configSnapshot) // pruned at once anyway. // save noop entry in transaction log NoopOperation op; - _feedHandler.storeOperation(op); + _feedHandler.storeOperationSync(op); sync(op.getSerialNum()); // Wipe everything in transaction log before initial config. try { @@ -609,7 +608,7 @@ DocumentDB::resumeSaveConfig() SerialNum confSerial = _feedHandler.incSerialNum(); // resume operation, i.e. save config entry in transaction log NewConfigOperation op(confSerial, *_config_store); - _feedHandler.storeOperation(op); + _feedHandler.storeOperationSync(op); sync(op.getSerialNum()); } @@ -776,7 +775,7 @@ DocumentDB::enterRedoReprocessState() runner.run(); _subDBs.onReprocessDone(_feedHandler.getSerialNum()); NoopOperation op; - _feedHandler.storeOperation(op); + _feedHandler.storeOperationSync(op); sync(op.getSerialNum()); _subDBs.pruneRemovedFields(op.getSerialNum()); } diff --git a/searchcore/src/vespa/searchcore/proton/server/feedhandler.cpp b/searchcore/src/vespa/searchcore/proton/server/feedhandler.cpp index b01ba43cb49..60e604e1b23 100644 --- a/searchcore/src/vespa/searchcore/proton/server/feedhandler.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/feedhandler.cpp @@ -12,7 +12,7 @@ #include <vespa/searchcore/proton/persistenceengine/i_resource_write_filter.h> #include <vespa/searchcore/proton/persistenceengine/transport_latch.h> #include <vespa/searchcorespi/index/ithreadingservice.h> -#include <vespa/searchlib/common/idestructorcallback.h> +#include <vespa/searchlib/common/gatecallback.h> #include <vespa/vespalib/util/exceptions.h> #include <unistd.h> @@ -46,8 +46,8 @@ ignoreOperation(const DocumentOperation &op) { } // namespace -void FeedHandler::TlsMgrWriter::storeOperation(const FeedOperation &op) { - TlcProxy(_tls_mgr.getDomainName(), *_tlsDirectWriter).storeOperation(op); +void FeedHandler::TlsMgrWriter::storeOperation(const FeedOperation &op, DoneCallback onDone) { + TlcProxy(_tls_mgr.getDomainName(), *_tlsDirectWriter).storeOperation(op, std::move(onDone)); } bool FeedHandler::TlsMgrWriter::erase(SerialNum oldest_to_keep) { return _tls_mgr.getSession()->erase(oldest_to_keep); @@ -72,7 +72,6 @@ FeedHandler::TlsMgrWriter::sync(SerialNum syncTo) LOG(spam, "Tls sync incomplete, reached %" PRIu64 ", retrying", syncedTo); } throw IllegalStateException(make_string("Failed to sync TLS to token %" PRIu64 ".", syncTo)); - return 0; } void @@ -94,7 +93,7 @@ void FeedHandler::performPut(FeedToken token, PutOperation &op) { } return; } - storeOperation(op); + storeOperation(op, token); if (token) { token->setResult(std::make_unique<Result>(), false); } @@ -121,7 +120,7 @@ FeedHandler::performUpdate(FeedToken token, UpdateOperation &op) void FeedHandler::performInternalUpdate(FeedToken token, UpdateOperation &op) { - storeOperation(op); + storeOperation(op, token); if (token) { token->setResult(ResultUP(new UpdateResult(op.getPrevTimestamp())), true); } @@ -137,7 +136,7 @@ FeedHandler::createNonExistingDocument(FeedToken token, const UpdateOperation &o op.getUpdate()->applyTo(*doc); PutOperation putOp(op.getBucketId(), op.getTimestamp(), doc); _activeFeedView->preparePut(putOp); - storeOperation(putOp); + storeOperation(putOp, token); if (token) { token->setResult(ResultUP(new UpdateResult(putOp.getTimestamp())), true); } @@ -160,7 +159,7 @@ void FeedHandler::performRemove(FeedToken token, RemoveOperation &op) { if (op.getPrevDbDocumentId().valid()) { assert(op.getValidNewOrPrevDbdId()); assert(op.notMovingLidInSameSubDb()); - storeOperation(op); + storeOperation(op, token); if (token) { bool documentWasFound = !op.getPrevMarkedAsRemoved(); token->setResult(ResultUP(new RemoveResult(documentWasFound)), documentWasFound); @@ -168,7 +167,7 @@ void FeedHandler::performRemove(FeedToken token, RemoveOperation &op) { _activeFeedView->handleRemove(std::move(token), op); } else if (op.hasDocType()) { assert(op.getDocType() == _docTypeName.getName()); - storeOperation(op); + storeOperation(op, token); if (token) { token->setResult(ResultUP(new RemoveResult(false)), false); } @@ -186,20 +185,16 @@ FeedHandler::performGarbageCollect(FeedToken token) (void) token; } - void FeedHandler::performCreateBucket(FeedToken token, CreateBucketOperation &op) { - (void) token; - storeOperation(op); + storeOperation(op, std::move(token)); _bucketDBHandler->handleCreateBucket(op.getBucketId()); } - void FeedHandler::performDeleteBucket(FeedToken token, DeleteBucketOperation &op) { - (void) token; _activeFeedView->prepareDeleteBucket(op); - storeOperation(op); + storeOperation(op, std::move(token)); // Delete documents in bucket _activeFeedView->handleDeleteBucket(op); // Delete bucket itself, should no longer have documents. @@ -207,21 +202,16 @@ void FeedHandler::performDeleteBucket(FeedToken token, DeleteBucketOperation &op } - void FeedHandler::performSplit(FeedToken token, SplitBucketOperation &op) { - (void) token; - storeOperation(op); + storeOperation(op, std::move(token)); _bucketDBHandler->handleSplit(op.getSerialNum(), op.getSource(), op.getTarget1(), op.getTarget2()); } - void FeedHandler::performJoin(FeedToken token, JoinBucketsOperation &op) { - (void) token; - storeOperation(op); + storeOperation(op, std::move(token)); _bucketDBHandler->handleJoin(op.getSerialNum(), op.getSource1(), op.getSource2(), op.getTarget()); } - void FeedHandler::performSync() { @@ -412,18 +402,28 @@ FeedHandler::isDoingReplay() const { return _tlsMgr.isDoingReplay(); } -bool FeedHandler::getTransactionLogReplayDone() const { +bool +FeedHandler::getTransactionLogReplayDone() const { return _tlsMgr.getReplayDone(); } -void FeedHandler::storeOperation(FeedOperation &op) { +void +FeedHandler::storeOperation(const FeedOperation &op, TlsWriter::DoneCallback onDone) { if (!op.getSerialNum()) { - op.setSerialNum(incSerialNum()); + const_cast<FeedOperation &>(op).setSerialNum(incSerialNum()); } - _tlsWriter.storeOperation(op); + _tlsWriter.storeOperation(op, std::move(onDone)); } -void FeedHandler::tlsPrune(SerialNum oldest_to_keep) { +void +FeedHandler::storeOperationSync(const FeedOperation &op) { + vespalib::Gate gate; + storeOperation(op, std::make_shared<search::GateCallback>(gate)); + gate.await(); +} + +void +FeedHandler::tlsPrune(SerialNum oldest_to_keep) { if (!_tlsWriter.erase(oldest_to_keep)) { throw IllegalStateException(make_string("Failed to prune TLS to token %" PRIu64 ".", oldest_to_keep)); } @@ -533,7 +533,7 @@ FeedHandler::handleMove(MoveOperation &op, std::shared_ptr<search::IDestructorCa assert(op.getValidDbdId()); assert(op.getValidPrevDbdId()); assert(op.getSubDbId() != op.getPrevSubDbId()); - storeOperation(op); + storeOperation(op, moveDoneCtx); _activeFeedView->handleMove(op, std::move(moveDoneCtx)); } @@ -577,7 +577,7 @@ performPruneRemovedDocuments(PruneRemovedDocumentsOperation &pruneOp) { const LidVectorContext::SP lids_to_remove = pruneOp.getLidsToRemove(); if (lids_to_remove && lids_to_remove->getNumLids() != 0) { - storeOperation(pruneOp); + storeOperationSync(pruneOp); _activeFeedView->handlePruneRemovedDocuments(pruneOp); } } diff --git a/searchcore/src/vespa/searchcore/proton/server/feedhandler.h b/searchcore/src/vespa/searchcore/proton/server/feedhandler.h index d717346883a..8c28fcdc1ea 100644 --- a/searchcore/src/vespa/searchcore/proton/server/feedhandler.h +++ b/searchcore/src/vespa/searchcore/proton/server/feedhandler.h @@ -65,7 +65,7 @@ private: _tls_mgr(tls_mgr), _tlsDirectWriter(tlsDirectWriter) { } - void storeOperation(const FeedOperation &op) override; + void storeOperation(const FeedOperation &op, DoneCallback onDone) override; bool erase(SerialNum oldest_to_keep) override; SerialNum sync(SerialNum syncTo) override; }; @@ -234,7 +234,8 @@ public: void eof() override; void performPruneRemovedDocuments(PruneRemovedDocumentsOperation &pruneOp) override; void syncTls(SerialNum syncTo); - void storeOperation(FeedOperation &op) override; + void storeOperation(const FeedOperation &op, DoneCallback onDone) override; + void storeOperationSync(const FeedOperation & op); void considerDelayedPrune(); }; diff --git a/searchcore/src/vespa/searchcore/proton/server/i_operation_storer.h b/searchcore/src/vespa/searchcore/proton/server/i_operation_storer.h index 4e5958cd9e2..760250844a8 100644 --- a/searchcore/src/vespa/searchcore/proton/server/i_operation_storer.h +++ b/searchcore/src/vespa/searchcore/proton/server/i_operation_storer.h @@ -2,6 +2,8 @@ #pragma once +#include <vespa/searchlib/transactionlog/common.h> + namespace proton { class FeedOperation; @@ -11,12 +13,13 @@ class FeedOperation; */ struct IOperationStorer { - virtual ~IOperationStorer() {} + using DoneCallback = search::transactionlog::Writer::DoneCallback; + virtual ~IOperationStorer() = default; /** * Assign serial number to (if not set) and store the given operation. */ - virtual void storeOperation(FeedOperation &op) = 0; + virtual void storeOperation(const FeedOperation &op, DoneCallback onDone) = 0; }; } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/server/lid_space_compaction_job.cpp b/searchcore/src/vespa/searchcore/proton/server/lid_space_compaction_job.cpp index d6c1a032cea..2ae8d826ebc 100644 --- a/searchcore/src/vespa/searchcore/proton/server/lid_space_compaction_job.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/lid_space_compaction_job.cpp @@ -5,7 +5,8 @@ #include "imaintenancejobrunner.h" #include "lid_space_compaction_job.h" #include <vespa/searchcore/proton/common/eventlogger.h> -#include <vespa/searchlib/common/idestructorcallback.h> +#include <vespa/searchlib/common/gatecallback.h> +#include <vespa/vespalib/util/sync.h> #include <cassert> #include <vespa/log/log.h> @@ -55,8 +56,9 @@ LidSpaceCompactionJob::scanDocuments(const LidUsageStats &stats) return true; } else { MoveOperation::UP op = _handler.createMoveOperation(document, stats.getLowestFreeLid()); - _opStorer.storeOperation(*op); - _handler.handleMove(*op, _moveOpsLimiter->beginOperation()); + search::IDestructorCallback::SP context = _moveOpsLimiter->beginOperation(); + _opStorer.storeOperation(*op, context); + _handler.handleMove(*op, std::move(context)); if (isBlocked(BlockedReason::OUTSTANDING_OPS)) { return true; } @@ -79,7 +81,9 @@ LidSpaceCompactionJob::compactLidSpace(const LidUsageStats &stats) { uint32_t wantedLidLimit = stats.getHighestUsedLid() + 1; CompactLidSpaceOperation op(_handler.getSubDbId(), wantedLidLimit); - _opStorer.storeOperation(op); + vespalib::Gate gate; + _opStorer.storeOperation(op, std::make_shared<search::GateCallback>(gate)); + gate.await(); _handler.handleCompactLidSpace(op); EventLogger::lidSpaceCompactionComplete(_handler.getName(), wantedLidLimit); _shouldCompactLidSpace = false; diff --git a/searchcore/src/vespa/searchcore/proton/server/tlcproxy.cpp b/searchcore/src/vespa/searchcore/proton/server/tlcproxy.cpp index 215650b6664..bfc59dee35e 100644 --- a/searchcore/src/vespa/searchcore/proton/server/tlcproxy.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/tlcproxy.cpp @@ -11,24 +11,24 @@ using search::transactionlog::Packet; namespace proton { -void TlcProxy::commit(search::SerialNum serialNum, search::transactionlog::Type type, const vespalib::nbostream &buf) +void TlcProxy::commit(search::SerialNum serialNum, search::transactionlog::Type type, + const vespalib::nbostream &buf, DoneCallback onDone) { Packet::Entry entry(serialNum, type, vespalib::ConstBufferRef(buf.c_str(), buf.size())); Packet packet; packet.add(entry); packet.close(); - _tlsDirectWriter.commit(_domain, packet); - + _tlsDirectWriter.commit(_domain, packet, std::move(onDone)); } void -TlcProxy::storeOperation(const FeedOperation &op) +TlcProxy::storeOperation(const FeedOperation &op, DoneCallback onDone) { nbostream stream; op.serialize(stream); LOG(debug, "storeOperation(): serialNum(%" PRIu64 "), type(%u), size(%zu)", op.getSerialNum(), (uint32_t)op.getType(), stream.size()); - commit(op.getSerialNum(), (uint32_t)op.getType(), stream); + commit(op.getSerialNum(), (uint32_t)op.getType(), stream, std::move(onDone)); } } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/server/tlcproxy.h b/searchcore/src/vespa/searchcore/proton/server/tlcproxy.h index 8e4feb2f354..2dc6501731e 100644 --- a/searchcore/src/vespa/searchcore/proton/server/tlcproxy.h +++ b/searchcore/src/vespa/searchcore/proton/server/tlcproxy.h @@ -8,18 +8,20 @@ namespace proton { class FeedOperation; class TlcProxy { - vespalib::string _domain; - search::transactionlog::Writer & _tlsDirectWriter; + using DoneCallback = search::transactionlog::Writer::DoneCallback; + using Writer = search::transactionlog::Writer; + vespalib::string _domain; + Writer & _tlsDirectWriter; - void commit(search::SerialNum serialNum, search::transactionlog::Type type, const vespalib::nbostream &buf); + void commit(search::SerialNum serialNum, search::transactionlog::Type type, + const vespalib::nbostream &buf, DoneCallback onDone); public: typedef std::unique_ptr<TlcProxy> UP; - TlcProxy(const vespalib::string & domain, search::transactionlog::Writer & writer) + TlcProxy(const vespalib::string & domain, Writer & writer) : _domain(domain), _tlsDirectWriter(writer) {} - void storeOperation(const FeedOperation &op); + void storeOperation(const FeedOperation &op, DoneCallback onDone); }; } // namespace proton - diff --git a/searchcore/src/vespa/searchcore/proton/server/tlswriter.h b/searchcore/src/vespa/searchcore/proton/server/tlswriter.h index 0956c0ae011..5d51580c0ad 100644 --- a/searchcore/src/vespa/searchcore/proton/server/tlswriter.h +++ b/searchcore/src/vespa/searchcore/proton/server/tlswriter.h @@ -2,19 +2,17 @@ #pragma once +#include "i_operation_storer.h" #include <vespa/searchlib/common/serialnum.h> namespace proton { -class FeedOperation; - /** * Interface for writing to the TransactionLogServer. */ -struct TlsWriter { - virtual ~TlsWriter() {} +struct TlsWriter : public IOperationStorer { + virtual ~TlsWriter() = default; - virtual void storeOperation(const FeedOperation &op) = 0; virtual bool erase(search::SerialNum oldest_to_keep) = 0; virtual search::SerialNum sync(search::SerialNum syncTo) = 0; }; diff --git a/searchlib/src/vespa/searchlib/common/CMakeLists.txt b/searchlib/src/vespa/searchlib/common/CMakeLists.txt index b1f71303449..f9db738528c 100644 --- a/searchlib/src/vespa/searchlib/common/CMakeLists.txt +++ b/searchlib/src/vespa/searchlib/common/CMakeLists.txt @@ -13,6 +13,7 @@ vespa_add_library(searchlib_common OBJECT featureset.cpp fileheadercontext.cpp foregroundtaskexecutor.cpp + gatecallback.cpp growablebitvector.cpp indexmetainfo.cpp location.cpp diff --git a/searchlib/src/vespa/searchlib/common/gatecallback.cpp b/searchlib/src/vespa/searchlib/common/gatecallback.cpp new file mode 100644 index 00000000000..a853909be71 --- /dev/null +++ b/searchlib/src/vespa/searchlib/common/gatecallback.cpp @@ -0,0 +1,12 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "gatecallback.h" +#include <vespa/vespalib/util/sync.h> + +namespace search { + +GateCallback::~GateCallback() { + _gate.countDown(); +} + +} diff --git a/searchlib/src/vespa/searchlib/common/gatecallback.h b/searchlib/src/vespa/searchlib/common/gatecallback.h new file mode 100644 index 00000000000..1e85d796089 --- /dev/null +++ b/searchlib/src/vespa/searchlib/common/gatecallback.h @@ -0,0 +1,24 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include "idestructorcallback.h" + +namespace vespalib { class Gate; } + +namespace search { + +class GateCallback : public IDestructorCallback { +public: + GateCallback(vespalib::Gate & gate) : _gate(gate) {} + ~GateCallback() override; +private: + vespalib::Gate & _gate; +}; + +class IgnoreCallback : public IDestructorCallback { +public: + IgnoreCallback() { } + ~IgnoreCallback() override = default; +}; + +} // namespace search diff --git a/searchlib/src/vespa/searchlib/common/idestructorcallback.h b/searchlib/src/vespa/searchlib/common/idestructorcallback.h index 4c42f68f0e4..77adba7a4cc 100644 --- a/searchlib/src/vespa/searchlib/common/idestructorcallback.h +++ b/searchlib/src/vespa/searchlib/common/idestructorcallback.h @@ -3,8 +3,7 @@ #include <memory> -namespace search -{ +namespace search { /** * Interface for class that performs a callback when instance is @@ -17,7 +16,7 @@ class IDestructorCallback { public: using SP = std::shared_ptr<IDestructorCallback>; - virtual ~IDestructorCallback() { } + virtual ~IDestructorCallback() = default; }; } // namespace search diff --git a/searchlib/src/vespa/searchlib/transactionlog/common.h b/searchlib/src/vespa/searchlib/transactionlog/common.h index 65ef8f363c0..db8b9727daa 100644 --- a/searchlib/src/vespa/searchlib/transactionlog/common.h +++ b/searchlib/src/vespa/searchlib/transactionlog/common.h @@ -2,6 +2,7 @@ #pragma once #include <vespa/searchlib/common/serialnum.h> +#include <vespa/searchlib/common/idestructorcallback.h> #include <vespa/vespalib/objects/nbostream.h> #include <vespa/vespalib/util/buffer.h> @@ -90,8 +91,9 @@ int makeDirectory(const char * dir); class Writer { public: + using DoneCallback = std::shared_ptr<IDestructorCallback>; virtual ~Writer() { } - virtual void commit(const vespalib::string & domainName, const Packet & packet) = 0; + virtual void commit(const vespalib::string & domainName, const Packet & packet, DoneCallback done) = 0; }; } diff --git a/searchlib/src/vespa/searchlib/transactionlog/translogserver.cpp b/searchlib/src/vespa/searchlib/transactionlog/translogserver.cpp index ca17457bdb9..e793aafd38f 100644 --- a/searchlib/src/vespa/searchlib/transactionlog/translogserver.cpp +++ b/searchlib/src/vespa/searchlib/transactionlog/translogserver.cpp @@ -453,8 +453,9 @@ void TransLogServer::domainStatus(FRT_RPCRequest *req) } } -void TransLogServer::commit(const vespalib::string & domainName, const Packet & packet) +void TransLogServer::commit(const vespalib::string & domainName, const Packet & packet, DoneCallback done) { + (void) done; Domain::SP domain(findDomain(domainName)); if (domain) { domain->commit(packet); diff --git a/searchlib/src/vespa/searchlib/transactionlog/translogserver.h b/searchlib/src/vespa/searchlib/transactionlog/translogserver.h index 92832786059..c12e37dd1c8 100644 --- a/searchlib/src/vespa/searchlib/transactionlog/translogserver.h +++ b/searchlib/src/vespa/searchlib/transactionlog/translogserver.h @@ -31,7 +31,7 @@ public: virtual ~TransLogServer(); DomainStats getDomainStats() const; - void commit(const vespalib::string & domainName, const Packet & packet) override; + void commit(const vespalib::string & domainName, const Packet & packet, DoneCallback done) override; class Session diff --git a/travis/travis-build-cpp.sh b/travis/travis-build-cpp.sh index d5ac94fa6b9..f1a9954ed92 100755 --- a/travis/travis-build-cpp.sh +++ b/travis/travis-build-cpp.sh @@ -8,8 +8,13 @@ source ${SOURCE_DIR}/travis/cpp-prelude.sh BUILD_DIR=~/build mkdir "${BUILD_DIR}" +cd ${SOURCE_DIR} +./boostrap.sh java +mvn install --no-snapshot-updates --batch-mode --threads ${NUM_THREADS} \ + -Dmaven.test.skip=true -Dmaven.javadoc.skip=true -Dmaven.source.skip=true cd ${BUILD_DIR} -bash ${SOURCE_DIR}/bootstrap-cpp.sh ${SOURCE_DIR} ${BUILD_DIR} +bash ${SOURCE_DIR}/bootstrap-cmake.sh ${SOURCE_DIR} make -j ${NUM_THREADS} +make install ctest3 --output-on-failure -j ${NUM_THREADS} ccache --show-stats diff --git a/travis/travis-build-install.sh b/travis/travis-build-install.sh deleted file mode 100755 index 6469a346045..00000000000 --- a/travis/travis-build-install.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/bash -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -set -e - -source /source/travis/prelude.sh -source ${SOURCE_DIR}/travis/cpp-prelude.sh - -cd ${SOURCE_DIR} -sh ./bootstrap.sh java -mvn install --no-snapshot-updates --batch-mode --threads ${NUM_THREADS} --quiet\ - -Dmaven.test.skip=true -Dmaven.javadoc.skip=true -Dmaven.source.skip=true -bash ${SOURCE_DIR}/bootstrap-cmake.sh ${SOURCE_DIR} "-DEXCLUDE_TESTS_FROM_ALL=TRUE" -make -j ${NUM_THREADS} -make install diff --git a/vespalib/src/vespa/vespalib/util/sync.h b/vespalib/src/vespa/vespalib/util/sync.h index f961c280174..86e0a227c72 100644 --- a/vespalib/src/vespa/vespalib/util/sync.h +++ b/vespalib/src/vespa/vespalib/util/sync.h @@ -710,7 +710,7 @@ public: /** * Empty. Needs to be virtual to reduce compiler warnings. **/ - virtual ~CountDownLatch() {} + virtual ~CountDownLatch() = default; }; |