diff options
author | Henning Baldersheim <balder@yahoo-inc.com> | 2018-10-24 14:03:39 +0200 |
---|---|---|
committer | GitHub <noreply@github.com> | 2018-10-24 14:03:39 +0200 |
commit | 62a74d8034b80ba25390168b6e938d73535cdaee (patch) | |
tree | ca9141c1f049f799ea8388e3b67ecb577108ab44 | |
parent | 44b01e669ee64940c5db3b531594853bfd7dce17 (diff) | |
parent | eb9e7b8c2868f25683cb66a4e69a92fde3ceec60 (diff) |
Merge pull request #7435 from vespa-engine/balder/stabilize-test-with-race-condition
Balder/stabilize test with race condition
12 files changed, 139 insertions, 103 deletions
diff --git a/documentapi/src/main/java/com/yahoo/documentapi/DocumentIdResponse.java b/documentapi/src/main/java/com/yahoo/documentapi/DocumentIdResponse.java index 939879c5448..1f70cfdaaf0 100644 --- a/documentapi/src/main/java/com/yahoo/documentapi/DocumentIdResponse.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/DocumentIdResponse.java @@ -12,11 +12,12 @@ import com.yahoo.document.DocumentId; public class DocumentIdResponse extends Response { /** The document id of this response, if any */ - private DocumentId documentId = null; + private final DocumentId documentId; /** Creates a successful response */ public DocumentIdResponse(long requestId) { super(requestId); + documentId = null; } /** @@ -37,6 +38,7 @@ public class DocumentIdResponse extends Response { */ public DocumentIdResponse(long requestId, String textMessage, boolean success) { super(requestId, textMessage, success); + documentId = null; } /** diff --git a/documentapi/src/main/java/com/yahoo/documentapi/DocumentResponse.java b/documentapi/src/main/java/com/yahoo/documentapi/DocumentResponse.java index dab4f2e7855..983b4a0d5b2 100644 --- a/documentapi/src/main/java/com/yahoo/documentapi/DocumentResponse.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/DocumentResponse.java @@ -2,7 +2,6 @@ package com.yahoo.documentapi; import com.yahoo.document.Document; -import com.yahoo.component.Version; /** * The asynchronous response to a document put or get operation. @@ -13,11 +12,12 @@ import com.yahoo.component.Version; public class DocumentResponse extends Response { /** The document of this response, if any */ - private Document document = null; + private final Document document; /** Creates a successful response */ public DocumentResponse(long requestId) { super(requestId); + document = null; } /** @@ -38,6 +38,7 @@ public class DocumentResponse extends Response { */ public DocumentResponse(long requestId, String textMessage, boolean success) { super(requestId, textMessage, success); + document = null; } /** diff --git a/documentapi/src/main/java/com/yahoo/documentapi/Response.java b/documentapi/src/main/java/com/yahoo/documentapi/Response.java index 39b77c40e1d..4d8cb93d1d0 100644 --- a/documentapi/src/main/java/com/yahoo/documentapi/Response.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/Response.java @@ -11,9 +11,9 @@ package com.yahoo.documentapi; */ public class Response { - private long requestId; - private String textMessage = null; - private boolean success = true; + private final long requestId; + private final String textMessage; + private final boolean success; /** Creates a successful response containing no information */ public Response(long requestId) { diff --git a/documentapi/src/main/java/com/yahoo/documentapi/local/LocalAsyncSession.java b/documentapi/src/main/java/com/yahoo/documentapi/local/LocalAsyncSession.java index d365407f407..041c8187a97 100644 --- a/documentapi/src/main/java/com/yahoo/documentapi/local/LocalAsyncSession.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/local/LocalAsyncSession.java @@ -31,7 +31,6 @@ public class LocalAsyncSession implements AsyncSession { private final List<Response> responses = new LinkedList<>(); private final ResponseHandler handler; - private final LocalDocumentAccess access; private final SyncSession syncSession; private long requestId = 0; private Random random = new Random(); @@ -42,7 +41,6 @@ public class LocalAsyncSession implements AsyncSession { } public LocalAsyncSession(AsyncParameters params, LocalDocumentAccess access) { - this.access = access; this.handler = params.getResponseHandler(); random.setSeed(System.currentTimeMillis()); syncSession = access.createSyncSession(new SyncParameters.Builder().build()); diff --git a/documentapi/src/main/java/com/yahoo/documentapi/local/LocalDocumentAccess.java b/documentapi/src/main/java/com/yahoo/documentapi/local/LocalDocumentAccess.java index c37e871005c..202929130c7 100644 --- a/documentapi/src/main/java/com/yahoo/documentapi/local/LocalDocumentAccess.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/local/LocalDocumentAccess.java @@ -3,10 +3,21 @@ package com.yahoo.documentapi.local; import com.yahoo.document.Document; import com.yahoo.document.DocumentId; -import com.yahoo.documentapi.*; +import com.yahoo.documentapi.AsyncParameters; +import com.yahoo.documentapi.AsyncSession; +import com.yahoo.documentapi.DocumentAccess; +import com.yahoo.documentapi.DocumentAccessParams; +import com.yahoo.documentapi.SubscriptionParameters; +import com.yahoo.documentapi.SubscriptionSession; +import com.yahoo.documentapi.SyncParameters; +import com.yahoo.documentapi.SyncSession; +import com.yahoo.documentapi.VisitorDestinationParameters; +import com.yahoo.documentapi.VisitorDestinationSession; +import com.yahoo.documentapi.VisitorParameters; +import com.yahoo.documentapi.VisitorSession; -import java.util.LinkedHashMap; import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; /** * The main class of the local implementation of the document api @@ -15,7 +26,7 @@ import java.util.Map; */ public class LocalDocumentAccess extends DocumentAccess { - Map<DocumentId, Document> documents = new LinkedHashMap<DocumentId, Document>(); + Map<DocumentId, Document> documents = new ConcurrentHashMap<>(); public LocalDocumentAccess(DocumentAccessParams params) { super(params); diff --git a/documentapi/src/main/java/com/yahoo/documentapi/local/LocalSyncSession.java b/documentapi/src/main/java/com/yahoo/documentapi/local/LocalSyncSession.java index b1776b8c96d..45df2015da4 100755 --- a/documentapi/src/main/java/com/yahoo/documentapi/local/LocalSyncSession.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/local/LocalSyncSession.java @@ -5,13 +5,10 @@ import com.yahoo.document.Document; import com.yahoo.document.DocumentId; import com.yahoo.document.DocumentPut; import com.yahoo.document.DocumentRemove; -import com.yahoo.document.DocumentType; import com.yahoo.document.DocumentUpdate; -import com.yahoo.document.TestAndSetCondition; import com.yahoo.documentapi.Response; -import com.yahoo.documentapi.Result; import com.yahoo.documentapi.SyncSession; -import com.yahoo.documentapi.messagebus.protocol.*; +import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol; /** * @author bratseth diff --git a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/test/Destination.java b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/test/Destination.java index c18bfc7597a..ac2db526689 100644 --- a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/test/Destination.java +++ b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/test/Destination.java @@ -7,9 +7,22 @@ import com.yahoo.documentapi.DocumentAccessParams; import com.yahoo.documentapi.SyncParameters; import com.yahoo.documentapi.SyncSession; import com.yahoo.documentapi.local.LocalDocumentAccess; -import com.yahoo.documentapi.messagebus.protocol.*; -import com.yahoo.messagebus.*; +import com.yahoo.documentapi.messagebus.protocol.DocumentMessage; +import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol; +import com.yahoo.documentapi.messagebus.protocol.GetDocumentMessage; +import com.yahoo.documentapi.messagebus.protocol.GetDocumentReply; +import com.yahoo.documentapi.messagebus.protocol.PutDocumentMessage; +import com.yahoo.documentapi.messagebus.protocol.RemoveDocumentMessage; +import com.yahoo.documentapi.messagebus.protocol.UpdateDocumentMessage; +import com.yahoo.messagebus.DestinationSession; +import com.yahoo.messagebus.EmptyReply; import com.yahoo.messagebus.Error; +import com.yahoo.messagebus.ErrorCode; +import com.yahoo.messagebus.Message; +import com.yahoo.messagebus.MessageHandler; +import com.yahoo.messagebus.Protocol; +import com.yahoo.messagebus.RPCMessageBus; +import com.yahoo.messagebus.Reply; import com.yahoo.messagebus.network.Identity; import com.yahoo.messagebus.network.rpc.RPCNetworkParams; diff --git a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/test/MessageBusDocumentApiTestCase.java b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/test/MessageBusDocumentApiTestCase.java index bdac2112263..bb84b6f0104 100644 --- a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/test/MessageBusDocumentApiTestCase.java +++ b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/test/MessageBusDocumentApiTestCase.java @@ -2,7 +2,10 @@ package com.yahoo.documentapi.messagebus.test; import com.yahoo.document.select.parser.ParseException; -import com.yahoo.documentapi.*; +import com.yahoo.documentapi.DocumentAccess; +import com.yahoo.documentapi.ProgressToken; +import com.yahoo.documentapi.VisitorParameters; +import com.yahoo.documentapi.VisitorSession; import com.yahoo.documentapi.messagebus.MessageBusDocumentAccess; import com.yahoo.documentapi.messagebus.MessageBusParams; import com.yahoo.documentapi.messagebus.protocol.CreateVisitorReply; @@ -19,10 +22,6 @@ import org.junit.After; import org.junit.Before; import org.junit.Test; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.TimeUnit; - -import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; /** diff --git a/documentapi/src/test/java/com/yahoo/documentapi/test/AbstractDocumentApiTestCase.java b/documentapi/src/test/java/com/yahoo/documentapi/test/AbstractDocumentApiTestCase.java index 6160c9deca6..2b56d9c628d 100644 --- a/documentapi/src/test/java/com/yahoo/documentapi/test/AbstractDocumentApiTestCase.java +++ b/documentapi/src/test/java/com/yahoo/documentapi/test/AbstractDocumentApiTestCase.java @@ -6,7 +6,15 @@ import com.yahoo.document.DocumentId; import com.yahoo.document.DocumentPut; import com.yahoo.document.DocumentRemove; import com.yahoo.document.DocumentType; -import com.yahoo.documentapi.*; +import com.yahoo.documentapi.AsyncParameters; +import com.yahoo.documentapi.AsyncSession; +import com.yahoo.documentapi.DocumentAccess; +import com.yahoo.documentapi.DocumentResponse; +import com.yahoo.documentapi.Response; +import com.yahoo.documentapi.ResponseHandler; +import com.yahoo.documentapi.Result; +import com.yahoo.documentapi.SyncParameters; +import com.yahoo.documentapi.SyncSession; import org.junit.Test; import java.util.ArrayList; @@ -16,7 +24,10 @@ import java.util.List; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; -import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; /** * These tests should work with all implementations (who choose to implement these features) To test a certain diff --git a/messagebus/src/main/java/com/yahoo/messagebus/Sequencer.java b/messagebus/src/main/java/com/yahoo/messagebus/Sequencer.java index 6a99614fa4e..b804c776969 100644 --- a/messagebus/src/main/java/com/yahoo/messagebus/Sequencer.java +++ b/messagebus/src/main/java/com/yahoo/messagebus/Sequencer.java @@ -19,7 +19,7 @@ public class Sequencer implements MessageHandler, ReplyHandler { private final AtomicBoolean destroyed = new AtomicBoolean(false); private final MessageHandler sender; - private final Map<Long, Queue<Message>> seqMap = new HashMap<Long, Queue<Message>>(); + private final Map<Long, Queue<Message>> seqMap = new HashMap<>(); /** * Constructs a new sequencer on top of the given async sender. diff --git a/vespaclient-core/src/main/java/com/yahoo/feedhandler/ThreadedFeedAccess.java b/vespaclient-core/src/main/java/com/yahoo/feedhandler/ThreadedFeedAccess.java new file mode 100644 index 00000000000..3ad3e0b7f42 --- /dev/null +++ b/vespaclient-core/src/main/java/com/yahoo/feedhandler/ThreadedFeedAccess.java @@ -0,0 +1,82 @@ +package com.yahoo.feedhandler; + +import com.yahoo.concurrent.ThreadFactoryFactory; +import com.yahoo.document.Document; +import com.yahoo.document.DocumentId; +import com.yahoo.document.DocumentUpdate; +import com.yahoo.document.TestAndSetCondition; +import com.yahoo.feedapi.SimpleFeedAccess; + +import java.util.concurrent.Executor; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.SynchronousQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; + +final class ThreadedFeedAccess implements SimpleFeedAccess { + + private final SimpleFeedAccess simpleFeedAccess; + private final ExecutorService executorService; + private final Executor executor; + ThreadedFeedAccess(int numThreads, SimpleFeedAccess simpleFeedAccess) { + this.simpleFeedAccess = simpleFeedAccess; + if (numThreads <= 0) { + numThreads = Runtime.getRuntime().availableProcessors(); + } + if (numThreads > 1) { + executorService = new ThreadPoolExecutor(numThreads, numThreads, 0L, TimeUnit.SECONDS, + new SynchronousQueue<>(false), + ThreadFactoryFactory.getDaemonThreadFactory("feeder"), + new ThreadPoolExecutor.CallerRunsPolicy()); + executor = executorService; + } else { + executorService = null; + executor = new Executor() { + @Override + public void execute(Runnable command) { + command.run(); + } + }; + } + } + @Override + public void put(Document doc) { + executor.execute(() -> simpleFeedAccess.put(doc)); + } + + @Override + public void remove(DocumentId docId) { + executor.execute(() -> simpleFeedAccess.remove(docId)); + } + + @Override + public void update(DocumentUpdate update) { + executor.execute(() -> simpleFeedAccess.update(update)); + } + + @Override + public void put(Document doc, TestAndSetCondition condition) { + executor.execute(() -> simpleFeedAccess.put(doc, condition)); + } + + @Override + public void remove(DocumentId docId, TestAndSetCondition condition) { + executor.execute(() -> simpleFeedAccess.remove(docId, condition)); + } + + @Override + public void update(DocumentUpdate update, TestAndSetCondition condition) { + executor.execute(() -> simpleFeedAccess.update(update, condition)); + } + + @Override + public boolean isAborted() { + return simpleFeedAccess.isAborted(); + } + @Override + public void close() { + if (executorService != null) { + executorService.shutdown(); + } + } +} diff --git a/vespaclient-core/src/main/java/com/yahoo/feedhandler/VespaFeedHandler.java b/vespaclient-core/src/main/java/com/yahoo/feedhandler/VespaFeedHandler.java index c94cc10b098..32c2d848b82 100755 --- a/vespaclient-core/src/main/java/com/yahoo/feedhandler/VespaFeedHandler.java +++ b/vespaclient-core/src/main/java/com/yahoo/feedhandler/VespaFeedHandler.java @@ -5,15 +5,10 @@ import com.google.inject.Inject; import com.yahoo.clientmetrics.RouteMetricSet; import com.yahoo.cloud.config.ClusterListConfig; import com.yahoo.cloud.config.SlobroksConfig; -import com.yahoo.concurrent.ThreadFactoryFactory; import com.yahoo.container.jdisc.EmptyResponse; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.container.protect.Error; -import com.yahoo.document.Document; -import com.yahoo.document.DocumentId; -import com.yahoo.document.DocumentUpdate; -import com.yahoo.document.TestAndSetCondition; import com.yahoo.document.config.DocumentmanagerConfig; import com.yahoo.feedapi.DocprocMessageProcessor; import com.yahoo.feedapi.FeedContext; @@ -29,11 +24,6 @@ import com.yahoo.vespaclient.config.FeederConfig; import java.util.List; import java.util.concurrent.Executor; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.SynchronousQueue; -import java.util.concurrent.ThreadPoolExecutor; -import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; @@ -77,74 +67,6 @@ public final class VespaFeedHandler extends VespaFeedHandlerBase { return handle(request, null, 1); } - static final class ThreadedFeedAccess implements SimpleFeedAccess { - - private final SimpleFeedAccess simpleFeedAccess; - private final ExecutorService executorService; - private final Executor executor; - ThreadedFeedAccess(int numThreads, SimpleFeedAccess simpleFeedAccess) { - this.simpleFeedAccess = simpleFeedAccess; - if (numThreads <= 0) { - numThreads = Runtime.getRuntime().availableProcessors(); - } - if (numThreads > 1) { - executorService = new ThreadPoolExecutor(numThreads, numThreads, 0L, TimeUnit.SECONDS, - new SynchronousQueue<>(false), - ThreadFactoryFactory.getDaemonThreadFactory("feeder"), - new ThreadPoolExecutor.CallerRunsPolicy()); - executor = executorService; - } else { - executorService = null; - executor = new Executor() { - @Override - public void execute(Runnable command) { - command.run(); - } - }; - } - } - @Override - public void put(Document doc) { - executor.execute(() -> simpleFeedAccess.put(doc)); - } - - @Override - public void remove(DocumentId docId) { - executor.execute(() -> simpleFeedAccess.remove(docId)); - } - - @Override - public void update(DocumentUpdate update) { - executor.execute(() -> simpleFeedAccess.update(update)); - } - - @Override - public void put(Document doc, TestAndSetCondition condition) { - executor.execute(() -> simpleFeedAccess.put(doc, condition)); - } - - @Override - public void remove(DocumentId docId, TestAndSetCondition condition) { - executor.execute(() -> simpleFeedAccess.remove(docId, condition)); - } - - @Override - public void update(DocumentUpdate update, TestAndSetCondition condition) { - executor.execute(() -> simpleFeedAccess.update(update, condition)); - } - - @Override - public boolean isAborted() { - return simpleFeedAccess.isAborted(); - } - @Override - public void close() { - if (executorService != null) { - executorService.shutdown(); - } - } - } - public HttpResponse handle(HttpRequest request, RouteMetricSet.ProgressCallback callback, int numThreads) { if (request.getProperty("status") != null) { return new MetricResponse(context.getMetrics().getMetricSet()); |